Hadoop 3.2.0配置

~/.bashrc

export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
export HADOOP_HOME=/home/qianwen/hadoop-3.2.0
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME

hadoop-env.sh

export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
export HADOOP_HOME=/home/qianwen/hadoop-3.2.0
export HADOOP_CONF_DIR=/home/qianwen/hadoop-3.2.0/etc/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME


for file in /home/qianwen/hadoop-3.2.0/share/hadoop/*/*.jar
do
       export CLASSPATH=$CLASSPATH:$file
done

for file in /home/qianwen/hadoop-3.2.0/share/hadoop/*/lib/*.jar
do
       export CLASSPATH=$CLASSPATH:$file
done

core-site.xml

       
               hadoop.tmp.dir
               /home/qianwen/hadoop-3.2.0/tmp
       


       
               fs.defaultFS
               hdfs://falcon-1:9000
       

       
           topology.script.file.name
           /home/qianwen/hadoop-3.2.0/etc/hadoop/rack_topology.sh
       

mapred-site.xml

       
               mapreduce.framework.name
               yarn
       

       
                 yarn.app.mapreduce.am.env //必须要有
                 HADOOP_MAPRED_HOME=/home/qianwen/hadoop-3.2.0
       

       
                 mapreduce.map.env //必须要有
                 HADOOP_MAPRED_HOME=/home/qianwen/hadoop-3.2.0
       

       
                 mapreduce.reduce.env  //必须要有
                 HADOOP_MAPRED_HOME=/home/qianwen/hadoop-3.2.0
       

yarn-site.xml

        
                yarn.resourcemanager.address
                falcon-1:18040
        

        
                yarn.resourcemanager.scheduler.address
                falcon-1:18030
        

        
                yarn.resourcemanager.webapp.address
                falcon-1:18088
        

        
                yarn.resourcemanager.resource-tracker.address
                falcon-1:18025
        

        
                yarn.resourcemanager.admin.address
                falcon-1:18141
        

        
                yarn.nodemanager.aux-services
                mapreduce_shuffle
        

        
                yarn.nodemanager.aux-services.mapreduce.shuffle.class
                org.apache.hadoop.mapred.ShuffleHandler
        

hdfs-site.xml

        
                dfs.replication
                2
        

        
                dfs.namenode.name.dir
                /home/qianwen/hadoop-3.2.0/dfs/name
        

        
                dfs.datanode.data.dir
                /home/qianwen/hadoop-3.2.0/dfs/data
        

workers

falcon-2
falcon-3
falcon-4
cat-2
cat-4

rack_topology.sh

#!/bin/bash
HADOOP_CONF=/home/qianwen/hadoop-3.2.0/etc/hadoop
#while [ $# -gt 0 ] ; do
  nodeArg=$1
  exec<${HADOOP_CONF}/rack_topology.data
  result=""
  while read line ; do
    ar=( $line )
    if [ "${ar[0]}" = "$nodeArg" ]; then
      result="${ar[1]}"
    fi
  done
  shift
  if [ -z "$result" ] ; then
    echo -n "/default-rack"
  else
    echo -n "$result"
  fi
# done

rack_topology.data

10.102.2.33 /rack01
10.102.2.41 /rack01
10.102.2.49 /rack01
10.102.2.35 /rack02
10.102.2.54 /rack02