hadoop环境搭建

阅读更多

192.168.23.231 server1
192.168.23.234 server2
192.168.23.239 server3


http://mirrors.163.com/centos/6/isos/


关闭防火墙
service iptables stop
chkconfig iptables off
停闭selinux
vim /etc/sysconfig/selinux
SELINUX=disables


export JAVA_HOME=/usr/jdk1.8.0_25
export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin

BASE_HOME=/myhome/usr
JAVA_HOME=/usr/jdk1.8.0_25
CLASSPATH=$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JAVA_HOME/jre/lib/rt.jar
PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH
export PATH

ssh-keygen -t rsa
cat id_rsa.pub>> authorized_keys
ssh [email protected] cat ~/.ssh/id_rsa.pub>> authorized_keys
ssh [email protected] cat ~/.ssh/id_rsa.pub>> authorized_keys

scp authorized_keys root@server2:~/.ssh/
scp authorized_keys root@server3:~/.ssh/


mkdir -p /myhome/usr#已安装的目录
mkdir -p /myhome/root
mkdir -p /myhome/tmp
mkdir -p /myhome/src
mkdir -p /myhome/logs
mkdir -p /myhome/dev
mkdir -p /myhome/bin
mkdir -p /myhome/usr/hadoop/yarn/local
mkdir -p /myhome/usr/hadoop/yarn/logs
mkdir -p /myhome/usr/hadoop/journal/zjenterprise
mkdir -p /myhome/usr/hadoop/hdfs/name
mkdir -p /myhome/usr/hadoop/hdfs/data
mkdir -p /myhome/usr/hadoop/tmp
mkdir -p /myhome/usr/hadoop/journal/
mkdir -p /myhome/usr/zookeeper/data/


touch /myhome/usr/zookeeper/data/myid
echo 1 > /myhome/usr/zookeeper/data/myid


core-site.xml

   
        fs.defaultFS
        hdfs://192.168.23.231:9000
   
   
        hadoop.tmp.dir
        file:/myhome/usr/hadoop/tmp
   
   
        io.file.buffer.size
        131702
   


hdfs-site.xml

   
        dfs.namenode.name.dir
        file:/myhome/usr/hadoop/dfs/name
   
   
        dfs.datanode.data.dir
        file:/myhome/usr/hadoop/dfs/data
   
   
        dfs.replication
        2
   
   
        dfs.namenode.secondary.http-address
        192.168.23.231:9001
   
   
    dfs.webhdfs.enabled
    true
   


mapred-site.xml




   
        mapreduce.framework.name
        yarn
   
   
        mapreduce.jobhistory.address
        192.168.23.231:10020
   
   
        mapreduce.jobhistory.webapp.address
        192.168.23.231:19888
   



yarn-site.xml



   
        yarn.nodemanager.aux-services
        mapreduce_shuffle
   
   
        yarn.nodemanager.auxservices.mapreduce.shuffle.class
        org.apache.hadoop.mapred.ShuffleHandler
   
   
        yarn.resourcemanager.address
        192.168.23.231:8032
   
   
        yarn.resourcemanager.scheduler.address
        192.168.23.231:8030
   
   
        yarn.resourcemanager.resource-tracker.address
        192.168.23.231:8031
   
   
        yarn.resourcemanager.admin.address
        192.168.23.231:8033
   
   
        yarn.resourcemanager.webapp.address
        192.168.23.231:8088
   
   
        yarn.nodemanager.resource.memory-mb
        768
   



scp -r /myhome/usr/hadoop 192.168.23.234:/myhome/usr
scp -r /myhome/usr/hadoop 192.168.23.239:/myhome/usr
在Master服务器启动hadoop，从节点会自动启动，进入/home/hadoop/hadoop-2.7.0目录
(1)初始化，输入命令，bin/hdfs namenode -format
(2)全部启动sbin/start-all.sh，也可以分开sbin/start-dfs.sh、sbin/start-yarn.sh
(3)停止的话，输入命令，sbin/stop-all.sh
(4)输入命令，jps，可以看到相关信息