华为云服务器centos7.2下hadoop伪分布式wordcount运行实例

1.jdk配置
[root@ecs-f5cc ~]#  vim /etc/profile
JAVA_HOME=/usr/local/src/java/jdk
PATH=$JAVA_HOME/bin:$PATH
CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export JAVA_HOME  PATH CLASSPATH
export HADOOP_HOME=/usr/local/src/app/hadoop-2.6.0-cdh5.12.0
export PATH=$PATH:$HADOOP_HOME/bin
HIVE_HOME=/usr/local/src/app/hive-1.1.0-cdh5.12.0
export PATH=$PATH:$HIVE_HOME/bin
[root@ecs-f5cc ~]#  source /etc/profile

2.ssh免密登陆
[root@ecs-5198 ~]#  vim /etc/ssh/ssh_config
StrictHostKeyChecking no
UserKnownHostsFile /dev/null
[root@ecs-5198 ~]#  systemctl restart sshd.service
免密登陆
[root@ecs-5198 ~]# ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
[root@ecs-5198 ~]# cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
#这一句我自己没用,也是可以的
[root@ecs-5198 ~]# chmod 0600 ~/.ssh/authorized_keys

3.下载CDH
可更改版本
http://archive.cloudera.com/cdh5/cdh/5/hadoop-2.6.0-cdh5.12.0.tar.gz


4.上传解压jdk, hadoop
[root@ecs-f5cc ~]# cd /usr/local/src
[root@ecs-f5cc src]# mkdir app java software
[root@ecs-f5cc src]# ls
app  java  software
[root@ecs-f5cc software]# ls
hadoop-2.6.0-cdh5.12.0.tar.gz   jdk-8u131-linux-x64.tar.gz
[root@ecs-f5cc software]#  tar -zxvf jdk-8u131-linux-x64.tar.gz -C ../java/
[root@ecs-f5cc software]# tar -zxvf hadoop-2.6.0-cdh5.12.0.tar.gz -C ../app/
[root@ecs-dce7 software]# tar -zxvf hive-1.1.0-cdh5.12.0.tar.gz -C ../app/
[root@ecs-f5cc software]# cd ..
[root@ecs-f5cc src]# cd java/
[root@ecs-f5cc java]# ls
jdk1.8.0_131
[root@ecs-f5cc java]# mv jdk1.8.0_131 jdk
[root@ecs-f5cc java]# cd ..
[root@ecs-f5cc src]# cd app/
[root@ecs-f5cc app]# ls
hadoop-2.6.0-cdh5.12.0 
[root@ecs-f5cc app]# cd hadoop-2.6.0-cdh5.12.0/
[root@ecs-f5cc hadoop-2.6.0-cdh5.12.0]# cd etc/hadoop
[root@ecs-f5cc hadoop]# vim hadoop-env.sh 
export JAVA_HOME=/usr/local/src/java/jdk
[root@ecs-f5cc hadoop]# vim core-site.xml 

    
        fs.defaultFS
        hdfs://localhost:8020
    
    
        hadoop.tmp.dir
        /usr/local/src/app/tmp
    

[root@ecs-f5cc hadoop]# vim hdfs-site.xml 

    
        dfs.replication
        1
    

[root@ecs-f5cc hadoop]# vim yarn-site.xml

    
        yarn.nodemanager.aux-services
        mapreduce_shuffle
    

[root@ecs-f5cc hadoop]# cp mapred-site.xml.template  mapred-site.xml
[root@ecs-f5cc hadoop]# vim mapred-site.xml

    
        mapreduce.framework.name
        yarn
    


5.启动

[root@ecs-f5cc hadoop]# cd ..
[root@ecs-f5cc etc]# cd ..
[root@ecs-f5cc hadoop-2.6.0-cdh5.12.0]# bin/hdfs namenode -format
[root@ecs-f5cc hadoop-2.6.0-cdh5.12.0]# sbin/start-dfs.sh
[root@ecs-f5cc hadoop-2.6.0-cdh5.12.0]# sbin/start-yarn.sh
[root@ecs-f5cc hadoop-2.6.0-cdh5.12.0]# jps
18995 Jps
12372 NodeManager
12085 ResourceManager
5481 NameNode
5597 DataNode
5757 SecondaryNameNode
[root@ecs-f5cc hadoop-2.6.0-cdh5.12.0]# hadoop fs -mkdir -p /input/wordcount/
[root@ecs-f5cc hadoop-2.6.0-cdh5.12.0]# vim hello.txt
I am a chinese.
I love China.
I am proud of China.
[root@ecs-f5cc hadoop-2.6.0-cdh5.12.0]# hadoop fs -put hello.txt  /input/wordcount/
[root@ecs-f5cc hadoop-2.6.0-cdh5.12.0]# 


6.运行
[root@ecs-f5cc hadoop-2.6.0-cdh5.12.0]# ls share/hadoop/mapreduce/ |grep example
#运行这个自带的例子
hadoop-mapreduce-examples-2.6.0-cdh5.12.0.jar
lib-examples

[root@ecs-f5cc hadoop-2.6.0-cdh5.12.0]# hadoop jar /usr/local/src/app/hadoop-2.6.0-cdh5.12.0/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.0-cdh5.12.0.jar   wordcount  /input/wordcount/hello.txt  /output/wordcount/

[root@ecs-f5cc hadoop-2.6.0-cdh5.12.0]# hadoop fs -ls /output/wordcount/
Found 2 items
-rw-r--r--   1 root supergroup          0 2017-07-21 21:22 /output/wordcount/_SUCCESS
-rw-r--r--   1 root supergroup         53 2017-07-21 21:22 /output/wordcount/part-r-00000
[root@ecs-f5cc hadoop-2.6.0-cdh5.12.0]# hadoop fs -cat /output/wordcount/part-r-00000
China.  2
I   3
a   1
am  2
chinese.    1
love    1
of  1
proud   1
[root@ecs-f5cc hadoop-2.6.0-cdh5.12.0]# 


7.stop
[root@ecs-f5cc hadoop-2.6.0-cdh5.12.0]# sbin/stop-all.sh

补充:

HADOOP_HOME=/usr/local/src/app/hadoop-2.6.0-cdh5.12.0

hive-site.xml




        javax.jdo.option.ConnectionURL
        jdbc:mysql://localhost:3306/sparksql?createDatabaseIfNotExist=true
    
    
        javax.jdo.option.ConnectionDriverName
        com.mysql.jdbc.Driver
    
    
        javax.jdo.option.ConnectionUserName
        root
    
    
        javax.jdo.option.ConnectionPassword
        root