安装JDK
[server@hadoop102 software]$ tar -zxvf jdk-8u65-linux-x64.tar.gz -C /opt/module/
[server@hadoop102 software]$ cd /opt/module/
[server@hadoop102 module]$ cd jdk1.8.0_65/
[server@hadoop102 jdk1.8.0_65]$ pwd
/opt/module/jdk1.8.0_65
[server@hadoop102 software]$ cd /opt/module/
[server@hadoop102 module]$ cd jdk1.8.0_65/
[server@hadoop102 jdk1.8.0_65]$ pwd
/opt/module/jdk1.8.0_65
[server@hadoop102 profile.d]$ sudo vim my_env.sh
#JAVA_HOME
export JAVA_HOME=/opt/module/jdk1.8.0_65
export PATH=$PATH:$JAVA_HOME/bin
[server@hadoop102 profile.d]$ source /etc/profile
安装Hadoop
[server@hadoop102 profile.d]$ cd /opt/software/
[server@hadoop102 software]$ tar -zxvf hadoop-3.1.3.tar.gz -C /opt/module/
[server@hadoop102 software]$ cd /opt/module/hadoop-3.1.3/
[server@hadoop102 hadoop-3.1.3]$ pwd
/opt/module/hadoop-3.1.3
[server@hadoop102 hadoop-3.1.3]$ sudo vim /etc/profile.d/my_env.sh
#JAVA_HOME
export JAVA_HOME=/opt/module/jdk1.8.0_65
export PATH=$PATH:$JAVA_HOME/bin
#HADOOP
export HADOOP_HOME=/opt/module/hadoop-3.1.3
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin
[server@hadoop102 hadoop-3.1.3]$ source /etc/profile
[server@hadoop102 hadoop-3.1.3]$ pwd
/opt/module/hadoop-3.1.3
[server@hadoop102 hadoop-3.1.3]$ mkdir wcinput
[server@hadoop102 hadoop-3.1.3]$ cd wcinput/
[server@hadoop102 wcinput]$ vim word.txt
hadoop yarn
hadoop mapreduce
server
server
[server@hadoop102 wcinput]$ cd ..
[server@hadoop102 hadoop-3.1.3]$ pwd
/opt/module/hadoop-3.1.3
[server@hadoop102 module]$ scp -r jdk1.8.0_65/ server@hadoop103:/opt/module/
The authenticity of host 'hadoop103 (192.168.100.103)' can't be established.
ECDSA key fingerprint is SHA256:UiwjKSR7xNbDOtrjmdcTRAPJ+4XpLhq324M3gwLV4QQ.
ECDSA key fingerprint is MD5:68:ee:49:6c:67:a9:44:05:b6:ad:f3:09:53:d0:17:48.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'hadoop103,192.168.100.103' (ECDSA) to the list of known hosts.
server@hadoop103's password:
[server@hadoop103 module]$ scp -r server@hadoop102:/opt/module/hadoop-3.1.3/ ./
The authenticity of host 'hadoop102 (192.168.100.102)' can't be established.
ECDSA key fingerprint is SHA256:UiwjKSR7xNbDOtrjmdcTRAPJ+4XpLhq324M3gwLV4QQ.
ECDSA key fingerprint is MD5:68:ee:49:6c:67:a9:44:05:b6:ad:f3:09:53:d0:17:48.
Are you sure you want to continue connecting (yes/no)? yes
[server@hadoop103 ~]$ scp -r server@hadoop102:/opt/module/* server@hadoop104:/opt/module/
server@hadoop102's password:
server@hadoop104's password:
编写分发脚本
[server@hadoop102 ~]$ pwd
/home/server
[server@hadoop102 ~]$ mkdir bin
[server@hadoop102 ~]$ cd bin
[server@hadoop102 bin]$ pwd
/home/server/bin
[server@hadoop102 bin]$ vim xsync
#!/bin/bash
#1. 判断参数个数
if [ $# -lt 1 ]
then
echo Not Enough Argument!
exit;
fi
#2. 遍历集群所有机器
for host in hadoop102 hadoop103 hadoop104
do
echo ======= $host ======
#3. 遍历所有目录，挨个发送
for file in $@
do
#4. 判断文件是否存在
if [ -e $file ]
then
#5. 获取父目录
pdir=$(cd -P $(dirname $file);pwd)
#6. 获取当前文件的名称
fname=$(basename $file)
ssh $host "mkdir -p $pdir"
rsync -av $pdir/$fname $host:$pdir
else
echo $file does not exits!
fi
done
done
[server@hadoop102 bin]$ chmod 777 xsync
[server@hadoop102 bin]$ cd ..
[server@hadoop102 ~]$ pwd
/home/server
[server@hadoop102 ~]$ xsync bin
[server@hadoop102 ~]$ sudo ./bin/xsync /etc/profile.d/my_env.sh
[server@hadoop103 ~]$ source /etc/profile
[server@hadoop104 ~]$ source /etc/profile
配置ssh免密
[server@hadoop102 .ssh]$ pwd
/home/server/.ssh
[server@hadoop102 .ssh]$ ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/home/server/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /home/server/.ssh/id_rsa.
Your public key has been saved in /home/server/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:mrBuH90QRmLXNINEjiq4uR1B8o2kCLr0ex/yoSHMqrY server@hadoop102
The key's randomart image is:
+---[RSA 2048]----+
| oo=++ |
| . *. .o |
|o o . + |
|+B o . . . |
|=.= + S |
|.++o o + o |
|+ .=.o+o. . |
| +.o+.=.o |
|=Eoooooo |
+----[SHA256]-----+
[server@hadoop102 .ssh]$ ssh-copy-id hadoop103
[server@hadoop102 .ssh]$ ssh-copy-id hadoop104
[server@hadoop102 .ssh]$ ssh-copy-id hadoop102
[server@hadoop103 ~]$ ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/home/server/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /home/server/.ssh/id_rsa.
Your public key has been saved in /home/server/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:S4zSSMvPjybYQV5jC0yLNHsVVsthUoEkUgIu/yXPGWE server@hadoop103
The key's randomart image is:
+---[RSA 2048]----+
|..o.o.==*. |
|. oo.oo+ o |
|.o *.oE o |
|..oo*=++ |
| .+*+=oS |
| .oO.+ . |
| o..* . |
| . o .o |
| o. . |
+----[SHA256]-----+
[server@hadoop103 ~]$ ssh-copy-id hadoop102
[server@hadoop103 ~]$ ssh-copy-id hadoop103
[server@hadoop103 ~]$ ssh-copy-id hadoop104
[server@hadoop104 ~]$ ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/home/server/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /home/server/.ssh/id_rsa.
Your public key has been saved in /home/server/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:TS8Q71wHQEQ9MFKyXOq+pSeBT8tek13kS3rVON0/AHU server@hadoop104
The key's randomart image is:
+---[RSA 2048]----+
| +=X+.. E |
| . O .+.. |
| = o...o |
| . * o.+ oo|
| .S = ..* =|
| ..o + +.+.|
| +.o= o o..|
| ==.. . .|
| .+o |
+----[SHA256]-----+
[server@hadoop104 ~]$ ssh-copy-id hadoop102
[server@hadoop104 ~]$ ssh-copy-id hadoop103
[server@hadoop104 ~]$ ssh-copy-id hadoop104
[server@hadoop102 .ssh]$ su root
密码：
[root@hadoop102 .ssh]# ssh-keygen -t rsa
[root@hadoop102 .ssh]# ssh-copy-id hadoop102
[root@hadoop102 .ssh]# ssh-copy-id hadoop103
[root@hadoop102 .ssh]# ssh-copy-id hadoop104
[server@hadoop103 ~]$ su root
密码：
[root@hadoop103 server]# ssh-keygen -t rsa
[root@hadoop103 server]# ssh-copy-id hadoop102
[root@hadoop103 server]# ssh-copy-id hadoop103
[root@hadoop103 server]# ssh-copy-id hadoop104
[server@hadoop104 ~]$ su root
密码：
[root@hadoop104 server]# ssh-keygen -t rsa
[root@hadoop104 server]# ssh-copy-id hadoop102
[root@hadoop104 server]# ssh-copy-id hadoop103
[root@hadoop104 server]# ssh-copy-id hadoop104
[root@hadoop102 hadoop]# su server
[server@hadoop102 hadoop]$ pwd
/opt/module/hadoop-3.1.3/etc/hadoop
[server@hadoop102 hadoop]$ vim core-site.xml
fs.defaultFS
hdfs://hadoop102:8020
hadoop.tmp.dir
/opt/module/hadoop-3.1.3/data
hadoop.http.staticuser.user
server
[server@hadoop102 hadoop]$ vim hdfs-site.xml
dfs.namenode.http-address
hadoop102:9870
dfs.namenode.secondary.http-address
hadoop104:9868
[server@hadoop102 hadoop]$ vim yarn-site.xml
yarn.nodemanager.aux-services
mapreduce_shuffle
yarn.resourcemanager.hostname
hadoop103
yarn.nodemanager.vmem-check-enabled
false
yarn.nodemanager.env-whitelist
JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME
[server@hadoop102 hadoop]$ vim mapred-site.xml
mapreduce.framework.name
yarn
[server@hadoop102 etc]$ pwd
/opt/module/hadoop-3.1.3/etc
[server@hadoop102 etc]$ xsync hadoop/
[server@hadoop102 hadoop]$ pwd
/opt/module/hadoop-3.1.3/etc/hadoop
[server@hadoop102 hadoop]$ vim workers
hadoop102
hadoop103
hadoop104
[server@hadoop102 hadoop]$ xsync workers
[server@hadoop102 hadoop]$ pwd
/opt/module/hadoop-3.1.3/etc/hadoop
[server@hadoop102 hadoop]$ vim hadoop-env.sh
export HADOOP_CLIENT_OPTS="-Xmx512m"
[server@hadoop102 hadoop-3.1.3]$ pwd
/opt/module/hadoop-3.1.3
[server@hadoop102 hadoop-3.1.3]$ ll
总用量 176
drwxr-xr-x. 2 server server 183 9月 12 2019 bin
drwxr-xr-x. 3 server server 20 9月 12 2019 etc
drwxr-xr-x. 2 server server 106 9月 12 2019 include
drwxr-xr-x. 3 server server 20 9月 12 2019 lib
drwxr-xr-x. 4 server server 288 9月 12 2019 libexec
-rw-rw-r--. 1 server server 147145 9月 4 2019 LICENSE.txt
-rw-rw-r--. 1 server server 21867 9月 4 2019 NOTICE.txt
-rw-rw-r--. 1 server server 1366 9月 4 2019 README.txt
drwxr-xr-x. 3 server server 4096 9月 12 2019 sbin
drwxr-xr-x. 4 server server 31 9月 12 2019 share
drwxrwxr-x. 2 server server 22 7月 31 16:59 wcinput
drwxr-xr-x. 2 server server 88 7月 31 17:06 wcoutput
[server@hadoop102 hadoop-3.1.3]$ hdfs namenode -format
[server@hadoop102 hadoop-3.1.3]$ ll
总用量 176
drwxr-xr-x. 2 server server 183 9月 12 2019 bin
drwxrwxr-x. 3 server server 17 8月 2 22:22 data
drwxr-xr-x. 3 server server 20 9月 12 2019 etc
drwxr-xr-x. 2 server server 106 9月 12 2019 include
drwxr-xr-x. 3 server server 20 9月 12 2019 lib
drwxr-xr-x. 4 server server 288 9月 12 2019 libexec
-rw-rw-r--. 1 server server 147145 9月 4 2019 LICENSE.txt
drwxrwxr-x. 2 server server 39 8月 2 22:22 logs
-rw-rw-r--. 1 server server 21867 9月 4 2019 NOTICE.txt
-rw-rw-r--. 1 server server 1366 9月 4 2019 README.txt
drwxr-xr-x. 3 server server 4096 9月 12 2019 sbin
drwxr-xr-x. 4 server server 31 9月 12 2019 share
drwxrwxr-x. 2 server server 22 7月 31 16:59 wcinput
drwxr-xr-x. 2 server server 88 7月 31 17:06 wcoutput
[server@hadoop102 current]$ pwd
/opt/module/hadoop-3.1.3/data/dfs/name/current
[server@hadoop102 current]$ cat VERSION
#Mon Aug 02 22:22:25 CST 2021
namespaceID=1302361015
clusterID=CID-d0018180-104d-4046-a57a-4404cceec25d
cTime=1627914145631
storageType=NAME_NODE
blockpoolID=BP-642981798-192.168.100.102-1627914145631
layoutVersion=-64
[server@hadoop102 cu
[server@hadoop102 hadoop-3.1.3]$ pwd
/opt/module/hadoop-3.1.3
[server@hadoop102 hadoop-3.1.3]$ sbin/start-dfs.sh
Starting namenodes on [hadoop102]
Starting datanodes
hadoop103: WARNING: /opt/module/hadoop-3.1.3/logs does not exist. Creating.
hadoop104: WARNING: /opt/module/hadoop-3.1.3/logs does not exist. Creating.
Starting secondary namenodes [hadoop104]
[server@hadoop102 hadoop-3.1.3]$ jps
9849 DataNode
10122 Jps
9740 NameNode
[server@hadoop103 hadoop]$ jps
9249 DataNode
9321 Jps
[server@hadoop104 ~]$ jps
9410 Jps
9365 SecondaryNameNode
9254 DataNode
http://hadoop102:9870/
[server@hadoop103 hadoop-3.1.3]$ pwd
/opt/module/hadoop-3.1.3
[server@hadoop103 hadoop-3.1.3]$ sbin/start-yarn.sh
Starting resourcemanager
Starting nodemanagers
[server@hadoop103 hadoop-3.1.3]$ jps
9249 DataNode
9698 Jps
9603 NodeManager
9496 ResourceManager
http://hadoop103:8088
[server@hadoop102 hadoop-3.1.3]$ hadoop fs -mkdir /wcinput
[server@hadoop102 hadoop-3.1.3]$ hadoop fs -put wcinput/word.txt /wcinput
2021-08-02 22:37:55,591 INFO sasl.SaslDataTransferClient: SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false
[server@hadoop104 subdir0]$ pwd
/opt/module/hadoop-3.1.3/data/dfs/data/current/BP-642981798-192.168.100.102-1627914145631/current/finalized/subdir0/subdir0
[server@hadoop104 subdir0]$ ll
总用量 8
-rw-rw-r--. 1 server server 43 8月 2 22:37 blk_1073741825
-rw-rw-r--. 1 server server 11 8月 2 22:37 blk_1073741825_1001.meta
[server@hadoop104 subdir0]$ cat blk_1073741825
hadoop yarn
hadoop mapreduce
server
server
[server@hadoop103 subdir0]$ pwd
/opt/module/hadoop-3.1.3/data/dfs/data/current/BP-642981798-192.168.100.102-1627914145631/current/finalized/subdir0/subdir0
[server@hadoop103 subdir0]$ ll
总用量 8
-rw-rw-r--. 1 server server 43 8月 2 22:37 blk_1073741825
-rw-rw-r--. 1 server server 11 8月 2 22:37 blk_1073741825_1001.meta
[server@hadoop103 subdir0]$ cat blk_1073741825
hadoop yarn
hadoop mapreduce
server
server
[server@hadoop102 hadoop-3.1.3]$ cd /opt/module/hadoop-3.1.3/data/dfs/data/current/BP-642981798-192.168.100.102-1627914145631/current/finalized/subdir0/subdir0
[server@hadoop102 subdir0]$ ll
总用量 8
-rw-rw-r--. 1 server server 43 8月 2 22:37 blk_1073741825
-rw-rw-r--. 1 server server 11 8月 2 22:37 blk_1073741825_1001.meta
[server@hadoop102 subdir0]$ cat blk_1073741825
hadoop yarn
hadoop mapreduce
server
server
[server@hadoop102 hadoop-3.1.3]$ hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-3.1.3.jar wordcount /wcinput /wcoutput