学习步骤:
开发技术栈:
Hadoop是一个适合海量数据存储与计算的平台。是基于Google的GoogleFS、Map Reduce、BigTable实现的。
架构分析:
架构分析:
架构分析:
192.168.56.101
cent7-1
cent7-1 localhost
systemctl status firewalld.service
systemctl stop firewalld
systemctl status firewalld.service
ssh-keygen -t rsa
cd /root
cd .ssh/
cat id_rsa
cat id_rsa.pub >> authorized_keys
ssh cent7-1
tar -zxvf jdk-8u191-linux-x64.tar.gz
vi /etc/profile
source /etc/profile
# profile配置内容
export JAVA_HOME=/home/jdk8
export PATH=.:$JAVA_HOME/bin:$PATH
# 解压Hadoop
tar -zxvf hadoop-3.2.4.tar.gz
# 进入配置文件目录
cd /home/hadoop-3.2.4/etc/hadoop
vi core-site.xml
vi hdfs-site.xml
<property>
<name>fs.defaultFSname>
<value>hdfs://cent7-1:9000value>
<final>truefinal>
property>
<property>
<name>hadoop.tmp.dirname>
<value>/home/hadoop_repovalue>
property>
<property>
<name>dfs.replicationname>
<value>1value>
property>
<property>
<name>mapreduce.framework.namename>
<value>yarnvalue>
property>
<property>
<name>yarn.nodemanager.aux-servicesname>
<value>mapreduce_shufflevalue>
property>
<property>
<name>yarn.nodemanager.env-whitelistname>
<value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOMEvalue>
property>
export JAVA_HOME=/home/jdk8
export HADOOP_LOG_DIR=/home/hadoop_repo/logs/hadoop
# 在Hadoop的目录下执行以下命令,
bin/hdfs namenode -format
看到以下内容说明执行成功!注意:hdfs格式化只能执行一次,如果失败需要删除文件夹后再进行格式化。
[root@cent7-1 hadoop-3.2.4]# sbin/start-all.sh
Starting namenodes on [cent7-1]
ERROR: Attempting to operate on hdfs namenode as root
ERROR: but there is no HDFS_NAMENODE_USER defined. Aborting operation.
Starting datanodes
ERROR: Attempting to operate on hdfs datanode as root
ERROR: but there is no HDFS_DATANODE_USER defined. Aborting operation.
Starting secondary namenodes [cent7-1]
ERROR: Attempting to operate on hdfs secondarynamenode as root
ERROR: but there is no HDFS_SECONDARYNAMENODE_USER defined. Aborting operation.
Starting resourcemanager
ERROR: Attempting to operate on yarn resourcemanager as root
ERROR: but there is no YARN_RESOURCEMANAGER_USER defined. Aborting operation.
Starting nodemanagers
ERROR: Attempting to operate on yarn nodemanager as root
ERROR: but there is no YARN_NODEMANAGER_USER defined. Aborting operation.
# 提示缺少hdfs、yarn的用户信息
vi sbin/start-dfs.sh
vi sbin/stop-dfs.sh
#增加配置
HDFS_DATANODE_USER=root
HADOOP_SECURE_DN_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root
vi sbin/start-yarn.sh
vi sbin/stop-yarn.sh
#增加配置
YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=yarn
YARN_NODEMANAGER_USER=root
[root@cent7-1 hadoop-3.2.4]# sbin/start-all.sh
WARNING: HADOOP_SECURE_DN_USER has been replaced by HDFS_DATANODE_SECURE_USER. Using value of HADOOP_SECURE_DN_USER.
Starting namenodes on [cent7-1]
上一次登录:三 8月 30 19:05:12 CST 2023从 192.168.56.1pts/1 上
Starting datanodes
上一次登录:三 8月 30 21:02:51 CST 2023pts/0 上
localhost: Warning: Permanently added 'localhost' (ECDSA) to the list of known hosts.
Starting secondary namenodes [cent7-1]
上一次登录:三 8月 30 21:02:56 CST 2023pts/0 上
Starting resourcemanager
上一次登录:三 8月 30 21:03:49 CST 2023从 192.168.56.1pts/3 上
Starting nodemanagers
上一次登录:三 8月 30 21:04:13 CST 2023pts/0 上
[root@cent7-1 hadoop-3.2.4]# jps
10146 NameNode
10386 DataNode
10883 SecondaryNameNode
11833 ResourceManager
12954 Jps
12155 NodeManager
# 展示除了jps外的五个Hadoop组件进程表示启动成功
sbin/stop-all.sh
yum install mysql
apache-hive-hive-3.1.3安装包下载_开源镜像站-阿里云 (aliyun.com)
source /etc/profile
export HIVE_HOME=/home/hive
export PATH=$HIVE_HOME/bin:$PATH
配置hive/conf/hive-site.xml文件
<configuration>
<property>
<name>javax.jdo.option.ConnectionURLname>
<value>jdbc:mysql://cent7-1:3306/hive?createDatabaseIfNotExist=true&useSSL=falsevalue>
property>
<property>
<name>javax.jdo.option.ConnectionDriverNamename>
<value>com.mysql.jdbc.Drivervalue>
property>
<property>
<name>javax.jdo.option.ConnectionUserNamename>
<value>hdpvalue>
property>
<property>
<name>javax.jdo.option.ConnectionPasswordname>
<value>adminvalue>
property>
<property>
<name>hive.server2.authenticationname>
<value>CUSTOMvalue>
property>
<property>
<name>hive.server2.custom.authentication.classname>
<value>com.ylw.CustomHiveServer2Authvalue>
property>
<property>
<name>hive.server2.custom.authentication.filename>
<value>/home/hive/user.pwd.confvalue>
property>
<property>
<name>hive.jdbc_passwd.auth.rootname>
<value>adminvalue>
property>
<property>
<name>hive.metastore.portname>
<value>9083value>
<description>Hive metastore listener portdescription>
property>
<property>
<name>hive.server2.thrift.portname>
<value>10000value>
<description>Port number of HiveServer2 Thrift interface when hive.server2.transport.mode is 'binary'.description>
property>
<property>
<name>hive.server2.thrift.max.worker.threadsname>
<value>200value>
property>
<property>
<name>hive.metastore.localname>
<value>falsevalue>
<description>controls whether to connect to remote metastore server or open a new metastore server in Hive Client JVMdescription>
property>
<property>
<name>hive.server2.transport.modename>
<value>binaryvalue>
<description>
Expects one of [binary, http].
Transport mode of HiveServer2.
description>
property>
configuration>
nohup hive --server metastore &
nohup hive --service hiveserver2 &
jps
#看到是否有两个runJar ,如果有说明启动成功
# 查看端口占用
netstat -anop |grep 10000
ps -aux|grep hive