软件 | 版本 | 备注 |
---|---|---|
hadoop | hadoop-3.3.4 | hdfs基础 |
spark | spark-3.2.4-bin-hadoop3.2 | 计算框架 |
zookeeper | apache-zookeeper-3.5.7-bin | 分布式服务器 |
hbase | hbase-2.4.11 | 列式存储 |
hive | apache-hive-3.1.3-bin | 数仓元数据 |
su - hadoop
-- 启动hadoop和spark
exec-hadoop.sh start
exec-hadoop.sh stop
-- 启动数据库
systemctl start docker // pgsql hive
-- 启动hive
nohup hive --service metastore >> /export/server/hive/logs/metastore.log 2>&1 &
-- 启动hbase
/export/server/hbase/bin/start-hbase.sh
cd /export/server/hbase && bin/stop-hbase.sh
lsb_release -a
cat /etc/redhat-release
hostnamectl set-hostname <新主机名>
/etc/hostname
hostname
node1
node2
node3
参考安装:
https://docs.docker.com/engine/install/centos/#installation-methods
systemctl enable docker.service
systemctl is-enabled docker.service
https://github.com/docker/compose/releases/tag/v2.17.2
mv docker-compose-linux-x86_64 /usr/local/bin/docker-compose
chmod +x docker-compose
docker-compose version
参考:https://blog.csdn.net/jinying_51eqhappy/article/details/131956003
adduser hadoop
passwd hadoop
cd /export/server/hadoop/etc/hadoop
mkdir -p /export/server/hadoop/logs
mkdir -p /export/server/hadoop/tmpdata
vim workers
node1
node2
node3
vim hadoop-env.sh
export JAVA_HOME=/export/server/jdk
export HADOOP_HOME=/export/server/hadoop
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export HADOOP_LOG_DIR=$HADOOP_HOME/logs
export HADOOP_PID_DIR=/home/hadoop/tmp
export HADOOP_SECURE_PID_DIR=/home/hadoop/tmp
<configuration>
<property>
<name>fs.defaultFSname>
<value>hdfs://node2:8020value>
property>
<property>
<name>hadoop.tmp.dirname>
<value>/export/server/hadoop/tmpdatavalue>
property>
<property>
<name>hadoop.http.staticuser.username>
<value>hadoopvalue>
property>
<property>
<name>hadoop.proxyuser.hadoop.hostsname>
<value>*value>
property>
<property>
<name>hadoop.proxyuser.hadoop.groupsname>
<value>*value>
property>
configuration>
<configuration>
<property>
<name>dfs.namenode.http-addressname>
<value>node2:9870value>
property>
<property>
<name>dfs.namenode.secondary.http-addressname>
<value>node1:9868value>
property>
configuration>
export JAVA_HOME=/export/server/jdk
export HADOOP_HOME=/export/server/hadoop
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export HADOOP_LOG_DIR=$HADOOP_HOME/logs
export HADOOP_PID_DIR=/home/hadoop/tmp
export HADOOP_SECURE_PID_DIR=/home/hadoop/tmp
export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}
export JAVA_HOME=/export/server/jdk
export JAVA_HOME=/export/server/jdk
export HADOOP_HOME=/export/server/hadoop
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export HADOOP_LOG_DIR=$HADOOP_HOME/logs
<configuration>
<property>
<name>mapreduce.framework.namename>
<value>yarnvalue>
<description>Execution framework set to Hadoop YARN.description>
property>
<property>
<name>mapreduce.jobhistory.addressname>
<value>node2:10020value>
property>
<property>
<name>mapreduce.jobhistory.webapp.addressname>
<value>node2:19888value>
property>
<property>
<name>yarn.app.mapreduce.am.envname>
<value>HADOOP_MAPRED_HOME=$HADOOP_HOMEvalue>
<description>MAPREDUCE home设置为HADOOP_HOMEdescription>
property>
<property>
<name>mapreduce.map.envname>
<value>HADOOP_MAPRED_HOME=$HADOOP_HOMEvalue>
<description>MAPREDUCE HOME 设置为HADOOP_HOMEdescription>
property>
<property>
<name>mapreduce.reduce.envname>
<value>HADOOP_MAPRED_HOME=$HADOOP_HOMEvalue>
<description>mAPREDUCE HOME 设置为HADOOP_HOMEdescription>
property>
configuration>
<configuration>
<property>
<name>yarn.nodemanager.aux-servicesname>
<value>mapreduce_shufflevalue>
property>
<property>
<name>yarn.resourcemanager.hostnamename>
<value>node1value>
property>
<property>
<name>yarn.log-aggregation-enablename>
<value>truevalue>
property>
<property>
<name>yarn.log.server.urlname>
<value>http://node2:19888/jobhistory/logsvalue>
property>
<property>
<name>yarn.log-aggregation.retain-secondsname>
<value>604800value>
property>
configuration>
vim /etc/profile
export JAVA_HOME=/export/server/jdk
export PATH=$PATH:$JAVA_HOME/bin
export HADOOP_HOME=/export/server/hadoop
export PYSPARK_PYTHON=/export/server/anaconda3/bin
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export SPARK_HOME=/export/server/spark
export HBASE_HOME=/export/server/hbase
export HIVE_HOME=/export/server/hive
export PATH=$PATH:$HADOOP_HOME/bin:$PYSPARK_PYTHON
export PATH=$PATH:$HADOOP_HOME/sbin:$HBASE_HOME/bin:$HIVE_HOME/bin:$SPARK_HOME/bin
mkdir -p /home/hadoop/bin && cd /home/hadoop/bin
vim xsync
#!/bin/bash
#1. 判断参数个数
if [ $# -lt 1 ]
then
echo Not Arguement
exit;
fi
#2. 遍历集群所有机器
for host in node1 node2 node3
do
echo ==================== $host ====================
#3. 遍历所有目录,挨个发送
for file in $@
do
#4. 判断文件是否存在
if [ -e $file ]
then
#5. 获取父目录
pdir=$(cd -P $(dirname $file); pwd)
#6. 获取当前文件的名称
fname=$(basename $file)
ssh $host "mkdir -p $pdir"
rsync -av $pdir/$fname $host:$pdir
else
echo $file does not exists!
fi
done
done
添加执行权限
chmod +x xsync
同步Hadoop
cd /export/server
sudo /home/hadoop/bin/xsync hadoop
vim /export/server/hadoop/sbin/jpsall.sh
#!/bin/bash
for host in node1 node2 node3
do
echo =============== $host ===============
ssh $host $JAVA_HOME/bin/jps
done
添加执行权限
chmod +x jpsall.sh
hadoop namenode -format
配置 /etc/profile 的hive同上;
cd hadoop/etc/hadoop
vim core-site.xml
<property>
<name>hadoop.proxyuser.hadoop.hostsname>
<value>*value>
property>
<property>
<name>hadoop.proxyuser.hadoop.groupsname>
<value>*value>
property>
同上
cd hive/conf
mv hive-env.sh.template hive-env.sh
vim hive-env.sh
export HADOOP_HOME=/export/server/hadoop
export HIVE_CONF_DIR=/export/server/hive/conf
export HIVE_AUX_JARS_PATH=/export/server/hive/lib
vim hive-site.xml
<configuration>
<property>
<name>javax.jdo.option.ConnectionURLname>
<value>jdbc:postgresql://node2:5432/hivedata?createDatabaseIfNotExist=truevalue>
<description>JDBC connect string for a JDBC metastoredescription>
property>
<property>
<name>javax.jdo.option.ConnectionDriverNamename>
<value>org.postgresql.Drivervalue>
<description>Driver class name for a JDBC metastoredescription>
property>
<property>
<name>javax.jdo.option.ConnectionUserNamename>
<value>hiveuservalue>
<description>username to use against metastore databasedescription>
property>
<property>
<name>javax.jdo.option.ConnectionPasswordname>
<value>123456value>
<description>password to use against metastore databasedescription>
property>
<property>
<name>hive.server2.thrift.bind.hostname>
<value>node2value>
property>
<property>
<name>hive.metastore.urisname>
<value>thrift://node2:9083value>
property>
<property>
<name>hive.metastore.event.db.notification.api.authname>
<value>falsevalue>
property>
configuration>
cd /export/server/hive/bin
./schematool -dbType postgres -initSchema
mkdir -p /export/server/hive/logs
后台启动
>nohup hive --service metastore >> /bigdata/server/hive/logs/metastore.log 2>&1 &
>hive 交互式操作
hadoop fs -mkdir /sparklog
/ect/profile配置同上;
node1
node2
node3
#!/usr/bin/env bash
JAVA_HOME=/export/server/jdk
HADOOP_CONF_DIR=/export/server/hadoop/etc/hadoop
YARN_CONF_DIR=/export/server/hadoop/etc/hadoop
export SPARK_MASTER_HOST=node2
export SPARK_MASTER_PORT=7077
SPARK_MASTER_WEBUI_PORT=8080
SPARK_WORKER_CORES=1
SPARK_WORKER_MEMORY=1g
SPARK_WORKER_PORT=7078
SPARK_WORKER_WEBUI_PORT=8081
SPARK_HISTORY_OPTS="-Dspark.history.fs.logDirectory=hdfs://node2:8020/sparklog/ -Dspark.history.fs.cleaner.enabled=true"
<configuration>
<property>
<name>hive.metastore.warehouse.dirname>
<value>/user/hive/warehousevalue>
property>
<property>
<name>hive.metastore.localname>
<value>falsevalue>
property>
<property>
<name>hive.metastore.urisname>
<value>thrift://node2:9083value>
property>
configuration>
cd /export/hadoop/sbin
vim exec-hadoop.sh
#!/bin/bash
if [ $# -lt 1 ]
then
echo "No Args Input..."
exit ;
fi
case $1 in
"start")
echo " =================== 启动 hadoop集群 ==================="
echo " --------------- 启动 hdfs ---------------"
ssh node2 $HADOOP_HOME/sbin/start-dfs.sh
echo " --------------- 启动 yarn ---------------"
ssh node1 $HADOOP_HOME/sbin/start-yarn.sh
echo " --------------- 启动 historyserver ---------------"
ssh node2 $HADOOP_HOME/bin/mapred --daemon start historyserver
echo " --------------- 启动 spark ---------------"
ssh node2 $SPARK_HOME/sbin/start-all.sh
echo " --------------- 启动 spark 历史服务器 ---------------"
ssh node2 $SPARK_HOME/sbin/start-history-server.sh
# ssh node2 /export/server/zookeeper/bin/zk.sh start
;;
"stop")
echo " =================== 关闭 hadoop集群 ==================="
echo " --------------- stop historyserver ---------------"
ssh node2 $HADOOP_HOME/bin/mapred --daemon stop historyserver
echo " --------------- stop yarn ---------------"
ssh node1 $HADOOP_HOME/sbin/stop-yarn.sh
echo " --------------- stop hdfs ---------------"
ssh node2 $HADOOP_HOME/sbin/stop-dfs.sh
echo " --------------- stop spark ---------------"
ssh node2 $SPARK_HOME/sbin/stop-all.sh
echo " --------------- stop spark 历史服务器 ---------------"
ssh node2 $SPARK_HOME/sbin/stop-history-server.sh
# ssh node2 /export/server/zookeeper/bin/zk.sh stop
;;
*)
echo "Input Args Error..."
;;
esac
chmod +x exec-hadoop.sh
exec-hadoop.sh start
exec-hadoop.sh stop
pyspark --master spark://ecs-qar1-0002:7077
pyspark --master yarn
sc.parallelize([1,2,3]).map(lambda x:x*10).collect()
yarn客户端模式
spark-submit --master yarn --deploy-mode client --driver-memory 512m --executor-memory 512m --num-executors 2 --total-executor-cores 2 /bigdata/server/spark/examples/src/main/python/pi.py 10
参考:https://blog.csdn.net/jinying_51eqhappy/article/details/132013638
参考:https://blog.csdn.net/jinying_51eqhappy/article/details/132030894