CDH5.11

export JAVA_HOME=/usr/local/jdk1.8.0_191
export M2_HOME=/usr/local/maven
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export JRE_HOME=$JAVA_HOME/jre
export SCALA_HOME=/usr/local/scala-2.11.6
export HBASE_HOME=/opt/cloudera/parcels/CDH-5.11.1-1.cdh5.11.1.p0.4/lib/hbase
export HADOOP_HOME=/opt/cloudera/parcels/CDH-5.11.1-1.cdh5.11.1.p0.4/
export HIVE_HOME=/opt/cloudera/parcels/CDH-5.11.1-1.cdh5.11.1.p0.4/lib/hive
export ZOOKEEPER_HOME=/opt/cloudera/parcels/CDH-5.11.1-1.cdh5.11.1.p0.4/lib/zookeeper
export HADOOP_CMD=/opt/cloudera/parcels/CDH-5.11.1-1.cdh5.11.1.p0.4/lib/hadoop/bin/hadoop
export HCAT_HOME=/opt/cloudera/parcels/CDH-5.11.1-1.cdh5.11.1.p0.4/lib/hive-hcatalog
export KYLIN_HOME=/usr/local/kylin
export HADOOP_CONF_DIR=/etc/hadoop/conf
export HIVE_CONF=/etc/hive/conf
export MYSQL_HOME=/usr/local/mysql-5.6.43
export SQOOP_HOME=/usr/local/sqoop
export SPARK_HOME=/opt/cloudera/parcels/SPARK2/lib/spark2
export PATH=${PATH}:${JAVA_HOME}/bin:${PATH}:${KYLIN_HOME}/bin:${PATH}:${MYSQL_HOME}/bin:$SQOOP_HOME/bin:$M2_HOME/bin:$SCALA_HOME/bin:$SPARK_HOME/bin


 
#vim /etc/hosts
#vim /etc/hostname
#vim /etc/sysconfig/network
#HOSTNAME=name
#systemctl stop firewalld
#systemctl disable firewalld
#rpm -qa | grep maria 
#rpm -e --nodeps
#重启

    ssh-keygen -t rsa
   ssh-copy-id  -p 20822 root@worker1
   ssh -p 20822 root@worker1

yum install -y nfs-utils rpcbind
systemctl start rpcbind
systemctl enable rpcbind   
yum -y install vim gcc gcc-c++ make cmake ncurses ncurses-devel man ncurses libxml2 libxml2-devel openssl-devel  bison bison-devel libxml2-devel libxslt-devel perl perl-devel autoconf net-tools psmisc
yum -y install cyrus-sasl-plain cyrus-sasl-devel cyrus-sasl-gssapi lrzsz vim ntp libaio*
yum install -y epel-release
yum install -y jq
systemctl enable ntpd

service ntpd restart
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime


#java
mkdir -p /user/spark/applicationHistory
mkdir -p /user/spark/spark2ApplicationHistory
mkdir -p /usr/share/
mkdir -p /usr/java/
tar -zxvf /usr/local/soft/jdk-8u191-linux-x64.tar.gz -C /usr/local
ln -s /usr/local/jdk1.8.0_191 /usr/local/jdk1.8
ln -s /usr/local/jdk1.8.0_191 /usr/local/java
ln -s /usr/local/jdk1.8.0_191 /usr/share/java
ln -s /usr/local/jdk1.8.0_191 /usr/java/default

#mysql
注意mysql大小写问题
mkdir -p /data/mysql
cp /usr/local/soft/mysql-connector-java.jar /usr/share/java/
mkdir -p /usr/local/mysql-5.6.43 && tar -zxvf /usr/local/soft/mysql-5.6.43.tar.gz -C /usr/local/mysql-5.6.43 --strip-components 1
ln -s /usr/local/mysql-5.6.43 /usr/local/mysql
chown -R root:root /usr/local/mysql-5.6.43
cp /usr/local/soft/my.cnf /etc
cp /usr/local/soft/mysql.server /etc/init.d/mysql
chmod 777 /etc/init.d/mysql

cd /usr/local/mysql/ && ./scripts/mysql_install_db --user=root
service mysql start

use mysql;
update user set password=password("123456");

CREATE DATABASE hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

CREATE DATABASE amon DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

CREATE DATABASE hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

CREATE DATABASE monitor DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

CREATE DATABASE oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'yier34wu6' WITH GRANT OPTION;

FLUSH PRIVILEGES;

service mysql restart

cd /etc/init.d/ && chkconfig --add mysql
chkconfig mysql on
ln -s /usr/local/mysql-5.6.43/lib/libmysqlclient.so.18 /usr/lib64/libmysqlclient.so.18

#CDH

mkdir -p /var/cloudera-scm-server
mkdir -p /opt/cloudera/parcel-repo
mkdir -p /opt/cloudera-manager
cp /usr/local/soft/CDH* /opt/cloudera/parcel-repo/
cp /usr/local/soft/manifest.json /opt/cloudera/parcel-repo/
tar xf /usr/local/soft/cloudera-manager-centos7-cm5.11.1_x86_64.tar.gz -C /opt/cloudera-manager
useradd --system --home=/opt/cloudera-manager/cm-5.11.1/run/cloudera-scm-server --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm
chown cloudera-scm:cloudera-scm /var/cloudera-scm-server
chown cloudera-scm:cloudera-scm /opt/cloudera-manager
chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo
cd /opt/cloudera-manager/cm-5.11.1/share/cmf/schema/ && ./scm_prepare_database.sh mysql -uroot  –p123456 --scm-host localhost scm scm scm
vim  /opt/cloudera-manager/cm-5.11.1/etc/cloudera-scm-agent/config.ini


tar xf /usr/local/soft/apache-kylin-2.6.1-bin-cdh57.tar.gz -C /usr/local/
mv /usr/local/apache-kylin-2.6.1-bin-cdh57/ /usr/local/kylin-2.6.1-bin-cdh57/
ln -s /usr/local/kylin-2.6.1-bin-cdh57/ /usr/local/kylin
mv /usr/local/kylin/conf/kylin.properties  /usr/local/kylin/conf/kylin.properties.ba
mv /usr/local/kylin/conf/setenv.sh  /usr/local/kylin/conf/setenv.sh.ba
cp /usr/local/soft/setenv.sh /usr/local/kylin/conf/
chmod 777 /usr/local/kylin/conf/setenv.sh
mv /usr/local/soft/kylin.properties /usr/local/kylin/conf/
vim /usr/local/soft/kylin.properties

tar xf /usr/local/soft/sqoop-1.4.6-cdh5.10.0.tar.gz -C /opt/
ln -s /opt/sqoop-1.4.6-cdh5.10.0/ /usr/local/sqoop
cp /usr/local/soft/sqoop-env.sh /usr/local/sqoop/conf
chmod 777 /usr/local/sqoop/conf/sqoop-env.sh
64M  小文件太多 注意这三个目录大小 /dfs /var /tmp
/opt/cloudera-manager/cm-5.11.1/etc/init.d/cloudera-scm-server start
/opt/cloudera-manager/cm-5.11.1/etc/init.d/cloudera-scm-agent start
tail -fn 200 /opt/cloudera-manager/cm-5.10.0/log/cloudera-scm-server/cloudera-scm-server.log

tail -fn 200 /opt/cloudera-manager/cm-5.10.0/log/cloudera-scm-agent/cloudera-scm-agent.log

cp /usr/java/default/mysql-connector-java.jar /opt/sqoop-1.4.6-cdh5.10.0/lib/

yum  install -y git
git config --global user.name "zck"
git config --global user.email "[email protected]"
git config --global credential.helper store
cd /usr/local && wget http://mirrors.hust.edu.cn/apache/maven/maven-3/3.1.1/binaries/apache-maven-3.1.1-bin.tar.gz
tar zxf apache-maven-3.1.1-bin.tar.gz && mv apache-maven-3.1.1 /usr/local/maven




添加spark2.1 目录下面不能有其他的jar  集群停止
https://blog.csdn.net/high2011/article/details/79138493
mkdir -p /opt/cloudera/csd
mv /usr/local/soft/spark-2.3/SPARK2_ON_YARN-2.3.0.cloudera3.jar /opt/cloudera/csd
chown cloudera-scm:cloudera-scm   /opt/cloudera/csd
chgrp cloudera-scm SPARK2_ON_YARN-2.3.0.cloudera3.jar
chown cloudera-scm SPARK2_ON_YARN-2.3.0.cloudera3.jar
需要的parcel放入到parcel_repo
parcel URL添加
http://archive.cloudera.com/spark2/parcels/2.3.0.cloudera3/



# mount /dev/sda1 /storage      挂载新硬盘
# vi /etc/fstab      设置系统自动挂载硬盘，加入一行：
/dev/sda1               /storage            ext4    defaults        0 0