CDH5.10.0 KYLIN2.6 SPARK2.1 心得

hostname不能有下划线

#先执行了卸载 maria java  cp了环境变量 
export JAVA_HOME=/usr/local/jdk1.8.0_191
export M2_HOME=/usr/local/maven
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export JRE_HOME=$JAVA_HOME/jre
export SCALA_HOME=/usr/local/scala-2.11.6
export HBASE_HOME=/opt/cloudera/parcels/CDH-5.10.0-1.cdh5.10.0.p0.41/lib/hbase
export HADOOP_HOME=/opt/cloudera/parcels/CDH-5.10.0-1.cdh5.10.0.p0.41/
export HIVE_HOME=/opt/cloudera/parcels/CDH-5.10.0-1.cdh5.10.0.p0.41/lib/hive
export ZOOKEEPER_HOME=/opt/cloudera/parcels/CDH-5.10.0-1.cdh5.10.0.p0.41/lib/zookeeper
export HADOOP_CMD=/opt/cloudera/parcels/CDH-5.10.0-1.cdh5.10.0.p0.41/lib/hadoop/bin/hadoop
export HCAT_HOME=/opt/cloudera/parcels/CDH-5.10.0-1.cdh5.10.0.p0.41/lib/hive-hcatalog
export KYLIN_HOME=/usr/local/kylin
export HADOOP_CONF_DIR=/etc/hadoop/conf
export HIVE_CONF=/etc/hive/conf
export MYSQL_HOME=/usr/local/mysql-5.6.43
export SQOOP_HOME=/usr/local/sqoop
export PATH=${PATH}:${JAVA_HOME}/bin:${PATH}:${KYLIN_HOME}/bin:${PATH}:${MYSQL_HOME}/bin:$SQOOP_HOME/bin:$M2_HOME/bin:$SCALA_HOME/bin

 
#vim /etc/hosts
#vim /etc/hostname
#vim /etc/sysconfig/network
#HOSTNAME=name
#systemctl stop firewalld
#systemctl disable firewalld
#rpm -qa | grep maria 
#rpm -e --nodeps
#重启


yum -y install gcc gcc-c++ make cmake ncurses ncurses-devel man ncurses libxml2 libxml2-devel openssl-devel  bison bison-devel libxml2-devel libxslt-devel perl perl-devel autoconf net-tools psmisc
yum -y install cyrus-sasl-plain cyrus-sasl-devel cyrus-sasl-gssapi lrzsz vim ntp libaio*
systemctl enable ntpd

service ntpd restart
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime


#java
mkdir -p /user/spark/applicationHistory
mkdir -p /user/spark/spark2ApplicationHistory
mkdir -p /usr/share/
mkdir -p /usr/java/
tar -zxvf /usr/local/soft/jdk-8u191-linux-x64.tar.gz -C /usr/local
ln -s /usr/local/jdk1.8.0_191 /usr/local/jdk1.8
ln -s /usr/local/jdk1.8.0_191 /usr/local/java
ln -s /usr/local/jdk1.8.0_191 /usr/share/java
ln -s /usr/local/jdk1.8.0_191 /usr/java/default

#mysql
mkdir -p /data/mysql
cp /usr/local/soft/mysql-connector-java.jar /usr/share/java/
mkdir -p /usr/local/mysql-5.6.43 && tar -zxvf /usr/local/soft/mysql-5.6.43.tar.gz -C /usr/local/mysql-5.6.43 --strip-components 1
ln -s /usr/local/mysql-5.6.43 /usr/local/mysql
chown -R root:root /usr/local/mysql-5.6.43
cp /usr/local/soft/my.cnf /etc
cp /usr/local/soft/mysql.server /etc/init.d/mysql
chmod 777 /etc/init.d/mysql

cd /usr/local/mysql/ && ./scripts/mysql_install_db --user=root
service mysql start

use mysql;
update user set password=password("123456");

CREATE DATABASE hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

CREATE DATABASE amon DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

CREATE DATABASE hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

CREATE DATABASE monitor DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

CREATE DATABASE oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '123456' WITH GRANT OPTION;

FLUSH PRIVILEGES;

service mysql restart

cd /etc/init.d/ && chkconfig --add mysql
chkconfig mysql on
ln -s /usr/local/mysql-5.6.43/lib/libmysqlclient.so.18 /usr/lib64/libmysqlclient.so.18

#CDH

mkdir -p /var/cloudera-scm-server
mkdir -p /opt/cloudera/parcel-repo
mkdir -p /opt/cloudera-manager
cp /usr/local/soft/CDH* /opt/cloudera/parcel-repo/
cp /usr/local/soft/manifest.json /opt/cloudera/parcel-repo/
tar xf /usr/local/soft/cloudera-manager-centos7-cm5.10.0_x86_64.tar.gz -C /opt/cloudera-manager
useradd --system --home=/opt/cloudera-manager/cm-5.10.0/run/cloudera-scm-server --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm
chown cloudera-scm:cloudera-scm /var/cloudera-scm-server
chown cloudera-scm:cloudera-scm /opt/cloudera-manager
chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo
cd /opt/cloudera-manager/cm-5.10.0/share/cmf/schema/ && ./scm_prepare_database.sh mysql -uroot  –p123456 --scm-host localhost scm scm scm
vim  /opt/cloudera-manager/cm-5.10.0/etc/cloudera-scm-agent/config.ini


tar xf /usr/local/soft/apache-kylin-2.6.1-bin-cdh57.tar.gz -C /usr/local/
mv /usr/local/apache-kylin-2.6.1-bin-cdh57/ /usr/local/kylin-2.6.1-bin-cdh57/
ln -s /usr/local/kylin-2.6.1-bin-cdh57/ /usr/local/kylin
mv /usr/local/kylin/conf/kylin.properties  /usr/local/kylin/conf/kylin.properties.ba
mv /usr/local/kylin/conf/setenv.sh  /usr/local/kylin/conf/setenv.sh.ba
cp /usr/local/soft/setenv.sh /usr/local/kylin/conf/
chmod 777 /usr/local/kylin/conf/setenv.sh
mv /usr/local/soft/kylin.properties /usr/local/kylin/conf/
vim /usr/local/soft/kylin.properties

tar xf /usr/local/soft/sqoop-1.4.6-cdh5.10.0.tar.gz -C /opt/
ln -s /opt/sqoop-1.4.6-cdh5.10.0/ /usr/local/sqoop
cp /usr/local/soft/sqoop-env.sh /usr/local/sqoop/conf
chmod 777 /usr/local/sqoop/conf/sqoop-env.sh
64M  小文件太多 注意这三个目录大小 /dfs /var /tmp
/opt/cloudera-manager/cm-5.10.0/etc/init.d/cloudera-scm-server start
/opt/cloudera-manager/cm-5.10.0/etc/init.d/cloudera-scm-agent start
tail -fn 200 /opt/cloudera-manager/cm-5.10.0/log/cloudera-scm-server/cloudera-scm-server.log

tail -fn 200 /opt/cloudera-manager/cm-5.10.0/log/cloudera-scm-agent/cloudera-scm-agent.log

#oozie 执行sh脚本时候 filenotfound 
HADOOP_USER_NAME=${wf:user()}
#ssh 免密登录
ssh-keygen -t rsa
ssh-copy-id  -p 20822 root@worker1
ssh -p 20822 root@worker1
#oozie spark的jar包
hadoop fs -mkdir /user/oozie/share/lib/lib_20190716230221/spark2
cd /opt/cloudera/parcels/SPARK2/lib/spark2/jars
hadoop fs -put *.jar /user/oozie/share/lib/lib_20190716230221/spark2
cd /opt/cloudera/parcels/CDH/lib/oozie/oozie-sharelib-yarn/lib/spark
hadoop fs -put oozie-sharelib-spark*.jar  /user/oozie/share/lib/lib_20190716230221/spark2
hadoop fs -chown -R oozie:oozie /user/oozie/share/lib/lib_20190716230221/spark2
hadoop fs -chmod -R 775 /user/oozie/share/lib/lib_20190716230221/spark2
oozie admin -oozie http://192.168.1.207:11000/oozie -sharelibupdate
oozie admin -oozie http://192.168.1.207:11000/oozie -shareliblist


添加spark2.1 目录下面不能有其他的jar  集群停止
https://blog.csdn.net/high2011/article/details/79138493
mkdir -p /opt/cloudera/csd
chown cloudera-scm:cloudera-scm   /opt/cloudera/csd
chgrp cloudera-scm SPARK2_ON_YARN-2.1.0.cloudera2.jar
chown cloudera-scm SPARK2_ON_YARN-2.1.0.cloudera2.jar
需要的parcel放入到parcel_repo
parcel URL添加
http://archive.cloudera.com/spark2/parcels/2.1.0.cloudera2/



# mount /dev/sda1 /storage      挂载新硬盘
# vi /etc/fstab      设置系统自动挂载硬盘，加入一行：
/dev/sda1               /storage            ext4    defaults        0 0

hdfs挂载
hadoop-fuse-dfs hdfs://master:8020 /hdfs
注意CDH版本
vim  /etc/yum.repos.d/CentOS-Base.repo
[cloudera-cdh5]
# Packages for Cloudera's Distribution for Hadoop, Version 5, on RedHat or CentOS 6 x86_64
name=Cloudera's Distribution for Hadoop, Version 5
baseurl=https://archive.cloudera.com/cdh5/redhat/7/x86_64/cdh/5/
gpgkey =https://archive.cloudera.com/cdh5/redhat/7/x86_64/cdh/RPM-GPG-KEY-cloudera
gpgcheck = 1

yum install -y hadoop-hdfs-fuse fuse-lib 

python3安装

wget https://www.python.org/ftp/python/3.6.4/Python-3.6.4.tgz
tar -xvf Python-3.6.4.tgz
mkdir /usr/local/python3
cd  Python-3.6.4
./configure --prefix=/usr/local/python3
make 
make install 
ln -s /usr/local/python3/bin/python3 /usr/bin/python3
python3 -m  pip 
python -m pip