CDH集群部署与设置

1. ctdn-1
vi /etc/host
#127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
#::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
10.11.8.20 ctdn-1
10.11.8.28 ctdn-2
10.11.8.31 ctdn-3
10.11.8.16 ctdn-4
10.11.8.32 ctdn-5
10.11.8.35 ctdn-6

2.ALL
ssh-keygen -t rsa
cd .ssh

cat id_rsa.pub

3.ctdn-1
copy各台id_rsa.pub中的内容至本台的文件authorized_keys

4.ctdn-1
将authorized_keys文件scp到各台服务器
scp authorized_keys root@ctdn-2:/root/.ssh
scp authorized_keys root@ctdn-3:/root/.ssh
...
scp authorized_keys root@ctdn-6:/root/.ssh
scp /etc/hosts root@ctdn-2:/etc/
...
scp /etc/hosts root@ctdn-6:/etc/

5.ctdn-1
vi /etc/sysconfig/network
+NETWORKING_IPV6=no
scp /etc/sysconfig/network root@ctdn-2:/etc/sysconfig
...
scp /etc/sysconfig/network root@ctdn-6:/etc/sysconfig

6.关闭防火墙 ALL

检查iptables和firewalld 服务均没有安装

7.设置时间同步
ALL：
yum -y install chrony
systemctl start chronyd
ctdn-1:
vi /etc/chrony.conf
allow 10.11/24
# Listen for commands only on localhost.
bindcmdaddress 127.0.0.1
bindcmdaddress ::1
# Serve time even if not synchronized to any NTP server.
local stratum 10
others：
vi /etc/chrony.conf
+server 10.11.8.20 iburst
ALL：

systemctl restart chronyd.service

8.系统优化 ALL
禁用交换分区
sysctl -w vm.swappiness=0
禁用透明大页面
echo never > /sys/kernel/mm/transparent_hugepage/defrag

9.删除自带jdk
检查没有任何Java
ctdn-1:
下载jdk-8u121-linux-x64.tar.gz到/opt
scp jdk-8u121-linux-x64.tar.gz root@ctdn-2:/opt
...
scp jdk-8u121-linux-x64.tar.gz root@ctdn-6:/opt

10.安装JDK ALL
tar zxvf jdk-8u121-linux-x64.tar.gz
ln -s /opt/jdk1.8.0_121 /opt/jdk
mkdir /usr/java

ln -s /opt/jdk /usr/java/default

11.设置Java环境变量 ctdn-1
vi /etc/profile
+
export JAVA_HOME=/opt/jdk
export PATH="$JAVA_HOME/bin:$PATH"

scp /etc/profile root@ctdn-2:/etc
...
scp /etc/profile root@ctdn-6:/etc

ALL:
source /etc/profile

12.ctdn-1:
删除MariaDB
rpm -qa|grep mariadb
rpm -e mariadb-libs-5.5.44-2.el7.centos.x86_64 --nodeps
wget http://ftp.tu-chemnitz.de/pub/linux/dag/redhat/el6/en/x86_64/rpmforge/RPMS/axel-2.4-1.el6.rf.x86_64.rpm
rpm -ivh axel-2.4-1.el6.rf.x86_64.rpm
axel -n 20 https://cdn.mysql.com//Downloads/MySQL-5.7/mysql-5.7.20-1.el7.x86_64.rpm-bundle.tar
tar -xvf mysql-5.7.20-1.el7.x86_64.rpm-bundle.tar

rpm -ivh mysql-community-common-5.7.20-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-5.7.20-1.el7.x86_64.rpm
rpm -ivh mysql-community-client-5.7.20-1.el7.x86_64.rpm
yum install -y libaio
yum install -y libaio-devel
rpm -ivh mysql-community-server-5.7.20-1.el7.x86_64.rpm
systemctl start mysqld.service
systemctl enable mysqld.service
修改root管理员密码
获得临时秘密：
grep 'temporary password' /var/log/mysqld.log
mysql -uroot -p
mysql> SET PASSWORD FOR 'root'@'localhost' = PASSWORD('Ctdn@v2M');
[ ctdn-5:
mysql> SET PASSWORD FOR 'root'@'localhost' = PASSWORD('SaaS@v2M');
]
为hive oozie创建数据库：
mysql> CREATE DATABASE hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
mysql> CREATE DATABASE oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
mysql> CREATE DATABASE hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;  
mysql> CREATE DATABASE amon DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
mysql> grant all privileges on *.* to 'cdh'@'%' identified by 'Ctdn@v2MD' with grant option;
mysql> flush privileges;

13.创建scm用户 all
useradd --system --home=/opt/cm-5.5.1/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm

14.创建 ctdn-1
# rm -rf /user/hive/warehouse
# mkdir -p /user/hive/warehouse
# chown cloudera-scm:cloudera-scm /user/hive/warehouse
# rm -rf /var/lib/cloudera-host-monitor
# mkdir -p /var/lib/cloudera-host-monitor
# chown cloudera-scm:cloudera-scm /var/lib/cloudera-host-monitor
# rm -rf  /var/lib/cloudera-service-monitor
# mkdir -p /var/lib/cloudera-service-monitor

# chown cloudera-scm:cloudera-scm /var/lib/cloudera-service-monitor

15.安装包 all
yum install -y psmisc libxlst libxslt-python

16.ctdn-1
下载cloudera-manager-centos7-cm5.8.0_x86_64.tar.gz
axel -n 50 http://archive.cloudera.com/cm5/cm/5/cloudera-manager-centos7-cm5.8.0_x86_64.tar.gz
下载CDH Percel包
axel -n 20 http://archive.cloudera.com/cdh5/parcels/5.8.0/CDH-5.8.0-1.cdh5.8.0.p0.42-el7.parcel
axel -n 10 http://archive.cloudera.com/cdh5/parcels/5.8.0/CDH-5.8.0-1.cdh5.8.0.p0.42-el7.parcel.sha1
wget http://archive.cloudera.com/cdh5/parcels/5.8.0/manifest.json
下载mysqljdbc驱动
wget https://cdn.mysql.com//Downloads/Connector-J/mysql-connector-java-5.1.44.tar.gz

17.安装
tar zxvf cloudera-manager-centos7-cm5.8.0_x86_64.tar.gz
tar zxvf mysql-connector-java-5.1.44.tar.gz
cd mysql-connector-java-5.1.44
cp mysql-connector-java-5.1.44-bin.jar /opt/cm-5.8.0/share/cmf/lib/
初始化Cloudera Manager数据库
cd /opt/cm-5.8.0/share/cmf/schema
./scm_prepare_database.sh mysql cm -hlocalhost -ucdh -p'Ctdn@v2MD' scm 'Scm@v2MD'
修改配置
cd /opt/cm-5.8.0/etc/cloudera-scm-agent
vi config.ini
+server_host=10.11.8.20
将cm-5.8.0目录同步到其他服务器：
scp -r cm-5.8.0 root@ctdn-2:/opt/
......
scp -r cm-5.8.0 root@ctdn-6:/opt/

18.parcel ctdn-1:
cd cloudera/parcel-repo/
cp /opt/CDH-5.8.0-1.cdh5.8.0.p0.42-el7.parcel .
cp /opt/CDH-5.8.0-1.cdh5.8.0.p0.42-el7.parcel.sha1 .
cp /opt/manifest.json .
mv CDH-5.8.0-1.cdh5.8.0.p0.42-el7.parcel.sha1 CDH-5.8.0-1.cdh5.8.0.p0.42-el7.parcel.sha

19.启动CM服务 
ctdn-1：
/opt/cm-5.8.0/etc/init.d/cloudera-scm-server start
hostnamectl set-hostname ctdn-1
vim /etc/cloud/cloud.cfg
+# - set_hostname
+# - update_hostname
mkdir /opt/cm-5.8.0/run/cloudera-scm-agent
/opt/cm-5.8.0/etc/init.d/cloudera-scm-server restart
/opt/cm-5.8.0/etc/init.d/cloudera-scm-agent restart
ctdn-2~ctdn-6：
hostnamectl set-hostname ctdn-*
vim /etc/cloud/cloud.cfg
+# - set_hostname
+# - update_hostname
mkdir -p /opt/cm-5.8.0/run/cloudera-scm-agent
/opt/cm-5.8.0/etc/init.d/cloudera-scm-agent start

20.安装集群，登录
http://10.11.8.20:7180
用户名/密码：admin/Ctdnc01@2

21
正在安装选定 Parcel 步骤时无法分配，采取如下措施：
ALL 修改/etc/hosts文件,去掉如下所在行的#注释，如下：
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4

22.检查主机正确性，根据提示做如下修改：
ALL：
echo 'vm.swappiness=10'>> /etc/sysctl.conf
echo never > /sys/kernel/mm/transparent_hugepage/defrag

23群集设置
dfs.data.dir, dfs.datanode.data.dir:/data/dfs/dn
dfs.name.dir, dfs.namenode.name.dir:/data/dfs/nn
fs.checkpoint.dir, dfs.namenode.checkpoint.dir:/data/dfs/snn
Oozie 服务器数据目录:/var/lib/oozie/data
ZooKeeper Znode:/solr
HDFS 数据目录:/solr
NodeManager 本地目录 yarn.nodemanager.local-dirs:/data/yarn/nm

24.启动集群时遇到如下问题，修改时钟同步后，这样错误没有出现
Can't open /opt/cm-5.8.0/run/cloudera-scm-agent/process/46-hbase-MASTER/supervisor.conf: Permission denied.
ctdn-2
cd /opt/cm-5.8.0/run/cloudera-scm-agent/process/46-hbase-MASTER
发现：
-rw------- 1 root  root  3406 Nov 21 01:47 supervisor.conf
操作：
chown -R hbase:hbase *
问题没有解决，重新生成新的48-hbase-MASTER，找到hbase.sh
/opt/cm-5.8.0/lib64/cmf/service/hbase/hbase.sh

补充：
cp mysql-connector-java-5.1.44-bin.jar /opt/cloudera/parcels/CDH/lib/hive/lib
cp mysql-connector-java-5.1.44-bin.jar /opt/cloudera/parcels/CDH/lib/hadoop
cp mysql-connector-java-5.1.44-bin.jar /var/lib/oozie

1.all
sudo timedatectl set-timezone 'Asia/Shanghai'
yum install ntp -y
ctdn-1
systemctl restart ntpd.service
vi /etc/ntp.conf
+
server 0.cn.pool.ntp.org
server 0.asia.pool.ntp.org
server 3.asia.pool.ntp.org

# allow update time by the upper server
# 允许上层时间服务器主动修改本机时间
restrict 0.cn.pool.ntp.org nomodify notrap noquery
restrict 0.asia.pool.ntp.org nomodify notrap noquery
restrict 3.asia.pool.ntp.org nomodify notrap noquery

# Undisciplined Local Clock. This is a fake driver intended for backup
# and when no outside source of synchronized time is available.
# 外部时间服务器不可用时，以本地时间作为时间服务
server  127.127.1.0     # local clock
fudge   127.127.1.0 stratum 10

systemctl restart ntpd.service
chkconfig --level 35 ntpd on
netstat -tlunp | grep ntp (检查123端口，且协议为udp，需要开启此协议)

ctdn-2~ctdn-6：
vi /etc/ntp.conf
+ server 10.51.120.12 prefer
chkconfig ntpd on
systemctl restart ntpd.service
ntpdate -u 10.51.120.12
hwclock --systohc #把系统时间同步到硬件BIO
2. datanode启动失败
查看stderr
Can't open /opt/cm-5.8.0/run/cloudera-scm-agent/process/345-hdfs-DATANODE/supervisor.conf: Permission denied.
一直以为访问权限有问题，但是无论chmod还是chown均不能解决问题，而且有的成功有的失败。
cd /var/log/hadoop-hdfs
tail -f -n 2000 hadoop-cmf-hdfs-DATANODE-ctdn-1.log.out
查看/var/log/hadoop-hdfs/hadoop-cmf-hdfs-NAMENODE-master1.log.out 
WARN  Failed to add storage directory [DISK]file:/data/dfs/dn/
java.io.IOException: Incompatible clusterIDs in /data/dfs/dn: namenode clusterID = cluster34; datanode clusterID = cluster21
FATAL Initialization failed for Block pool (Datanode Uuid 357950bd-2e5b-4e89-b731-f58694461c55) service to ctdn-2/10.11.8.28:8022. Exiting.
解决：
3. namenode
没有格式化
hadoop namenode -format
chown -R hdfs:hadoop /data/dfs/
4.NFS Gateway不能启动
错误：No portmap or rpcbind service is running on this host
all
yum install rpcbind -y

hdfs-site.xml

dfs.permissions
false

否则：
Permission denied: user=mapred, access=WRITE, inode="/":hdfs:supergroup:drwxr-xr-x
+++++++++++++++++++++++++++++++++++++++++++++++
pip3 install flask-appbuilder
+++++++++++++++++++++++++++++++++++++++++
ctdn01:
hadoop distcp hdfs://ctdn/code hdfs://10.11.8.31:8022/ success
hadoop distcp hdfs://ctdn/data hdfs://10.11.8.31:8022/ success
hadoop distcp hdfs://ctdn/hbase/data hdfs://10.11.8.31:8022/ doing
执行至：hadoop distcp hdfs://ctdn/hbase/data/default/sinacommentstotal hdfs://10.11.8.31:8022/hbase/data/default
全部迁移完成后，执行
sudo -u hbase hbase hbck  -fixAssignments -fixMeta
hadoop distcp hdfs://ctdn/nash hdfs://10.11.8.31:8022/ success
hadoop distcp hdfs://ctdn/root hdfs://10.11.8.31:8022/ success
hadoop distcp hdfs://ctdn/test hdfs://10.11.8.31:8022/ success
hadoop distcp hdfs://ctdn/usr hdfs://10.11.8.28:8022/ quit
hadoop distcp hdfs://ctdn/var hdfs://10.11.8.28:8022/ quit
hadoop distcp hdfs://ctdn/key hdfs://10.11.8.31:8022/success
hadoop distcp hdfs://ctdn/tmp hdfs://10.11.8.28:8022/ quit
hadoop distcp hdfs://ctdn/user/hive/warehouse hdfs://10.11.8.31:8022/ failed,retry
hadoop distcp hdfs://ctdn/user/fanxing hdfs://10.11.8.31:8022/user/ success
hadoop distcp hdfs://ctdn/user/ctdn hdfs://10.11.8.31:8022/ success
drwxr-xr-x   - hbase hadoop          0 2017-11-06 02:39 /hbase/archive
drwxr-xr-x   - hbase hadoop          0 2017-04-01 14:39 /hbase/corrupt
drwxr-xr-x   - hbase hadoop          0 2017-04-01 14:39 /hbase/data
hdfs haadmin -getServiceState namenode94
hdfs haadmin -getServiceState namenode131
hdfs haadmin -transitionToActive --forcemanual namenode131
ctdn01:
mysqldump -uxhhlhive -pH@D!X19 -h 10.10.0.144 -t hive > hive.sql
jrpH@D!X19i4n9g
mysqldump -urecomm -p -h 10.11.8.20 -t recomm --table project_inverst_bak > project_inverst.sql
ctdn-1：
mysql>source /opt/hive.sql
配置NameNode HA:
进入HDFS界面，在右上角的“操作”中，选择点击“启用High Availability”
输入NameService名称，这里设置为：ctdn，点击继续按钮。
设置另一个NameNode节点，这里设置为：cdh-node3.grc。设置JournalNode节点，这里设置为：cdh-node[2-4].grc，一共3个节点。注意：NournalNode必须设置>=3个节点。
设置JournalNode目录，cdh-node[2-4]上，这里全部设置为：/data/dfs/jn
启用HDFS的High Availability
如果发现有如下错误信息，NameNode格式化失败，可以忽略。 
成功启用HA
更新Hive Metastore NameNodes 
重启集群失败，报错：Journal Storage Directory /data/dfs/jn/ctdn not formatted，进入NameNode配置界面，在右上角操作中，选择点击“初始化共享编辑目录”
journalnode同步
hdfs namenode -bootstrapStand

Hue：
用户名/密码：ctdnadhoc/Aaddhmoi

配置hue支持DB查询
1. ctdn-5:
cd /etc/hue/conf
vi hue.ini
[librdbms]
[[databases]]
[[[mysql]]] （放开）
nice_name="My SQL DB"（放开）
name=ctdn （放开）
engine=mysql（放开）
host=10.9.130.142 （修改）
port=3306（放开）
user=root （修改）
password=IhNtPz6E2V34 （修改）
2.cloudera manager重启hue
3.报错：JournalError loading MySQLdb module: libmysqlclient.so.18: cannot open shared object file: No such file or directory
4.检查：ctdn-6上没有安装MySQL lib
axel -n 50 https://cdn.mysql.com/archives/mysql-5.7/mysql-5.7.12-1.el7.x86_64.rpm-bundle.tar
tar -xvf mysql-5.7.12-1.el7.x86_64.rpm-bundle.tar
rpm -qa|grep mariadb
rpm -e mariadb-libs-5.5.44-2.el7.centos.x86_64 --nodeps
wget ftp://mirror.switch.ch/pool/4/mirror/mysql/Downloads/MySQL-5.5/MySQL-shared-5.5.57-1.el7.x86_64.rpm
rpm -ivh MySQL-shared-5.5.57-1.el7.x86_64.rpm
[root@ctdn-6 lib64]# find / -name libmysqlclient.so.18
/usr/lib64/libmysqlclient.so.18
5.hue:rdbms没有配置
转到cloudera manager
hue->配置
范围->Hue Server
类别->高级
hue_safety_valve_server.ini 的 Hue Server 高级配置代码段（安全阀）：
[librdbms]
# The RDBMS app can have any number of databases configured in the databases
# section. A database is known by its section name
# (IE sqlite, mysql, psql, and oracle in the list below).
[[databases]]
# sqlite configuration.
## [[[sqlite]]]
# Name to show in the UI.
## nice_name=SQLite
# For SQLite, name defines the path to the database.
## name=/tmp/sqlite.db
# Database backend to use.
## engine=sqlite
# Database options to send to the server when connecting.
# https://docs.djangoproject.com/en/1.4/ref/databases/
## options={}
# mysql, oracle, or postgresql configuration.
[[[mysql]]]
# Name to show in the UI.
nice_name="My SQL DB"
# For MySQL and PostgreSQL, name is the name of the database.
# For Oracle, Name is instance of the Oracle server. For express edition
# this is 'xe' by default.
name=ctdn
# Database backend to use. This can be:
# 1. mysql
# 2. postgresql
# 3. oracle
engine=mysql
# IP or hostname of the database to connect to.
host=10.9.130.142
# Port the database server is listening to. Defaults are:
# 1. MySQL: 3306
# 2. PostgreSQL: 5432
# 3. Oracle Express Edition: 1521
port=3306
# Username to authenticate with when connecting to the database.
user=root
# Password matching the username to authenticate with when
# connecting to the database.
password=IhNtPz6E
# Database options to send to the server when connecting.
# https://docs.djangoproject.com/en/1.4/ref/databases/
## options={}
6.cloudera manager重启hue

hue支持spark
1.ctdn-5
cd /etc/yum.repos.d/
curl https://bintray.com/sbt/rpm/rpm > bintray-sbt-rpm.repo
yum -y install sbt
2. 
cd /data/tools
git clone https://github.com/ooyala/spark-jobserver.git
cd spark-jobserver
sbt
第二方案
1. wget http://archive.cloudera.com/beta/livy/livy-server-0.2.0.zip
unzip livy-server-0.2.0.zip
2. vi /etc/profile
JAVA_HOME SPARK_HOME 
3. su hdfs
cd /data/tools/livy-server-0.2.0
nohup bin/livy-server &
4. 退回到root用户
vi /etc/hue/conf/hue.ini
[spark]
  # Host address of the Livy Server.
   livy_server_host=ctdn-5
+++++++++++++++++++++++++++++++++++++++++++
Kafka·
服务器：ctdn-2~ctdn-6
1. 创建持久化目录
mkdir /data/kafkaLogs
2. ctdn-2:
cd /opt
wget http://mirrors.hust.edu.cn/apache/kafka/1.0.0/kafka_2.11-1.0.0.tgz
tar zxvf kafka_2.11-1.0.0.tgz
mv kafka_2.11-1.0.0 kafka
cd kafka/config
vi server.properties
确定(配置broker id) broker.id=0
打开监听端口（取消这一行注释）：listeners=PLAINTEXT://:9092
设置zookeeper.connect：zookeeper.connect=ctdn-3:2181,ctdn-4:2181,ctdn-5:2181
确定超时设置：zookeeper.connection.timeout.ms=6000
添加启用删除topic配置：delete.topic.enable=true
关闭自动创建topic：auto.create.topics.enable=false
修改 log 的目录：log.dirs=/data/kafkaLogs
一个topic默认1个分区数(确定)：num.partitions=1 
num.recovery.threads.per.data.dir
3.同步到其他节点
scp -r kafka root@ctdn-3:/opt/
......
scp -r kafka root@ctdn-6:/opt/
4. ctdn-2~ctdn-6：
vi /etc/profile
+export KAFKA_HOME=/opt/kafka
+export PATH="$JAVA_HOME/bin:$KAFKA_HOME/bin:$PATH"
source /etc/profile
5.ctdn-3~ctdn-6：
cd /opt/kafka/config
vi server.properties
+broker.id=1(ctdn-3:1,...,ctdn-6:4)
6.启动Kafka（ctdn-2~ctdn-6）
cd /opt/kafka
kafka-server-start.sh server.properties &
（可选：JMX_PORT=9997 bin/kafka-server-start.sh -daemon config/server.properties &）
关闭 
jps
kill -9 ****
7.测试集群
创建topic test(ctdn-2)
bin/kafka-topics.sh --create --zookeeper ctdn-3:2181,ctdn-4:2181,ctdn-5:2181 --replication-factor 1 --partitions 1 --topic test
查看所有topic
bin/kafka-topics.sh --list --zookeeper ctdn-3:2181,ctdn-4:2181,ctdn-5:2181
发送消息，生产者（ctdn-2）
bin/kafka-console-producer.sh --broker-list ctdn-2:9092 --topic test
启动消费者(ctdn-3、ctdn-4)
bin/kafka-console-consumer.sh --zookeeper ctdn-3:2181,ctdn-4:2181,ctdn-5:2181 --topic test --from-beginning 
8.python支持
http://pykafka.readthedocs.io/en/latest/usage.html
pip3 install pykafka
------生产者pd.py-------
# -* coding:utf8 *-
from pykafka import KafkaClient
host = '10.11.8.16' #生产者服务器IP
client = KafkaClient(hosts="%s:9092" % host)
print(client.topics)
topicdocu = client.topics[b'test']
producer = topicdocu.get_producer()
for i in range(4):
     print(i)
     producer.produce('test message '.encode('utf-8') + str(i ** 2).encode('utf-8'))
producer.stop()
------消费者cm.py-------
# -* coding:utf8 *-
from pykafka import KafkaClient
host = '10.11.8.16'  #消费者服务器IP
client = KafkaClient(hosts="%s:9092" % host)
print(client.topics)
topic = client.topics[b'test']
consumer = topic.get_simple_consumer(consumer_group=b'test', auto_commit_enable=True, consumer_id=b'test')
for message in consumer:
    if message is not None:
        print(message.offset, message.value)
 
python3 cm.py
python3 pd.py

9.KSQL
ctdn-5
https://github.com/confluentinc/ksql
http://geek.csdn.net/news/detail/235801
bin/kafka-topics.sh --create --zookeeper ctdn-3:2181,ctdn-4:2181,ctdn-5:2181 --replication-factor 1 --partitions 1 --topic pageviews
bin/kafka-topics.sh --create --zookeeper ctdn-3:2181,ctdn-4:2181,ctdn-5:2181 --replication-factor 1 --partitions 1 --topic users

1. git clone https://github.com/confluentinc/ksql.git
2. cd ksql
mvn clean compile install -DskipTests
3. 进入KSQL环境
单机模式Standalone，KSQL客户端和服务器端在同一台服务器上，共用同一JVM
./bin/ksql-cli local
./bin/ksql-cli local --bootstrap-server kafka-broker-1:9092 \
                       --properties-file path/to/ksql-cli.properties
   
CS模式Client-server，在远程服务器、VM或容器上启动KSQL池，CLI通过HTTP连接它们
启动服务器节点
./bin/ksql-server-start
./bin/ksql-server-start ksql-server.properties
ksql-server.properties内容如下：
# You must set at least the following two properties
bootstrap.servers=kafka-broker-1:9092
# Note: `application.id` is not really needed but you must set it
#       because of a known issue in the KSQL Developer Preview
application.id=app-id-setting-is-ignored
# Optional settings below, only for illustration purposes
# The hostname/port on which the server node will listen for client connections
listeners=http://0.0.0.0:8090

启动客户端，指定KSQL服务器地址
./bin/ksql-cli remote http://my-ksql-server:8090
10. KSQL实验
生产数据
topics：pageviews、users
为pageviews生产数据
java -jar ksql-examples/target/ksql-examples-4.1.0-SNAPSHOT-standalone.jar quickstart=pageviews format=delimited topic=pageviews maxInterval=10000
为users生产数据
java -jar ksql-examples/target/ksql-examples-4.1.0-SNAPSHOT-standalone.jar quickstart=users format=json topic=users maxInterval=10000
命令行方式：
kafka-console-producer --broker-list localhost:9092  \
                         --topic t1 \
                         --property parse.key=true \
                         --property key.separator=:
建表
ksql> 
CREATE STREAM pageviews_original (viewtime bigint, userid varchar, pageid varchar) WITH (kafka_topic='pageviews', value_format='DELIMITED');
DESCRIBE pageviews_original;
CREATE TABLE users_original (registertime bigint, gender varchar, regionid varchar, userid varchar) WITH (kafka_topic='users', value_format='JSON', key = 'userid');
DESCRIBE users_original;

SHOW STREAMS;
SHOW TABLES;

查询
ksql> SELECT pageid FROM pageviews_original LIMIT 3;
ksql> CREATE STREAM pageviews_female AS SELECT users_original.userid AS userid, pageid, regionid, gender FROM pageviews_original LEFT JOIN users_original ON pageviews_original.userid = users_original.userid WHERE gender = 'FEMALE';
ksql> DESCRIBE pageviews_female;
ksql> SELECT * FROM pageviews_female;
ksql> CREATE STREAM pageviews_female_like_89 WITH (kafka_topic='pageviews_enriched_r8_r9', value_format='DELIMITED') AS SELECT * FROM pageviews_female WHERE regionid LIKE '%_8' OR regionid LIKE '%_9';
ksql> CREATE TABLE pageviews_regions WITH (value_format='avro') AS SELECT gender, regionid , COUNT(*) AS numusers FROM pageviews_female WINDOW TUMBLING (size 30 second) GROUP BY gender, regionid HAVING COUNT(*) > 1;
ksql> DESCRIBE pageviews_regions;
ksql> SHOW QUERIES;
bin/ksql-cli local --exec "SELECT * FROM pageviews_original LIMIT 5;"
KSQL配置
SET 'auto.offset.reset'='earliest';
默认值latest(最新的),从当前的offset读取数据，可修改如上
SET 'commit.interval.ms'='5000';
默认值2000
++++++++++++++定时清理日志++++++++++++++++++++
ctdn-2~ctdn-6：
1. 
cd /root/nash
vi clean_cloudslog.sh
+
#!/bin/bash
###Description:This script is used to clear kafka logs, not message file.
#####1.kafka
# log file dir.
logDir=/opt/kafka/logs
# Reserved 7 files.
COUNT=7
ls -t $logDir/server.log* | tail -n +$[$COUNT+1] | xargs rm -f
ls -t $logDir/controller.log* | tail -n +$[$COUNT+1] | xargs rm -f
ls -t $logDir/state-change.log* | tail -n +$[$COUNT+1] | xargs rm -f
ls -t $logDir/log-cleaner.log* | tail -n +$[$COUNT+1] | xargs rm –f
#####2.hbase
hbaseDir=/var/log/hbase
2.
crontab -e
+
0 0 * * 0 /root/nash/clean_cloudslog.sh
################kafka manager################################
安装sbt
cd /etc/yum.repos.d/
curl https://bintray.com/sbt/rpm/rpm | sudo tee /etc/yum.repos.d/bintray-sbt-rpm.repo
下载Yahoo kafka manager
cd /opt
git clone https://github.com/yahoo/kafka-manager.git
cd kafka-manager
sbt clean dist
看到：[info] Your package is ready in /opt/kafka-manager/target/universal/kafka-manager-1.3.3.14.zip
成功打包
cd /opt/kafka-manager/target/universal
cp kafka-manager-1.3.3.14.zip ~/
unzip -oq kafka-manager-1.3.3.14.zip
cd kafka-manager-1.3.3.14
vim conf/application.conf
+
#kafka-manager.zkhosts="kafka-manager-zookeeper:2181"
#kafka-manager.zkhosts=${?ZK_HOSTS}
kafka-manager.zkhosts="10.11.8.31:2181:10.11.8.16:2181:10.11.8.32:2181"
启动
bin/kafka-manager 
This application is already running (Or delete /root/kafka-manager-1.3.3.14/RUNNING_PID file).
你也可以在启动时指定配置文件和监听端口：
# bin/kafka-manager -Dconfig.file=/root/kafka-manager-1.3.3.14/conf/application.conf -Dhttp.port=8088
+++++++++++++++++python ctdn-1~ctdn-6++++++++++++
开始时只在ctdn-6上部署Python，运行pyspark，报错：pyspark: ImportError: No module named numpy
经检查，不是本台服务器没有安装，而是其他节点没有部署Python环境。
https://www.python.org/ftp/python
yum -y install gcc-c++
yum -y install gcc
mkdir -p /usr/local/python3
cd /usr/local/python3
wget https://www.python.org/ftp/python/3.4.4/Python-3.4.4.tar.xz
tar xvf Python-3.4.4.tar
cd Python-3.4.4
./configure --prefix=/usr/local/python3/python344
make
make install
ln -s /usr/local/python3/python344/bin/python3 /usr/local/bin/python3
ln -s /usr/local/python3/python344/bin/pip3    /usr/local/bin/pip3
pip3 install numpy
不要做：pip3 install --upgrade pip
pip3 install pandas
pip3 install scipy-1.0.0-cp34-cp34m-manylinux1_x86_64.whl
pip3 install scikit_learn-0.19.1-cp34-cp34m-manylinux1_x86_64.whl
pip3 install matplotlib
cd /opt/cloudera/parcels/CDH-5.8.0-1.cdh5.8.0.p0.42/lib/spark/python
cp -r pyspark /usr/local/python3/python344/lib/python3.4/site-packages/
上述操作依旧没有解决问题，设置环境变量如下，解决：
vi /etc/profile
+
export PYTHONPATH=/usr/local/python3/python344/lib/python3.4
export PYSPARK_PYTHON=/usr/local/bin/python3
但是python不能使用，
在/etc/profile中删除PYTHONPATH也不管用，如下解决：
unset PYTHONPATH
+++++++++++++++CDH升级Spark2++++++++++++++++++++++++++++
1.所需软件
http://archive.cloudera.com/spark2/csd/
下载SPARK2_ON_YARN-2.1.0.cloudera1.jar 
axel -n 20 http://archive.cloudera.com/spark2/csd/SPARK2_ON_YARN-2.1.0.cloudera1.jar
http://archive.cloudera.com/spark2/parcels/2.1.0.cloudera1/
下载SPARK2-2.1.0.cloudera1-1.cdh5.7.0.p0.120904-el7.parcel 
SPARK2-2.1.0.cloudera1-1.cdh5.7.0.p0.120904-el7.parcel.sha1
manifest.json
axel -n 50 http://archive.cloudera.com/spark2/parcels/2.1.0.cloudera1/SPARK2-2.1.0.cloudera1-1.cdh5.7.0.p0.120904-el7.parcel
wget http://archive.cloudera.com/spark2/parcels/2.1.0.cloudera1/SPARK2-2.1.0.cloudera1-1.cdh5.7.0.p0.120904-el7.parcel.sha1
wget http://archive.cloudera.com/spark2/parcels/2.1.0.cloudera1/manifest.json
2.ctdn-1 
cp SPARK2_ON_YARN-2.1.0.cloudera1.jar /data/cloudera/csd
cp SPARK2-2.1.0.cloudera1-1.cdh5.7.0.p0.120904-el7.parcel /data/cloudera/parcel-repo
cp SPARK2-2.1.0.cloudera1-1.cdh5.7.0.p0.120904-el7.parcel.sha1 /data/cloudera/parcel-repo
cd /data/cloudera/parcel-repo/
mv SPARK2-2.1.0.cloudera1-1.cdh5.7.0.p0.120904-el7.parcel.sha1 SPARK2-2.1.0.cloudera1-1.cdh5.7.0.p0.120904-el7.parcel.sha
mv manifest.json manifest.json.bak
cd /opt
cp manifest.json /opt/cloudera/parcel-repo/

ctdn-2~ctdn-6：
cd /opt/cloudera 
mkdir csd
mkdir parcel-repo

ctdn-1：
scp SPARK2_ON_YARN-2.1.0.cloudera1.jar root@ctdn-2:/opt/cloudera/csd
......
scp SPARK2_ON_YARN-2.1.0.cloudera1.jar root@ctdn-6:/opt/cloudera/csd
scp SPARK2-2.1.0.cloudera1-1.cdh5.7.0.p0.120904-el7.parcel root@ctdn-2:/opt/cloudera/parcel-repo/
......
scp SPARK2-2.1.0.cloudera1-1.cdh5.7.0.p0.120904-el7.parcel root@ctdn-6:/opt/cloudera/parcel-repo/
scp SPARK2-2.1.0.cloudera1-1.cdh5.7.0.p0.120904-el7.parcel.sha root@ctdn-2:/opt/cloudera/parcel-repo/
......
scp SPARK2-2.1.0.cloudera1-1.cdh5.7.0.p0.120904-el7.parcel.sha root@ctdn-6:/opt/cloudera/parcel-repo/
修改文件的用户和组,我没有做，没有出问题，需要跟同目录下其他文件一样。

3.停掉CM和集群
然后重启cm

4.登录cm
主机->Parcel,左侧列表找到Spark2，点击，右上依次点击分配、激活

5.返回主页
集群->添加服务，添加spark2服务。
http://blog.csdn.net/u010936936/article/details/73650417
cd /opt/cloudera/parcels/SPARK2-2.1.0.cloudera1-1.cdh5.7.0.p0.120904/etc/spark2/conf.dist/
cp /etc/spark/conf/spark-env.sh .
cp /etc/spark/conf/classpath.txt .

vi spark-env.sh
+
#export SPARK_HOME=/opt/cloudera/parcels/CDH-5.8.0-1.cdh5.8.0.p0.42/lib/spark
export SPARK_HOME=/opt/cloudera/parcels/SPARK2-2.1.0.cloudera1-1.cdh5.7.0.p0.120904/lib/spark2
export PYSPARK_PYTHON=/usr/local/bin/python3
export PYSPARK_DRIVER_PYTHON=python3
补充：
spark_master: ctdn-4:7077修改为spark_master: yarn  （基于yarn的设置为yarn, standonly(master-slave)的设置为masterIp:7077）
+++++++++++根目录满+++++++++
查看inode使用率: df -i
df -lh
cd /
du -h -x --max-depth=1
发现/opt 5G
cd /opt
du -h -x --max-depth=1
/var/lib/cloudera-service-monitor/ts
+++++++++++agent假死+++++++++
[root@ctdn-1 init.d]# /opt/cm-5.8.0/etc/init.d/cloudera-scm-agent stop
Usage: grep [OPTION]... PATTERN [FILE]...                  [FAILED]
Try 'grep --help' for more information.
[root@ctdn-1 init.d]# /opt/cm-5.8.0/etc/init.d/cloudera-scm-agent start
cloudera-scm-agent is already running
[root@ctdn-1 init.d]# /opt/cm-5.8.0/etc/init.d/cloudera-scm-agent status
cloudera-scm-agent dead but pid file exists
[root@ctdn-1 cloudera-scm-agent]# find / -name cloudera-scm-agent.pid
find: ‘/proc/28696’: No such file or directory
/opt/cm-5.8.0/run/cloudera-scm-agent/cloudera-scm-agent.pid
[root@ctdn-1 cloudera-scm-agent]# cd /opt/cm-5.8.0/run/cloudera-scm-agent/
[root@ctdn-1 cloudera-scm-agent]# ll
total 4
drwxr-x--x   2 root         root            6 Nov 20 19:02 cgroups
-rw-r--r--   1 root         root            1 Dec  1 10:27 cloudera-scm-agent.pid
[root@ctdn-1 cloudera-scm-agent]# rm -f cloudera-scm-agent.pid
+++++++++++++++++++++++++HUE+++++++++++++++++++++++++
Could not start SASL: Error in sasl_client_start (-4) SASL(-4): no mechanism available: No worthy mechs found
解决方案
yum -y install cyrus-sasl-plain
++++++++++++++++++++++++percona+++++++++++++++++++++++++++++++++
yum install http://www.percona.com/downloads/percona-release/redhat/0.1-4/percona-release-0.1-4.noarch.rpm
yum install percona-toolkit -y