CDH相关包的下载地址：

CDH相关包的下载地址：
http://archive.cloudera.com/cm5/cm/5/
http://archive.cloudera.com/cdh5/parcels/latest/

用户：
userdel -r cloudera-scm

ps -ef | grep cloudera-scm | awk '{ print $2 }' | sudo xargs kill -9

ps -ef | grep mysql | awk '{ print $2 }' | xargs kill -9

root TPaxuv_0
scm@2018@baoneng
jboss Qhsdvf_9

然后切换到root TPaxuv_0

;M0.vHxbR>lx

/qhapp/cdh
.
├── cloudera
│   ├── parcel-cache
│   └── parcels
├── cm
│   └── cm-5.10.0
├── software
│   ├── CDH-5.10.0-1.cdh5.10.0.p0.41-el6.parcel
│   ├── CDH-5.10.0-1.cdh5.10.0.p0.41-el6.parcel.sha
│   ├── cloudera-manager-el6-cm5.10.0_x86_64.tar.gz
│   ├── jdk1.7.0_80.zip
│   ├── manifest.json
│   ├── mysql-advanced-5.7.9-linux-glibc2.5-x86_64.tar.gz
│   └── mysql-connector-java-5.1.40.jar
└── var
├── data
├── he
├── lib
├── log
├── run
└── tmp
mkdir -p /qhapp/cdh/var/data
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
10.60.81.157 sz280317
10.60.81.158 sz280318
10.60.81.159 sz280319
10.60.81.160 sz280320
10.60.81.161 sz280321
10.60.81.162 sz280322
10.60.81.163 sz280323
10.60.81.164 sz280324
10.60.81.165 sz280325
10.60.81.166 sz280326
10.60.81.167 sz280327
10.60.81.168 sz280328
10.60.81.169 sz280329
10.60.81.170 sz280330
10.60.81.171 sz280331

NETWORKING=yes
HOSTNAME=sz280330

server 10.60.4.12 prefer
server 10.60.4.13
server 10.50.4.12
server 10.50.4.13
driftfile /var/lib/ntp/drift
keys /etc/ntp/keys
disable monitor
restrict default kod nomodify notrap nopeer noquery
restrict -6 default kod nomodify notrap nopeer noquery
restrict 127.0.0.1
restrict -6 ::1
tinker panic 120

root ALL=(ALL) ALL

%cloudera-scm ALL=(ALL) NOPASSWD: CDH

scp ~/.ssh/authorized_keys root@slave01:~/.ssh/

sz280325
sz280324
sz280326
sz280327
sz280328
sz280329
sz280330
sz280331

 

scp ~/.ssh/authorized_keys jboss@sz280317:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280318:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280319:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280320:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280321:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280322:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280323:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280324:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280325:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280326:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280327:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280328:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280329:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280330:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280331:~/.ssh/
Qhsdvf_9

rpm -ivh jdk-8u151-linux-x64.rpm

alter user 'root'@'localhost' identified by 'Hgx@123456';
flush privileges;
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'Hgx@123456' WITH GRANT OPTION;

/qhapp/cdh/cm/cm-5.15.0/share/cmf/schema/scm_prepare_database.sh mysql cm -hlocalhost -uroot -pMysql@123456 --scm-host localhost scm scm scm

chown -r cloudera-scm:cloudera-scm cdh

useradd --system --home=/qhapp/cdh/cm/cm-5.15.0/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm2

usermod -a -G root cloudera-scm

root TPaxuv_0
groupadd cloudera-scm
useradd -g cloudera-scm cloudera-scm
passwd cloudera-scm
scm@2018
rpm -e mysql-community-libs-5.7.20-1.el6.x86_64 --nodeps
rpm -e mysql-community-server-5.7.20-1.el6.x86_64 --nodeps
rpm -e mysql-community-common-5.7.20-1.el6.x86_64 --nodeps
rpm -e mysql-community-client-5.7.20-1.el6.x86_64 --nodeps

rpm -e mysql-community-libs-compat-5.7.20-1.el6.x86_64 --nodeps

sudo /qhapp/cdh/cm/cm-5.15.0/etc/init.d/cloudera-scm-server start
sudo /qhapp/cdh/cm/cm-5.15.0/etc/init.d/cloudera-scm-agent start

/qhapp/cdh/cm/cm-5.15.0/etc/init.d/cloudera-scm-server stop
/qhapp/cdh/cm/cm-5.15.0/etc/init.d/cloudera-scm-agent stop

chown -R cloudera-scm:cloudera-scm /qhapp/cdh

ps -ef |grep cloudera-scm |grep -v grep|cut -c 9-15|xargs kill -9

ps -ef |grep mycanal-1.0-SNAPSHOT.jar |grep -v grep| cut -c 9-15 | xargs kill -9

create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

cp mysql-connector-java-8.0.11.jar /qhapp/cdh/cloudera/parcels/CDH-5.15.0-1.cdh5.15.0.p0.21/lib/hive/lib/

kill -9 $(pgrep -f supervisord)

hadoop namenode -recover
hadoop namenode -recover
./hadoop-daemon.sh start journalnode

rpm -ivh mysql-community-common-5.7.20-1.el6.x86_64.rpm
rpm -ivh mysql-community-libs-5.7.20-1.el6.x86_64.rpm
rpm -ivh mysql-community-client-5.7.20-1.el6.x86_64.rpm
rpm -ivh mysql-community-server-5.7.20-1.el6.x86_64.rpm

alter user 'root'@'localhost' identified by 'Mysql@123456';

set password for 'root'@'localhost' = password('Mysql@123456');
alter user 'root'@'localhost' identified with mysql_native_password by 'Mysql@123456';
flush privileges;
3. 创建同步数据用户及授权用户
3.1．授权其他机器登陆
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'Mysql@123456' WITH GRANT OPTION;
flush privileges;

create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

rm -rf /qhapp/cdh/cloudera/parcel-cache /qhapp/cdh/cloudera/parcels

cp mysql-connector-java-5.1.35.jar /qhapp/cdh/cm/cm-5.15.0/share/cmf/lib/

cp mysql-connector-java-5.1.35.jar /opt/cm-5.3.8/share/cmf/lib/
/qhapp/cdh/cm/cm-5.15.0/share/cmf/schema/scm_prepare_database.sh mysql cm -hlocalhost -uroot -pMysql@123456 --scm-host localhost scm scm scm

/qhapp/cdh/cm/cm-5.15.0/etc/cloudera-scm-agent/config.ini
cloudera-scm
root TPaxuv_0
scm@2018

useradd --system --home=/qhapp/cdh/cm/cm-5.15.0/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm

!connect jdbc:hive2://localhost:10000

chown -R cloudera-scm:cloudera-scm /qhapp/cdh

/qhapp/cdh/cloudera/parcels/CDH/lib/hadoop

rm -rf /usr/share/cmf;
rm -rf /var/lib/cloudera*
rm -rf /var/cache/yum/x86_64/6/cloudera*
rm -rf /var/log/cloudera*
rm -rf /var/run/cloudera* /etc/cloudera*

grant all privileges on hue.* to 'hive'@'localhost' identified by 'hue';
grant all privileges on hue.* to 'hive'@'%' identified by 'hue';
flush privileges;

sudo -u hdfs hdfs dfs -chown hive:hive /user
sudo -u hdfs hdfs dfs -chown hue:hue /user

mkdir -p /qhapp/cdh/var/data
mkdir -p /qhapp/cdh/var/lib
mkdir -p /qhapp/cdh/var/log
mkdir -p /qhapp/cdh/var/run
mkdir -p /qhapp/cdh/var/tmp

chown -R cloudera-scm:cloudera-scm /qhapp/cdh

chown cloudera-scm:cloudera-scm /etc/profile
source /etc/profile

solrctl instancedir --generate /qhapp/cdh/var/lib/solr/demo_instancedir
/qhapp/cdh/var/lib/solr/data


ntpdate -u 10.60.81.168

./bin/spark-submit --class org.apache.spark.examples.SparkPi --master yarn lib/spark-examples-1.6.0-cdh5.15.0-hadoop2.6.0-cdh5.15.0.jar 10

{"head":{"binlog_pos":24824,"type":"DELETE","binlog_file":"mysql-bin.000005","db":"aa","table":"test_tb2"},"after":{"author":"sss","id":"666","title":"aadd","rowKey":"34555"}}
{"head":{"binlog_pos":24824,"type":"INSERT","binlog_file":"mysql-bin.000005","db":"aa","table":"test_tb2"},"after":{"author":"442","id":"666","title":"aadd","rowKey":"34555"}}

solrctl instancedir --generate /qhapp/cdh/var/lib/solr2
solrctl instancedir --create hgxtest /qhapp/cdh/var/lib/solr2
solrctl instancedir --list
solrctl collection --create hgxtest -s 2 -r 2

zookeeper.connect=10.60.81.166:2181,10.60.81.167:2181,10.60.81.168:2181
10.60.81.157:9092,10.60.81.158:9092,10.60.81.160:9092
zookeeper.connect=10.60.81.157:4183,10.60.81.158:4183,10.60.81.160:4183

./kafka-server-start.sh -daemon ../config/server.properties

/qhapp/cdh/kafka10/kafka_2.11-1.0.0/bin/kafka-topics.sh --create --zookeeper 10.60.81.160:4183 --replication-factor 2 --partitions 3 --topic zkKafka

#创建一个broker，发布者
/qhapp/cdh/kafka10/kafka_2.11-1.0.0/bin/kafka-console-producer.sh --broker-list 10.60.81.160:9092 --topic zkKafka

'''在一台服务器上创建一个订阅者'''
/qhapp/cdh/kafka10/kafka_2.11-1.0.0/bin/kafka-console-consumer.sh --zookeeper 10.60.81.160:4183 --topic canal2kafka2 --from-beginning

/qhapp/cdh/kafka10/kafka_2.11-1.0.0/bin/kafka-topics.sh --list --zookeeper 10.60.81.160:4183
/qhapp/cdh/kafka10/kafka_2.11-1.0.0/bin/kafka-topics.sh -describe --zookeeper 10.60.81.160:4183 --topic zkKafka
-describe
/qhapp/cdh/kafka/kafka_2.10-0.9.0.0/bin/kafka-topics.sh --delete --zookeeper 10.60.81.167:2181 --topic kafkae

:
rmr /brokers/topics/heguoxiu2
rmr /brokers/topics/heguoxiu921
rmr /brokers/topics/testh
rmr /brokers/topics/heguoxiu91
rmr /brokers/topics/heguoxiu7
rmr /brokers/topics/canal2kafka2
rmr /brokers/topics/heguoxiu9
rmr /brokers/topics/canal2kafka
rmr /brokers/topics/heguo
rmr /brokers/topics/canal2kafka3
zookeeper.connect=10.60.81.166:2181,10.60.81.168:2181,10.60.81.167:2181

sudo /qhapp/cdh/zookeeper/zookeeper-3.4.12/bin/zkServer.sh start /qhapp/cdh/zookeeper/zookeeper-3.4.12/conf/zoo.cfg

/qhapp/cdh/zookeeper/zookeeper-3.4.12/bin/zkServer.sh stop

/opt/zookeeper/zookeeper-3.4.6/bin/../conf/zoo.cfg

./kafka-server-start.sh -daemon ../config/server.properties

spark-submit --master yarn-client \
--num-executors 4 --driver-memory 2g --executor-memory 2g --executor-cores 4 \
--jars /qhapp/cdh/sparkproject/kafkalib/kafka_2.10-0.8.2.1.jar,/qhapp/cdh/sparkproject/kafkalib/zkclient-0.3.jar,/qhapp/cdh/sparkproject/kafkalib/metrics-core-2.2.0.jar \
--class com.bn.sparkstreaming.spark2hbase /qhapp/cdh/sparkproject/hgx/spark2hbase.jar

--jars /qhapp/cdh/sparkproject/sparklibs/zookeeper-3.4.12.jar

spark-submit --master yarn-client --num-executors 4 --driver-memory 2g --executor-memory 2g --executor-cores 2 \
--jars /qhapp/cdh/sparkproject/sparklibs/spark-examples-1.6.0-hadoop2.6.0.jar --class com.bn.sparkstreaming.spark2hbase /qhapp/cdh/sparkproject/hgx/spark2hbase.jar

{"head":{"binlog_pos":2002,"type":"INSERT","binlog_file":"mysql-bin.000001","db":"kafk2a2","table":"hgx"},"after":{"name":"dd","id":"111216","rowKey":"111216"}}
/qhapp/cdh/kafka011/kafka-logs

--jar /qhapp/cdh/cloudera/parcels/CDH/jars/hive-hbase-handler-1.1.0-cdh5.15.0.jar

show master status
show binlog events in 'mysql-bin.000001
/qhapp/cdh/cloudera/parcels/CDH/jars

mysql主库链接时起始的binlog文件
mysql主库链接时起始的binlog偏移量
mysql主库链接时起始的binlog的时间戳

/qhapp/cdh/cloudera/parcels/CDH/jars
ln -s test.log test2.log
ln -s ../../../jars/jline-2.11.jar jline-2.11.jar
231067965

1533191555000
1483200000

把文件添加到/etc/ld.so.conf,然后执行命令ldconfig 生效
/usr/lib64/mysql/libmysqlclient_r.so.16.0.0
/usr/lib64/mysql/libmysqlclient.so.16.0.0

转载于:https://www.cnblogs.com/heguoxiu/p/10900410.html