环境准备:https://blog.csdn.net/qq_39680564/article/details/98957057
安装包准备:https://blog.csdn.net/qq_39680564/article/details/99191318
yum -y install httpd createrepo
systemctl start httpd && systemctl enable httpd
cd /root/hadoop_CHD/cloudera-repos/ && createrepo .
结果
[root@cm cloudera-repos]# cd /root/hadoop_CHD/cloudera-repos/ && createrepo .
Spawning worker 0 with 3 pkgs
Spawning worker 1 with 3 pkgs
Workers Finished
Saving Primary metadata
Saving file lists metadata
Saving other metadata
Generating sqlite DBs
Sqlite DBs complete
将cloudera-repos目录移动到httpd的html目录下
mv /root/hadoop_CHD/cloudera-repos /var/www/html/
访问RPM包 http://39.98.224.169/cloudera-repos/
直接RPM安装官网提供的jdk
cd /var/www/html/cloudera-repos/
rpm -ivh oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm
yum -y remove mariadb-libs
解压之前下载的MySQL包
cd /root/hadoop_CHD/mysql/
tar -xvf mysql-5.7.27-1.el7.x86_64.rpm-bundle.tar
安装rpm包
yum install -y libaio \
&& rpm -ivh mysql-community-common-5.7.27-1.el7.x86_64.rpm \
&& rpm -ivh mysql-community-libs-5.7.27-1.el7.x86_64.rpm \
&& rpm -ivh mysql-community-client-5.7.27-1.el7.x86_64.rpm \
&& rpm -ivh mysql-community-server-5.7.27-1.el7.x86_64.rpm \
&& rpm -ivh mysql-community-libs-compat-5.7.27-1.el7.x86_64.rpm
vim /etc/my.cnf
官网推荐配置
[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
transaction-isolation = READ-COMMITTED
# Disabling symbolic-links is recommended to prevent assorted security risks;
# to do so, uncomment this line:
symbolic-links = 0
key_buffer_size = 32M
max_allowed_packet = 32M
thread_stack = 256K
thread_cache_size = 64
query_cache_limit = 8M
query_cache_size = 64M
query_cache_type = 1
max_connections = 550
#expire_logs_days = 10
#max_binlog_size = 100M
#log_bin should be on a disk with enough free space.
#Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your
#system and chown the specified folder to the mysql user.
log_bin=/var/lib/mysql/mysql_binary_log
#In later versions of MySQL, if you enable the binary log and do not set
#a server_id, MySQL will not start. The server_id must be unique within
#the replicating group.
server_id=1
binlog_format = mixed
read_buffer_size = 2M
read_rnd_buffer_size = 16M
sort_buffer_size = 8M
join_buffer_size = 8M
# InnoDB settings
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit = 2
innodb_log_buffer_size = 64M
innodb_buffer_pool_size = 4G
innodb_thread_concurrency = 8
innodb_flush_method = O_DIRECT
innodb_log_file_size = 512M
character-set-server=utf8
[mysqld_safe]
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid
sql_mode=STRICT_ALL_TABLES
启动服务
systemctl start mysqld
查看MySQL状态
[root@cm mysql]# systemctl status mysqld
● mysqld.service - MySQL Server
Loaded: loaded (/usr/lib/systemd/system/mysqld.service; enabled; vendor preset: disabled)
Active: active (running) since Sun 2019-08-11 23:10:13 CST; 1min 3s ago
Docs: man:mysqld(8)
http://dev.mysql.com/doc/refman/en/using-systemd.html
Process: 1947 ExecStart=/usr/sbin/mysqld --daemonize --pid-file=/var/run/mysqld/mysqld.pid $MYSQLD_OPTS (code=exited, status=0/SUCCESS)
Process: 1862 ExecStartPre=/usr/bin/mysqld_pre_systemd (code=exited, status=0/SUCCESS)
Main PID: 1951 (mysqld)
CGroup: /system.slice/mysqld.service
└─1951 /usr/sbin/mysqld --daemonize --pid-file=/var/run/mysqld/mysqld.pid
Aug 11 23:10:13 cm.cdh mysqld[1947]: 2019-08-11T15:10:13.335760Z 0 [Warning] CA certificate ca.pem is self signed.
Aug 11 23:10:13 cm.cdh mysqld[1947]: 2019-08-11T15:10:13.337683Z 0 [Note] Server hostname (bind-address): '*'; port: 3306
Aug 11 23:10:13 cm.cdh mysqld[1947]: 2019-08-11T15:10:13.337747Z 0 [Note] IPv6 is available.
Aug 11 23:10:13 cm.cdh mysqld[1947]: 2019-08-11T15:10:13.337761Z 0 [Note] - '::' resolves to '::';
Aug 11 23:10:13 cm.cdh mysqld[1947]: 2019-08-11T15:10:13.337784Z 0 [Note] Server socket created on IP: '::'.
Aug 11 23:10:13 cm.cdh mysqld[1947]: 2019-08-11T15:10:13.361833Z 0 [Note] Failed to start slave threads for channel ''
Aug 11 23:10:13 cm.cdh mysqld[1947]: 2019-08-11T15:10:13.366823Z 0 [Note] InnoDB: Buffer pool(s) load completed at 190811 23:10:13
Aug 11 23:10:13 cm.cdh mysqld[1947]: 2019-08-11T15:10:13.367377Z 0 [Note] Event Scheduler: Loaded 0 events
Aug 11 23:10:13 cm.cdh mysqld[1947]: 2019-08-11T15:10:13.367556Z 0 [Note] /usr/sbin/mysqld: ready for connections.
Aug 11 23:10:13 cm.cdh systemd[1]: Started MySQL Server.
获取初始密码,阿里云服务器获取不到初始密码解决方法:http://www.mamicode.com/info-detail-2505104.html
grep password /var/log/mysqld.log
修改root密码(密码策略为:不低于8位+数字+大小写字母+特殊字符)
mysql -uroot -p
mysql> set password for root@localhost = password('123456Aa.');
设置远程登录权限
mysql> grant all privileges on *.* to 'root'@'%' identified by '123456Aa.';
mysql> flush privileges;
解压mysql-connector-java-5.1.47.tar.gz文件
tar -zxvf mysql-connector-java-5.1.47.tar.gz
将解压出的mysql-connector-java-5.1.47-bin.jar
文件改名为mysql-connector-java.jar
,并复制到CM服务器节点上的/usr/share/java/目录下
mkdir -p /usr/share/java/
cd mysql-connector-java-5.1.47
cp mysql-connector-java-5.1.47-bin.jar /usr/share/java/mysql-connector-java.jar
Service | Database | User |
---|---|---|
Cloudera Manager Server | scm | scm |
Activity Monitor | amon | amon |
Reports Manager | rman | rman |
Hue | hue | hue |
Hive Metastore Server | metastore | hive |
Sentry Server | sentry | sentry |
Cloudera Navigator Audit Server | nav | nav |
Cloudera Navigator Metadata Server | navms | navms |
Oozie | oozie | oozie |
使用以下SQL语句建表建用户
CREATE DATABASE <database> DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON <database>.* TO '' @'%' IDENTIFIED BY '<password>';
建库
mysql> CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
Query OK, 1 row affected (0.00 sec)
mysql> CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
Query OK, 1 row affected (0.00 sec)
mysql> CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
Query OK, 1 row affected (0.00 sec)
mysql> CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
Query OK, 1 row affected (0.00 sec)
mysql> CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
Query OK, 1 row affected (0.00 sec)
mysql> CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
Query OK, 1 row affected (0.00 sec)
mysql> CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
Query OK, 1 row affected (0.00 sec)
mysql> CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
Query OK, 1 row affected (0.00 sec)
mysql> CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
Query OK, 1 row affected (0.00 sec)
授权
mysql> GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY '123456Aa.';
Query OK, 0 rows affected, 1 warning (0.00 sec)
mysql> GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY '123456Aa.';
Query OK, 0 rows affected, 1 warning (0.00 sec)
mysql> GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY '123456Aa.';
Query OK, 0 rows affected, 1 warning (0.01 sec)
mysql> GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY '123456Aa.';
Query OK, 0 rows affected, 1 warning (0.00 sec)
mysql> GRANT ALL ON metastore.* TO 'hive'@'%' IDENTIFIED BY '123456Aa.';
Query OK, 0 rows affected, 1 warning (0.00 sec)
mysql> GRANT ALL ON sentry.* TO 'sentry'@'%' IDENTIFIED BY '123456Aa.';
Query OK, 0 rows affected, 1 warning (0.00 sec)
mysql> GRANT ALL ON nav.* TO 'nav'@'%' IDENTIFIED BY '123456Aa.';
Query OK, 0 rows affected, 1 warning (0.00 sec)
mysql> GRANT ALL ON navms.* TO 'navms'@'%' IDENTIFIED BY '123456Aa.';
Query OK, 0 rows affected, 1 warning (0.00 sec)
mysql> GRANT ALL ON oozie.* TO 'oozie'@'%' IDENTIFIED BY '123456Aa.';
Query OK, 0 rows affected, 1 warning (0.00 sec)
查看数据库
mysql> SHOW DATABASES;
+--------------------+
| Database |
+--------------------+
| information_schema |
| amon |
| hue |
| metastore |
| mysql |
| nav |
| navms |
| oozie |
| performance_schema |
| rman |
| scm |
| sentry |
| sys |
+--------------------+
13 rows in set (0.00 sec)
查看授权
mysql> SHOW GRANTS FOR 'scm'@'%';
+----------------------------------------------+
| Grants for scm@% |
+----------------------------------------------+
| GRANT USAGE ON *.* TO 'scm'@'%' |
| GRANT ALL PRIVILEGES ON `scm`.* TO 'scm'@'%' |
+----------------------------------------------+
2 rows in set (0.00 sec)
mysql> SHOW GRANTS FOR 'amon'@'%';
+------------------------------------------------+
| Grants for amon@% |
+------------------------------------------------+
| GRANT USAGE ON *.* TO 'amon'@'%' |
| GRANT ALL PRIVILEGES ON `amon`.* TO 'amon'@'%' |
+------------------------------------------------+
2 rows in set (0.00 sec)
mysql> SHOW GRANTS FOR 'rman'@'%';
+------------------------------------------------+
| Grants for rman@% |
+------------------------------------------------+
| GRANT USAGE ON *.* TO 'rman'@'%' |
| GRANT ALL PRIVILEGES ON `rman`.* TO 'rman'@'%' |
+------------------------------------------------+
2 rows in set (0.00 sec)
mysql> SHOW GRANTS FOR 'hue'@'%';
+----------------------------------------------+
| Grants for hue@% |
+----------------------------------------------+
| GRANT USAGE ON *.* TO 'hue'@'%' |
| GRANT ALL PRIVILEGES ON `hue`.* TO 'hue'@'%' |
+----------------------------------------------+
mysql> SHOW GRANTS FOR 'hive'@'%';
+-----------------------------------------------------+
| Grants for hive@% |
+-----------------------------------------------------+
| GRANT USAGE ON *.* TO 'hive'@'%' |
| GRANT ALL PRIVILEGES ON `metastore`.* TO 'hive'@'%' |
+-----------------------------------------------------+
2 rows in set (0.00 sec)
mysql> SHOW GRANTS FOR 'sentry'@'%';
+----------------------------------------------------+
| Grants for sentry@% |
+----------------------------------------------------+
| GRANT USAGE ON *.* TO 'sentry'@'%' |
| GRANT ALL PRIVILEGES ON `sentry`.* TO 'sentry'@'%' |
+----------------------------------------------------+
2 rows in set (0.00 sec)
mysql> SHOW GRANTS FOR 'nav'@'%';
+----------------------------------------------+
| Grants for nav@% |
+----------------------------------------------+
| GRANT USAGE ON *.* TO 'nav'@'%' |
| GRANT ALL PRIVILEGES ON `nav`.* TO 'nav'@'%' |
+----------------------------------------------+
2 rows in set (0.00 sec)
mysql> SHOW GRANTS FOR 'navms'@'%';
+--------------------------------------------------+
| Grants for navms@% |
+--------------------------------------------------+
| GRANT USAGE ON *.* TO 'navms'@'%' |
| GRANT ALL PRIVILEGES ON `navms`.* TO 'navms'@'%' |
+--------------------------------------------------+
2 rows in set (0.00 sec)
mysql> SHOW GRANTS FOR 'oozie'@'%';
+--------------------------------------------------+
| Grants for oozie@% |
+--------------------------------------------------+
| GRANT USAGE ON *.* TO 'oozie'@'%' |
| GRANT ALL PRIVILEGES ON `oozie`.* TO 'oozie'@'%' |
+--------------------------------------------------+
2 rows in set (0.00 sec)
vim /etc/yum.repos.d/cloudera-manager.repo
内容如下
[cloudera-manager]
name=Cloudera Manager 6.3.0
baseurl=http://172.26.102.105/cloudera-repos/
gpgcheck=0
enabled=1
注意:baseurl有内外最好写内网,我写的就是阿里云的内网IP,这样同一局域网的服务器下载速度会很快。
生成yum缓存
yum clean all && yum makecache
yum安装agent、daemons、server
yum install -y cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server
查看是否安装成功
[root@cm ~]# rpm -qa|grep cloudera-manager
cloudera-manager-daemons-6.3.0-1281944.el7.x86_64
cloudera-manager-agent-6.3.0-1281944.el7.x86_64
cloudera-manager-server-6.3.0-1281944.el7.x86_64
将CDH-6.3.0-1.cdh6.3.0.p0.1279813-el7.parcel
和manifest.json
移到/opt/cloudera/parcel-repo/目录下
cd /opt/cloudera/parcel-repo/;mv /root/hadoop_CHD/parcel/* ./
生成sha文件
sha1sum CDH-6.3.0-1.cdh6.3.0.p0.1279813-el7.parcel | awk '{ print $1 }' > CDH-6.3.0-1.cdh6.3.0.p0.1279813-el7.parcel.sha
修改文件所有者
chown -R cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo/*
目录树
[root@cm parcel-repo]# ll
total 2036852
-rw-r--r-- 1 cloudera-scm cloudera-scm 2085690155 Aug 9 15:30 CDH-6.3.0-1.cdh6.3.0.p0.1279813-el7.parcel
-rw-r--r-- 1 cloudera-scm cloudera-scm 41 Aug 11 16:14 CDH-6.3.0-1.cdh6.3.0.p0.1279813-el7.parcel.sha
-rw-r--r-- 1 cloudera-scm cloudera-scm 33887 Aug 9 15:30 manifest.json
执行scm_prepare_database.sh脚本
格式:
/opt/cloudera/cm/schema/scm_prepare_database.sh [options] <databaseType> <databaseName> <databaseUser> <password>
a. mysql数据库与CM Server是同一台主机
[root@cm parcel-repo]# /opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm
Enter SCM password:
JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera
Verifying that we can write to /etc/cloudera-scm-server
Creating SCM configuration file in /etc/cloudera-scm-server
Executing: /usr/java/jdk1.8.0_181-cloudera/bin/java -cp /usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connector-java.jar:/usr/share/java/postgresql-connector-java.jar:/opt/cloudera/cm/schema/../lib/* com.cloudera.enterprise.dbutil.DbCommandExecutor /etc/cloudera-scm-server/db.properties com.cloudera.cmf.db.
Mon Aug 12 00:04:36 CST 2019 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.
[ main] DbCommandExecutor INFO Successfully connected to database.
All done, your SCM database is configured correctly!
b. mysql数据库与CM Server不在同一台主机上
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql -h db01.example.com --scm-host cm01.example.com scm scm
启动命令
systemctl start cloudera-scm-server
查看日志(8g两核的阿里云差不多三分钟启动时间)
tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log
当日志出现以下提示时,说明启动成功
INFO WebServerImpl:com.cloudera.server.cmf.WebServerImpl: Started Jetty server.
访问http://
默认账号密码为 admin/admin (阿里云服务器记得开安全组端口)
下一篇,安装第一个CDH节点:https://blog.csdn.net/qq_39680564/article/details/99301690