|
master |
slave1 |
slave2 |
HDFS |
NameNode |
NameNode |
|
DataNode |
DataNode |
DataNode |
|
journalnode |
journalnode |
journalnode |
|
YARM |
Resourcemanager |
JobHistory Server |
Resourcemanager |
NodeManager |
NodeManager |
NodeManager |
|
zookeeper |
zk-server |
zk-server |
zk-server |
hive |
hiveserver2 |
|
|
mysql |
mysql |
|
|
hbase |
regionservers |
regionservers |
regionservers |
spark |
|
|
master |
Worker |
Worker |
Worker |
|
hue |
|
|
server |
livy.server |
|
|
livy.server |
tar -zxvf scala-2.11.12.tgz -C /usr/local
mv /usr/local/scala-2.11.12 /user/local/scala
vim /etc/profile
#scala
export PATH=$PATH:/usr/local/scala/bin
source /etc/profile
tar -zxvf mysql-5.7.21-linux-glibc2.12-x86_64.tar.gz -C /usr/local
groupadd mysql
useradd -r -g mysql mysql
chown mysql:mysql -R mysql-5.7.21/
vi /etc/my.cnf
[client]
port = 3306
socket = /usr/local/mysql-5.7.21/mysql.sock
default-character-set = utf8mb4
[mysqld_safe]
socket = /usr/local/mysql-5.7.21/mysql.sock
nice = 0
[mysqld]
sql_mode = NO_AUTO_CREATE_USER,STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION,NO_ZERO_DATE,NO_ZERO_IN_DATE
explicit_defaults_for_timestamp = 1
skip-ssl
character-set-server = utf8
user = root
socket = /usr/local/mysql.sock
port = 3306
basedir = /usr/local/mysql-5.7.21
datadir = /usr/local/mysql-5.7.21/data
tmpdir = /tmp
pid-file = /usr/local/mysqld.pid
back_log = 1000
max_connections = 5000
max_connect_errors = 10
max_allowed_packet = 32M
server-id=1
skip-external-locking
skip-name-resolve
#skip-grant-tables
wait_timeout = 31536000
interactive_timeout = 31536000
max_heap_table_size = 16M
tmp_table_size = 16M
query_cache_size = 0
query_cache_type = 0
ft_min_word_len = 4
log_error_verbosity = 3
log_error = /usr/local/mysql-5.7.21/log/alert_systemapi.log
slow_query_log = 1
slow_query_log_file = /usr/local/mysql-5.7.21/log/mysql_slow.log
long_query_time = 0.5
key_buffer_size = 32M
read_buffer_size = 32M
read_rnd_buffer_size = 8M
bulk_insert_buffer_size = 8M
myisam_sort_buffer_size = 8M
myisam_max_sort_file_size = 16M
myisam_repair_threads = 1
myisam_recover_options = default
log_bin = /usr/local/mysql-5.7.21/binlog/cdh-1-bin.log
binlog_format = row
expire_logs_days = 2
max_binlog_size = 100M
binlog_cache_size = 128M
innodb_autoextend_increment = 64
innodb_log_file_size = 16M
创建所需要的目录:
mkdir /usr/local/mysql-5.7.21/log
mkdir /usr/local/mysql-5.7.21/binlog/
mkdir /usr/local/mysql-5.7.21/data
/usr/local/mysql-5.7.21/bin/mysqld --defaults-file=/etc/my.cnf --user=mysql --initialize-insecure
打开配置文件/etc/my.cnf,然后在里面找到 [mysqld] 这一项,然后在该配置项下添加skip-grant-tables 这个配置,然后保存文件。
启动mysql:
cp /usr/local/mysql-5.7.21/support-files/mysql.server /etc/init.d/mysql
service mysql start
root登录修改root密码:
UPDATE mysql.user SET authentication_string=PASSWORD('Jkwl@2021') WHERE User='root';
update mysql.user set host='%' where user='root';
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%';
去除配置文件/etc/my.cnf,然后在里面找到 [mysqld] 这一项,删除 skip-grant-tables 这个配置,重启mysql
create database hive DEFAULT CHARACTER SET utf8;
create database hue DEFAULT CHARACTER SET utf8;
grant all on hue.* TO 'hue'@'%' IDENTIFIED BY 'Jkwl@2021';
grant all on hive.* TO 'hive'@'%' IDENTIFIED BY 'Jkwl@2021';
tar -zxvf apache-hive-3.1.2-bin.tar.gz -C /opt/module
mv apache-hive-3.1.2 hive
vim /etc/profile 添加
#hive
export HIVE_HOME=/opt/module/hive
export PATH=$PATH:$HIVE_HOME/bin
source /etc/profile
所有服务器都需要更改。
cp hive-env.sh.template hive-env.sh
vi hive-env.sh
export HIVE_HOME=/opt/module/hive
export PATH=$HIVE_HOME/bin:$PATH
export HIVE_AUX_JARS_PATH=$HIVE_HOME/bin
export HIVE_CONF_DIR=$HIVE_HOME/conf
export HADOOP_USER_CLASSPATH_FIRST=true
cp hive-default.xml.template hive-site.xml
vi hive-site.xml
javax.jdo.option.ConnectionDriverName
com.mysql.jdbc.Driver
Driver class name for a JDBC metastore
javax.jdo.option.ConnectionURL
jdbc:mysql://master:3306/hive?createDatabaseIfNotExist=true
JDBC connect string for a JDBC metastore
javax.jdo.option.ConnectionUserName
hive
Username to use against metastore database
javax.jdo.option.ConnectionPassword
Jkwl@2021
password to use against metastore database
将所有 ${system:Java.io.tmpdir}替换成/opt/module/hive/iotmp
初始化mysql中hive元数据表
./schematool -dbType mysql -initSchema
nohup hive --service hiveserver2 --hiveconf hive.server2.thrift.port=10001 &
出现错误:Exception in thread "main" java.lang.NoSuchMethodError: com.google.common.base.Preconditions.checkArgument(ZLjava/lang/String;Ljava/lang/Object;)V
解决办法:删除/opt/module/hive/lib/guava-19.0.jar;将/opt/module/hadoop-3.2.2/share/hadoop/common/lib/guava-27.0-jre.jar复制到/opt/module/hive/lib/
出现错误:com.ctc.wstx.exc.WstxParsingException: Illegal character entity: expansion character (code 0x8at [row,col,system-id]: [3215,96,"file:/opt/module/hive/conf/hive-site.xml"]
解决办法:vim /opt/module/hive/conf/hive-site.xml +3215 删除
出现告警:WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.
解决办法:
注意:其他服务器不用启动服务
tar -zxvf hbase-2.3.6-bin.tar.gz -C /opt/module/
mv hbase-2.3.6 hbase
vim /etc/profile 添加
#hive
export HBASE_HOME=/opt/module/hbase
export PATH=$PATH:$HBASE_HOME/bin
source /etc/profile
vim hbase-env.sh
export JAVA_HOME=/usr/java/jdk1.8.0_181/
export HBASE_LOG_DIR=${HBASE_HOME}/logs
export HBASE_MANAGES_ZK=false
hbase-site.xml 添加下面类容
hbase.rootdir
hdfs://master:9000/hbase
hbase.cluster.distributed
true
hbase.master
hdfs://slave2:60000
hbase.zookeeper.quorum
master,slave1,slave2
vim regionservers
master
slave1
slave2
start-hbase.sh
日志报错:java.lang.IllegalArgumentException: object is not an instance of declaring class
Hadoop 3.3.X不支持hbase 2.3.X 换成2.4.6也不行。
重新安装hadoop3.2.X,检查是否安装成功:
[hadoop@master ~]$ hbase shell
hbase:001:0> list
TABLE
0 row(s)
Took 0.6578 seconds
=> []
任意一台启动thrift服务:hbase-daemon.sh start thrift
tar -zxvf spark-3.1.2-bin-hadoop3.2.tgz -C /opt/module/
mv spark-3.1.2 spark
vim /etc/profile
#spark
export SPARK_HOME=/opt/module/spark
export PATH=$PATH:$SPARK_HOME/bin
source /etc/profile
mv spark-env.sh.template spark-env.sh
vim spark-env.sh
export SCALA_HOME=/usr/local/scala
export JAVA_HOME=/usr/java/jdk1.8.0_181
export SPARK_MASTER_IP=192.168.159.134
export SPARK_WORKER_MEMORY=512m
export HADOOP_CONF_DIR=/opt/module/hadoop-3.2.2/etc/Hadoop
vim workers
master
slave1
slave2
下发到其他节点。
/opt/module/spark/sbin/start-all.sh
unzip apache-livy-0.7.1-incubating-bin.zip
mv apache-livy-0.7.1-incubating-bin /opt/module/livy
unzip hue-release-4.3.0.zip
mv hue-release-4.3.0 hue /opt/module/hue
Livy conf
mv livy.conf.template livy.conf
vim livy.conf
#默认使用hiveContext
livy.repl.enableHiveContext = true
#开启用户代理
livy.impersonation.enabled = true
#设置session空闲过期时间
livy.server.session.timeout = 1h
mv livy-env.sh.template livy-env.sh
vim livy-env.sh
export HADOOP_CONF_DIR=/opt/module/hadoop-3.2.2/etc/hadoop
export SPARK_HOME=/opt/module/spark
mv spark-blacklist.conf.template spark-blacklist.conf
vim spark-blacklist.conf
# spark.master
###########################################################################
Hue conf
yum install ant asciidoc cyrus-sasl-devel cyrus-sasl-gssapi cyrus-sasl-plain gcc gcc-c++
yum install krb5-devel libffi-devel libtidy libxml2-devel libxslt-devel make maven mysql-devel
yum install openldap-devel python-devel
yum install sqlite-devel openssl-devel
yum install gmp-devel
vim /etc/maven/settings.xml
alimaven
aliyun maven
http://maven.aliyun.com/nexus/content/groups/public/
central
cd /opt/module/hue
make apps
此时可能会出现错误:
SyntaxError: invalid syntax
默认安装traitlets 5.0.5 不支持python 2.7
解决办法:
vim build/env/lib/python2.7/site-packages/setuptools/command/easy_install.py +328
self.index_url = self.index_url or http://pypi.doubanio.com/simple/
./build/env/bin/easy_install traitlets==4.2.2
重新make apps
cd /opt/module/hue/desktop/conf
mv pseudo-distributed.ini.tmpl pseudo-distributed.ini
vim pseudo-distributed.ini
[desktop]
secret_key=dhaiodheqdhqiodhaldhalcbaklhfoadjaoda
http_host=slave2
http_port=8888
time_zone=Asia/Shanghai
dev=true
send_dbug_messages=true
erver_user=hadoop
server_group=hadoop
default_user=hadoop
default_hdfs_superuser=hadoop
[[django_admins]]
[[custom]]
[[auth]]
idle_session_timeout=-1
[[ldap]]
[[[users]]]
[[[groups]]]
[[[ldap_servers]]]
[[vcs]]
[[database]]
engine=mysql
host=master
port=3306
user=hue
password=Jkwl@2021
name=hue
[[session]]
[[smtp]]
host=localhost
port=25
user=
password=
tls=no
[[kerberos]]
[[oauth]]
[[oidc]]
[[metrics]]
[notebook]
[[interpreters]]
[[[hive]]]
name=Hive
interface=hiveserver2
[[[impala]]]
name=Impala
interface=hiveserver2
[[[spark]]]
name=Scala
interface=livy
[[[pyspark]]]
name=PySpark
interface=livy
[[[r]]]
name=R
interface=livy
[[[jar]]]
name=Spark Submit Jar
interface=livy-batch
[[[py]]]
name=Spark Submit Python
interface=livy-batch
[[[text]]]
name=Text
interface=text
[[[markdown]]]
name=Markdown
interface=text
[[[mysql]]]
name = MySQL
interface=rdbms
[[[sqlite]]]
name = SQLite
interface=rdbms
[[[postgresql]]]
name = PostgreSQL
interface=rdbms
[[[oracle]]]
name = Oracle
interface=rdbms
[[[solr]]]
name = Solr SQL
interface=solr
[[[pig]]]
name=Pig
interface=oozie
[[[java]]]
name=Java
interface=oozie
[[[spark2]]]
name=Spark
interface=oozie
[[[mapreduce]]]
name=MapReduce
interface=oozie
[[[sqoop1]]]
name=Sqoop1
interface=oozie
[[[distcp]]]
name=Distcp
interface=oozie
[[[shell]]]
name=Shell
interface=oozie
[dashboard]
[[engines]]
[hadoop]
[[hdfs_clusters]]
[[[default]]]
fs_defaultfs=hdfs://myNameNodeServer
webhdfs_url=http://master:14000/webhdfs/v1
hadoop_conf_dir=/opt/module/hadoop-3.2.2/etc/hadoop
[[yarn_clusters]]
[[[default]]]
resourcemanager_host=master
resourcemanager_port=8032
submit_to=True
history_server_api_url=http://slave1:19888
logical_name=yarnRM
resourcemanager_api_url=http://master:8088
[beeswax]
hive_server_host=master
hive_server_port=10001
[[ssl]]
[metastore]
[impala]
[[ssl]]
[spark]
[oozie]
[filebrowser]
[pig]
[sqoop]
[proxy]
[hbase]
hbase_clusters=(Cluster|master:9090,slave1:9090,slave2;9090)
hbase_conf_dir=/opt/module/hbase/conf
thrift_transport=framed
[search]
[libsolr]
[indexer]
[jobsub]
[jobbrowser]
[security]
[zookeeper]
[[clusters]]
[[[default]]]
host_ports=master:2181,slave1:2181,slave2:2181
principal_name=zookeeper
[useradmin]
[[password_policy]]
[liboozie]
[aws]
[[aws_accounts]]
[azure]
[[azure_accounts]]
[[[default]]]
[[adls_clusters]]
[[[default]]]
[libsentry]
[libzookeeper]
[librdbms]
[[databases]]
nice_name="My SQL DB"
name=mysql
engine=mysql
host=master
port=3306
user=root
password=Jkwl@2021
[libsaml]
[libopenid]
[liboauth]
[kafka]
[[kafka]]
[metadata]
[[optimizer]]
[[navigator]]
cd /opt/module/livy/bin/
nohup ./livy-server >/dev/null 2>&1 &
初始化hue数据库:
./build/env/bin/hue syncdb
./build/env/bin/hue migrate
cd /opt/module/hue/build/env/bin/
nohup ./supervisor &