最近公司hadoop集群升级ambari迁移后,发现我们用的azkaban是solo-server模式的,所以我要搭建个分布式multiple-executor模式(这样才对的起新集群平台)
Azkaban三种模式安装前准备
Azkaban3.x在安装前需要自己编译成二进制包。
[root@g1-jg-hadoop-48 ~]# yum install git
[root@g1-jg-hadoop-48 ~]# yum install gcc-c++
[root@g1-jg-hadoop-48 ~]# wget https://github.com/azkaban/azkaban/archive/3.42.0.tar.gz
[root@g1-jg-hadoop-48 ~]# mv 3.42.0.tar.gz azkaban-3.42.0.tar.gz
[root@g1-jg-hadoop-48 ~]# tar -zxvf azkaban-3.42.0.tar.gz
[root@g1-jg-hadoop-48 ~]# ls
[root@g1-jg-hadoop-48 ~]# cd azkaban-3.42.0
[root@g1-jg-hadoop-48 azkaban-3.42.0]# ./gradlew build installDist -x test #Gradle是一个基于Apache Ant和Apache Maven的项目自动化构建工具。-x test 跳过测试
(他们的tar.zp在*/build/distributions/)
#solo-server模式安装包路径
[root@g1-jg-hadoop-48 azkaban-3.42.0]# ls azkaban-solo-server/build/distributions/
#two-server模式和multiple-executor模式web-server安装包路径
[root@g1-jg-hadoop-48 azkaban-3.42.0]# ls azkaban-web-server/build/distributions/
#two-server模式和multiple-executor模式exec-server安装包路径
[root@g1-jg-hadoop-48 azkaban-3.42.0]# ls azkaban-exec-server/build/distributions/
tar -zxvf /root/azkaban-3.42.0/azkaban-solo-server/build/distributions/azkaban-solo-server-0.1.0-SNAPSHOT.tar.gz -C .
tar -zxvf /root/azkaban-3.42.0/azkaban-web-server/build/distributions/azkaban-web-server-0.1.0-SNAPSHOT.tar.gz -C .
tar -zxvf /root/azkaban-3.42.0/azkaban-exec-server/build/distributions/azkaban-exec-server-0.1.0-SNAPSHOT.tar.gz -C .
scp -r /root/azkaban-3.42.0/azkaban-db/build/install/azkaban-db/create-all-sql-0.1.0-SNAPSHOT.sql root@g1-jg-hadoop-47:~
#创建数据库
mysql> CREATE DATABASE azkaban;
mysql> use azkaban;
mysql> CREATE USER 'azkaban'@'%' IDENTIFIED BY 'azkaban';#创建用户
mysql> GRANT SELECT,INSERT,UPDATE,DELETE ON azkaban.* to 'azkaban'@'%' WITH GRANT OPTION;#给用户授权
mysql> source create-all-sql-0.1.0-SNAPSHOT.sql;#创建表
[root@g1-jg-hadoop-48 ~]# keytool -keystore keystore -alias jetty -genkey -keyalg RSA
注:密码和最后确认需要输入,其他默认即可。
拷贝conf目录和log4j.properties
[root@g1-jg-hadoop-48 ~]# cp -r /root/azkaban-solo-server-0.1.0-SNAPSHOT/conf /root/azkaban-web-server-0.1.0-SNAPSHOT/
[root@g1-jg-hadoop-48 ~]# find azkaban-3.42.0 -name 'log4j*'
[root@g1-jg-hadoop-48 ~]# cp azkaban-3.42.0/azkaban-web-server/src/test/resources/log4j.properties azkaban-web-server-0.1.0-SNAPSHOT/conf/
[root@g1-jg-hadoop-48 ~]# vim azkaban-web-server-0.1.0-SNAPSHOT/conf/azkaban.properties
# Azkaban Personalization Settings
azkaban.use.multiple.executors=true
azkaban.executorselector.comparator.NumberOfAssignedFlowComparator=1
azkaban.executorselector.comparator.Memory=1
azkaban.executorselector.comparator.LastDispatched=1
azkaban.executorselector.comparator.CpuUsage=1
azkaban.name=Test
azkaban.label=My Local Azkaban
azkaban.color=#FF3601
azkaban.default.servlet.path=/index
web.resource.dir=web/
default.timezone.id=Asia/Shanghai
# Azkaban UserManager class
user.manager.class=azkaban.user.XmlUserManager
user.manager.xml.file=conf/azkaban-users.xml
# Loader for projects
executor.global.properties=conf/global.properties
azkaban.project.dir=projects
database.type=mysql
mysql.port=3306
mysql.host=10.16.128.22
mysql.database=azkaban
mysql.user=rd
mysql.password=!Rd123123
mysql.numconnections=100
velocity.dev.mode=false
# Azkaban Jetty server properties.
jetty.use.ssl=false
jetty.maxThreads=25
jetty.port=8081
# Azkaban Executor settings
executor.port=12321
jetty.keystore=/root/keystore
jetty.password=123456
jetty.keypassword=123456
jetty.truststore=/root/keystore
jetty.trustpassword=123456
# mail settings
mail.sender=
mail.host=
job.failure.email=
job.success.email=
lockdown.create.projects=false
cache.directory=cache
# JMX stats
jetty.connector.stats=true
executor.connector.stats=true
# Azkaban plugin settings
azkaban.jobtype.plugin.dir=plugins/jobtypes
[root@g1-jg-hadoop-48 ~]# cd azkaban-web-server-0.1.0-SNAPSHOT/
[root@g1-jg-hadoop-48 azkaban-web-server-0.1.0-SNAPSHOT]
# bin/azkaban-web-start.sh
验证: jps=>AzkabanWebServer webUI=>http://node1:8081/index
# Azkaban Personalization Settings
azkaban.name=Test
azkaban.label=My Local Azkaban
azkaban.color=#FF3601
azkaban.default.servlet.path=/index
web.resource.dir=web/
default.timezone.id=Asia/Shanghai
# Azkaban UserManager class
user.manager.class=azkaban.user.XmlUserManager
user.manager.xml.file=conf/azkaban-users.xml
# Loader for projects
executor.global.properties=conf/global.properties
azkaban.project.dir=projects
database.type=mysql
mysql.port=3306
mysql.host=10.16.128.22
mysql.database=azkaban
mysql.user=rd
mysql.password=!Rd123123
mysql.numconnections=100
velocity.dev.mode=false
# Azkaban Jetty server properties.
jetty.use.ssl=false
jetty.maxThreads=25
jetty.port=8081
# Azkaban Executor settings
executor.port=12321
jetty.keystore=/root/keystore
jetty.password=123456
jetty.keypassword=123456
jetty.truststore=/root/keystore
jetty.trustpassword=123456
# mail settings
mail.sender=
mail.host=
# User facing web server configurations used to construct the user facing server URLs. They are useful when there is a reverse proxy
between Azkaban web servers and users.
job.failure.email=
job.success.email=
lockdown.create.projects=false
cache.directory=cache
# JMX stats
jetty.connector.stats=true
executor.connector.stats=true
# Azkaban plugin settings
azkaban.jobtype.plugin.dir=plugins/jobtypes
11-1生成ssl (生成得路径是jetty.truststore=/root/keystore)
[root@g1-jg-hadoop-48 ~]# keytool -keystore keystore -alias jetty -genkey -keyalg RSA
注:密码和最后确认需要输入,其他默认即可。
insert into executors(host,port,active) values("10.16.128.23",12321,1);
insert into executors(host,port,active) values("10.16.128.22",12321,1);