azkaban源码编译安装(解决[Azkaban] Unable to get scheduled flow triggers问题)

1 安装Mysql

mysql安装文档有很多这里不详细描述

2 打包编译azkaban

由于azkaban编译需要下载很多依赖JAR，国外镜像比较慢，所以需要修改为国内镜像

yum -y install git
cd /root/.gradle/
vi init.gradle
# 添加如下内容
allprojects{
    repositories {
        def ALIYUN_REPOSITORY_URL = 'http://maven.aliyun.com/nexus/content/groups/public'
        def ALIYUN_JCENTER_URL = 'http://maven.aliyun.com/nexus/content/repositories/jcenter'
        all { ArtifactRepository repo ->
            if(repo instanceof MavenArtifactRepository){
                def url = repo.url.toString()
                if (url.startsWith('https://repo1.maven.org/maven2')) {
                    project.logger.lifecycle "Repository ${repo.url} replaced by $ALIYUN_REPOSITORY_URL."
                    remove repo
                }
                if (url.startsWith('https://jcenter.bintray.com/')) {
                    project.logger.lifecycle "Repository ${repo.url} replaced by $ALIYUN_JCENTER_URL."
                    remove repo
                }
            }
        }
        maven {
                url ALIYUN_REPOSITORY_URL
            url ALIYUN_JCENTER_URL
        }
    }
}

# 然后执行
cd /app/azkaban-master
./gradlew clean
./gradlew build installDist -x test

编译之后的安装文件路径为

azkaban-exec-server/build/distributions/
azkaban-solo-server/build/distributions/
azkaban-web-server/build/distributions/
azkaban-db/build/distributions/

将里面的文件拷贝到安装目录，里面有.tar.gz和zip两种，选择其中一个就行，只是压缩方式不一样

mkdir -p /app/azkaban
cp /app/azkaban-master/azkaban-exec-server/build/distributions/azkaban-exec-server-0.1.0-SNAPSHOT.tar.gz  /app/azkaban
cp /app/azkaban-master/azkaban-solo-server/build/distributions/azkaban-solo-server-0.1.0-SNAPSHOT.tar.gz /app/azkaban
cp /app/azkaban-master/azkaban-web-server/build/distributions/azkaban-web-server-0.1.0-SNAPSHOT.tar.gz /app/azkaban
mv azkaban-db-0.1.0-SNAPSHOT azkaban-db
mv azkaban-exec-server-0.1.0-SNAPSHOT azkaban-exec-serve
mv azkaban-solo-server-0.1.0-SNAPSHOT azkaban-solo-server
mv azkaban-web-server-0.1.0-SNAPSHOT azkaban-web-server

3 安装azkaban

3.1 配置mysql部分

mysql -uroot -proot
mysql> create database azkaban;
use azkaban;
mysql> CREATE USER 'username'@'%' IDENTIFIED BY 'password';
# give the user INSERT, SELECT, UPDATE, DELETE permission on all tables in the Azkaban db.
mysql> GRANT SELECT,INSERT,UPDATE,DELETE ON azkaban.* to ''@'%' WITH GRANT OPTION;

由于我使用的是mysql8所以需要替换项目lib中mysql-connector-java-5换为mysql-connector-java-8.0.19.jar

修改mysql配置文件

vi /etc/my.cnf
[mysqld]
...
max_allowed_packet=1024M

3.2 配置Azkaban Tables

mysql -uroot -p
use azkaban;
source /app/azkaban/azkaban-db/create-all-sql-0.1.0-SNAPSHOT.sql

3.3 配置azkaban-exec-serve

建议写为绝对路径

cd /app/azkaban/azkaban-exec-serve/conf
vi azkaban.properties

# Azkaban Personalization Settings
azkaban.name=Test
azkaban.label=My Local Azkaban
azkaban.color=#FF3601
azkaban.default.servlet.path=/index
web.resource.dir=/app/azkaban/azkaban-web-server/web/
default.timezone.id=Asia/Shanghai
# Azkaban UserManager class
user.manager.class=azkaban.user.XmlUserManager
user.manager.xml.file=/app/azkaban/azkaban-exec-server/conf/azkaban-users.xml
# Loader for projects
executor.global.properties=/app/azkaban/azkaban-exec-server/conf/global.properties
azkaban.project.dir=projects
# Velocity dev mode
velocity.dev.mode=false
# Azkaban Jetty server properties.
jetty.use.ssl=false
jetty.maxThreads=25
jetty.port=8081
# Where the Azkaban web server is located
azkaban.webserver.url=http://0.0.0.0:8081
# mail settings
mail.sender=
mail.host=
# User facing web server configurations used to construct the user facing server URLs. They are useful when there is a reverse proxy between Azkaban web servers and users.
# enduser -> myazkabanhost:443 -> proxy -> localhost:8081
# when this parameters set then these parameters are used to generate email links.
# if these parameters are not set then jetty.hostname, and jetty.port(if ssl configured jetty.ssl.port) are used.
# azkaban.webserver.external_hostname=myazkabanhost.com
# azkaban.webserver.external_ssl_port=443
# azkaban.webserver.external_port=8081
job.failure.email=
job.success.email=
lockdown.create.projects=false
cache.directory=cache
# JMX stats
jetty.connector.stats=true
executor.connector.stats=true
# Azkaban plugin settings
azkaban.jobtype.plugin.dir=/app/azkaban/azkaban-exec-server/plugins/jobtypes
# Azkaban mysql settings by default. Users should configure their own username and password.
database.type=mysql
mysql.port=3306
mysql.host=localhost
mysql.database=azkaban
mysql.user=root
mysql.password=root
mysql.numconnections=100
# Azkaban Executor settings
executor.maxThreads=50
executor.flow.threads=30

3.4 配置azkaban-web-server

注意：假如写相对路径请在项目根目录下启动，否则可能出现各种异常

# Azkaban Personalization Settings
azkaban.name=Test
azkaban.label=My Local Azkaban
azkaban.color=#FF3601
azkaban.default.servlet.path=/index
web.resource.dir=/app/azkaban/azkaban-web-server/web/
azkaban.jobtype.plugin.dir=/app/azkaban/azkaban-web-server/plugins/jobtypes
default.timezone.id=Asia/Shanghai
# Azkaban UserManager class
user.manager.class=azkaban.user.XmlUserManager
user.manager.xml.file=/app/azkaban/azkaban-web-server/conf/azkaban-users.xml
# Loader for projects
executor.global.properties=/app/azkaban/azkaban-web-server/conf/global.properties
azkaban.project.dir=projects
# Velocity dev mode
velocity.dev.mode=false
# Azkaban Jetty server properties.
jetty.use.ssl=false
jetty.maxThreads=25
jetty.port=8081
# Azkaban Executor settings
# mail settings
mail.sender=
mail.host=
# User facing web server configurations used to construct the user facing server URLs. They are useful when there is a reverse proxy between Azkaban web servers and users.
# enduser -> myazkabanhost:443 -> proxy -> localhost:8081
# when this parameters set then these parameters are used to generate email links.
# if these parameters are not set then jetty.hostname, and jetty.port(if ssl configured jetty.ssl.port) are used.
# azkaban.webserver.external_hostname=myazkabanhost.com
# azkaban.webserver.external_ssl_port=443
# azkaban.webserver.external_port=8081
job.failure.email=
job.success.email=
lockdown.create.projects=false
cache.directory=cache
# JMX stats
jetty.connector.stats=true
executor.connector.stats=true
# Azkaban mysql settings by default. Users should configure their own username and password.
database.type=mysql
mysql.port=3306
mysql.host=localhost
mysql.database=azkaban
mysql.user=root
mysql.password=root
mysql.numconnections=100
#Multiple Executor
azkaban.use.multiple.executors=true
azkaban.executorselector.filters=StaticRemainingFlowSize,CpuStatus
azkaban.executorselector.comparator.NumberOfAssignedFlowComparator=1
azkaban.executorselector.comparator.Memory=1
azkaban.executorselector.comparator.LastDispatched=1
azkaban.executorselector.comparator.CpuUsage=1

azkaban.server.schedule.enable_quartz=true
azkaban.dependency.plugin.dir=/app/azkaban/azkaban-web-server/plugins/dependency

org.quartz.dataSource.quartzDS.driver=com.mysql.jdbc.Driver
org.quartz.dataSource.quartzDS.URL=jdbc:mysql://10.0.9.59:3306/azkaban?useUnicode=true&characterEncoding=utf8
org.quartz.dataSource.quartzDS.user=root
org.quartz.dataSource.quartzDS.password=root
org.quartz.threadPool.threadCount=3
org.quartz.jobStore.class=org.quartz.impl.jdbcjobstore.JobStoreTX
org.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.StdJDBCDelegate
org.quartz.jobStore.tablePrefix=QRTZ_
org.quartz.jobStore.dataSource=quartzDS

3.5启动Executor Server

执行

cd azkaban-exec-server/build/install/azkaban-exec-server
./bin/start-exec.sh

然后执行

cd azkaban-exec-server/build/install/azkaban-exec-server
# executor.port在执行目录的executor.port文件中
cat executor.port
# curl -G "localhost:$(<./executor.port)/executor?action=activate" && echo

curl -G "localhost:33702/executor?action=activate" && echo

3.6 启动

cd /app/azkaban/azkaban-web-server
./bin/start-web.sh

3.7 访问

http://192.168.200.210:8081/

#默认密码
azkaban/azkaban