Centos7部署AirFlow

Centos7部署AirFlow

官方文档

1.准备环境

1.1.安装Python3

yum install epel-release -y
yum install https://centos7.iuscommunity.org/ius-release.rpm -y
yum install python36u -y
ln -s /bin/python3.6 /bin/python3
yum install python36u-pip -y
ln -s /bin/pip3.6 /bin/pip3

1.2.换pip国内源

mkdir ~/.pip
vim ~/.pip/pip.conf

[global]
index-url = https://pypi.tuna.tsinghua.edu.cn/simple

1.3.配置虚拟环境

mkdir -p /usr/local/thirdparty/
cd /usr/local/thirdparty/
python3 -m venv airflow_env
source /usr/local/thirdparty/airflow_env/bin/activate

1.4.安装操作系统依赖

yum install -y gcc gcc-c++ python-devel python36u-devel

2.安装AirFlow

• 安装

export SLUGIFY_USES_TEXT_UNIDECODE=yes
pip install apache-airflow[all]

• 启动测试

mkdir /usr/local/airflow
cd /usr/local/airflow
export AIRFLOW_HOME=`pwd`
airflow initdb
# 开启两个终端，第一个执行
airflow webserver -p 8080
# 第二个执行
airflow scheduler

浏览器访问http://ip:8080查看服务是否启动成功

3.Systemd部署

3.1.创建MySQL库和用户

mysql> CREATE DATABASE airflow; 
mysql> GRANT all privileges on airflow.* TO 'airflow'@'localhost'  IDENTIFIED BY 'airflow'; 
mysql> FLUSH PRIVILEGES; 

3.2.创建airflow系统用户

useradd airflow

3.3.修改配置

vim /usr/local/airflow/airflow.cfg

# [core]部分修改
# 替换成mysql配置
sql_alchemy_conn =  mysql://airflow:[email protected]:3306/airflow
# 移除示例Dags
load_examples = False

# [webserver] 部分修改
# 开启认证
authenticate = True
# 添加认证后端(该项配置需要新增)
auth_backend = airflow.contrib.auth.backends.password_auth
# 按用户过滤
filter_by_owner = True

3.4.修改/usr/local/airflow属主

chown -R airflow:airflow /usr/local/airflow

3.5.配置systemd相关服务文件

• 获取文件

GitHub获取airflow源码,启动脚本在scripts/systemd目录下

获取systemd目录到/root下

cd /root/systemd

• 配置airflow文件

vim airflow

替换配置

AIRFLOW_CONFIG=/usr/local/airflow/airflow.cfg
AIRFLOW_HOME=/usr/local/airflow

• 配置airflow-webserver.service文件

 vim airflow-webserver.service
 

替换启动脚本成如下

ExecStart=/bin/sh -c 'source /usr/local/thirdparty/airflow_env/bin/activate && airflow webserver --pid /run/airflow/webserver.pid'

• 配置airflow-scheduler.service文件

vim airflow-scheduler.service

替换启动脚本成如下

ExecStart=/bin/sh -c 'source /usr/local/thirdparty/airflow_env/bin/activate && airflow scheduler'

• 配置airflow-flower.service文件

 vim airflow-flower.service

替换启动脚本成如下

ExecStart=/bin/sh -c 'source /usr/local/thirdparty/airflow_env/bin/activate && airflow flower'

• 配置airflow-worker.service文件

vim airflow-worker.service

替换启动脚本成如下

ExecStart=/bin/sh -c 'source /usr/local/thirdparty/airflow_env/bin/activate && airflow worker'

• 移动相关文件到对应目录

mkdir /run/airflow
chown -R airflow:airflow /run/airflow

cp airflow /etc/sysconfig/

cp airflow.conf /etc/tmpfiles.d/

cp airflow-*.service /usr/lib/systemd/system/

3.6.初始化db

source /usr/local/thirdparty/airflow_env/bin/activate
cd /usr/local/airflow
export AIRFLOW_HOME=`pwd`
airflow initdb

3.7.创建用户

source /usr/local/thirdparty/airflow_env/bin/activate
cd /usr/local/airflow
export AIRFLOW_HOME=`pwd`
vim user_manger.py

复制如下内容

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
from airflow import models, settings
from airflow.contrib.auth.backends.password_auth import PasswordUser


class UserManager(object):

    def __init__(self):
        self.user = models.User()
        self.passwd_user = PasswordUser(self.user)
        self.session = settings.Session()

    def add_user(self, username, passwd, email=None, is_super=False):
        self.passwd_user.username = username
        self.passwd_user.password = passwd
        if email:
            self.passwd_user.email = email
        if is_super is True:
            self.passwd_user.superuser = True
        self.session.add(self.passwd_user)
        self.session.commit()

    def change_passwd(self, username, new_passwd=None, email=None, superuser=None):
        if not any([new_passwd, email, superuser]):
            return
        info = {}
        if new_passwd:
            self.passwd_user.password = new_passwd
            info['password'] = self.passwd_user.password
        if email:
            info['email'] = email
        if superuser is True:
            info['superuser'] = True
        if superuser is False:
            info['superuser'] = False
        try:
            self.session.query(PasswordUser).filter(PasswordUser.username == username).update(info)
        except Exception as e:
            pass
        else:
            self.session.commit()

    def __del__(self):
        self.session.close()


if __name__ == '__main__':

    manager = UserManager()
    manager.add_user('admin', 'admin', 'admin@localhost', True)
    # manager.change_passwd('admin', 'admin', 'admin@localhost', True)

python user_manger.py

3.8.启动服务

systemctl start airflow-webserver.service
systemctl start airflow-scheduler.service

3.9.设置开机启动

systemctl enable airflow-webserver.service
systemctl enable airflow-scheduler.service

4.访问web

浏览器访问http://ip:8080

admin/admin