Airflow安装教程
1. 安装python-pip
[root@node1 ~]# yum install -y python-pip2. 安装airflow
[root@node1 ~]# pip install airflow
如果上面命令安装较慢，可以使用下面命令国内源安装。
[root@node1 ~]# pip install -i https://pypi.tuna.tsinghua.edu.cn/simple airflow3.初始化数据库
airflow默认使用sqlite作为数据库, 直接执行数据库初始化命令后, 会在环境变量路径下新建一个数据库文件airflow.db
[root@node1 ~]# airflow initdb
[2017-10-06 10:10:45,462] {__init__.py:57} INFO - Using executor SequentialExecutor
DB: sqlite:////root/airflow/airflow.db
[2017-10-06 10:10:46,218] {db.py:287} INFO - Creating tables
INFO [alembic.runtime.migration] Context impl SQLiteImpl.
INFO [alembic.runtime.migration] Will assume non-transactional DDL.
INFO [alembic.runtime.migration] Running upgrade -> e3a246e0dc1, current schema
INFO [alembic.runtime.migration] Running upgrade e3a246e0dc1 -> 1507a7289a2f, create is_encrypted
/usr/lib/python2.7/site-packages/alembic/util/messaging.py:69: UserWarning: Skipping unsupported ALTER for creation of implicit constraint
warnings.warn(msg)
INFO [alembic.runtime.migration] Running upgrade 1507a7289a2f -> 13eb55f81627, maintain history for compatibility with earlier migrations
INFO [alembic.runtime.migration] Running upgrade 13eb55f81627 -> 338e90f54d61, More logging into task_isntance
INFO [alembic.runtime.migration] Running upgrade 338e90f54d61 -> 52d714495f0, job_id indices
INFO [alembic.runtime.migration] Running upgrade 52d714495f0 -> 502898887f84, Adding extra to Log
INFO [alembic.runtime.migration] Running upgrade 502898887f84 -> 1b38cef5b76e, add dagrun
INFO [alembic.runtime.migration] Running upgrade 1b38cef5b76e -> 2e541a1dcfed, task_duration
INFO [alembic.runtime.migration] Running upgrade 2e541a1dcfed -> 40e67319e3a9, dagrun_config
INFO [alembic.runtime.migration] Running upgrade 40e67319e3a9 -> 561833c1c74b, add password column to user
INFO [alembic.runtime.migration] Running upgrade 561833c1c74b -> 4446e08588, dagrun start end
INFO [alembic.runtime.migration] Running upgrade 4446e08588 -> bbc73705a13e, Add notification_sent column to sla_miss
INFO [alembic.runtime.migration] Running upgrade bbc73705a13e -> bba5a7cfc896, Add a column to track the encryption state of the 'Extra' field in connection
INFO [alembic.runtime.migration] Running upgrade bba5a7cfc896 -> 1968acfc09e3, add is_encrypted column to variable table
INFO [alembic.runtime.migration] Running upgrade 1968acfc09e3 -> 2e82aab8ef20, rename user table
INFO [alembic.runtime.migration] Running upgrade 2e82aab8ef20 -> 211e584da130, add TI state index
INFO [alembic.runtime.migration] Running upgrade 211e584da130 -> 64de9cddf6c9, add task fails journal table
INFO [alembic.runtime.migration] Running upgrade 64de9cddf6c9 -> f2ca10b85618, add dag_stats table
INFO [alembic.runtime.migration] Running upgrade f2ca10b85618 -> 4addfa1236f1, Add fractional seconds to mysql tables
INFO [alembic.runtime.migration] Running upgrade 4addfa1236f1 -> 8504051e801b, xcom dag task indices
INFO [alembic.runtime.migration] Running upgrade 8504051e801b -> 5e7d17757c7a, add pid field to TaskInstance
INFO [alembic.runtime.migration] Running upgrade 5e7d17757c7a -> 127d2bf2dfa7, Add dag_id/state index on dag_run table
Done.
[root@node1 ~]# 4. 启动airflow webserver
默认端口为8080
[root@node1 ~]# airflow webserver
[2017-10-06 10:11:37,313] {__init__.py:57} INFO - Using executor SequentialExecutor
____________ _____________
____ |__( )_________ __/__ /________ __
____ /| |_ /__ ___/_ /_ __ /_ __ \_ | /| / /
___ ___ | / _ / _ __/ _ / / /_/ /_ |/ |/ /
_/_/ |_/_/ /_/ /_/ /_/ \____/____/|__/
/usr/lib64/python2.7/site-packages/flask/exthook.py:71: ExtDeprecationWarning: Importing flask.ext.cache is deprecated, use flask_cache instead.
.format(x=modname), ExtDeprecationWarning
[2017-10-06 10:11:37,969] [3087] {models.py:167} INFO - Filling up the DagBag from /root/airflow/dags
Running the Gunicorn Server with:
Workers: 4 sync
Host: 0.0.0.0:8080
Timeout: 120
Logfiles: - -
=================================================================
[2017-10-06 10:11:38,907] {__init__.py:57} INFO - Using executor SequentialExecutor
[2017-10-06 10:11:39 +0000] [3094] [INFO] Starting gunicorn 19.3.0
[2017-10-06 10:11:39 +0000] [3094] [INFO] Listening at: http://0.0.0.0:8080 (3094)
[2017-10-06 10:11:39 +0000] [3094] [INFO] Using worker: sync
[2017-10-06 10:11:39 +0000] [3103] [INFO] Booting worker with pid: 3103
[2017-10-06 10:11:39 +0000] [3104] [INFO] Booting worker with pid: 3104
[2017-10-06 10:11:39 +0000] [3105] [INFO] Booting worker with pid: 3105
/usr/lib64/python2.7/site-packages/flask/exthook.py:71: ExtDeprecationWarning: Importing flask.ext.cache is deprecated, use flask_cache instead.
.format(x=modname), ExtDeprecationWarning
[2017-10-06 10:11:39 +0000] [3106] [INFO] Booting worker with pid: 3106
/usr/lib64/python2.7/site-packages/flask/exthook.py:71: ExtDeprecationWarning: Importing flask.ext.cache is deprecated, use flask_cache instead.
.format(x=modname), ExtDeprecationWarning
/usr/lib64/python2.7/site-packages/flask/exthook.py:71: ExtDeprecationWarning: Importing flask.ext.cache is deprecated, use flask_cache instead.
.format(x=modname), ExtDeprecationWarning
/usr/lib64/python2.7/site-packages/flask/exthook.py:71: ExtDeprecationWarning: Importing flask.ext.cache is deprecated, use flask_cache instead.
.format(x=modname), ExtDeprecationWarning
[2017-10-06 10:11:40,897] [3103] {models.py:167} INFO - Filling up the DagBag from /root/airflow/dags
[2017-10-06 10:11:41,081] [3104] {models.py:167} INFO - Filling up the DagBag from /root/airflow/dags
[2017-10-06 10:11:41,448] [3105] {models.py:167} INFO - Filling up the DagBag from /root/airflow/dags
[2017-10-06 10:11:41,641] [3106] {models.py:167} INFO - Filling up the DagBag from /root/airflow/dags
5 安装airflow其间可能遇到的问题
（1）fatal error: Python.h: No such file or directory
creating build/temp.linux-x86_64-2.7/psutil
gcc -pthread -fno-strict-aliasing -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic -D_GNU_SOURCE -fPIC -fwrapv -DNDEBUG -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic -D_GNU_SOURCE -fPIC -fwrapv -fPIC -DPSUTIL_POSIX=1 -DPSUTIL_VERSION=442 -DPSUTIL_LINUX=1 -I/usr/include/python2.7 -c psutil/_psutil_linux.c -o build/temp.linux-x86_64-2.7/psutil/_psutil_linux.o
psutil/_psutil_linux.c:12:20: fatal error: Python.h: No such file or directory
#include <Python.h>
^
compilation terminated.
error: command 'gcc' failed with exit status 1
（2）解决办法
[root@node1 ~]# yum install python-devel（3）重新安装
[root@node1 ~]# pip install -i https://pypi.tuna.tsinghua.edu.cn/simple airflow
Collecting airflow
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/e7/ac/5f1ec362fc0695167d29b3c7b6f28d79898f1221e5a32ab1c6e651a55564/airflow-1.8.0.tar.gz
Requirement already satisfied (use --upgrade to upgrade): alembic<0.9,>=0.8.3 in /usr/lib/python2.7/site-packages (from airflow)
Requirement already satisfied (use --upgrade to upgrade): croniter<0.4,>=0.3.8 in /usr/lib/python2.7/site-packages (from airflow)
Requirement already satisfied (use --upgrade to upgrade): dill<0.3,>=0.2.2 in /usr/lib/python2.7/site-packages (from airflow)
Requirement already satisfied (use --upgrade to upgrade): flask<0.12,>=0.11 in /usr/lib64/python2.7/site-packages (from airflow)
Requirement already satisfied (use --upgrade to upgrade): flask-admin==1.4.1 in /usr/lib/python2.7/site-packages (from airflow)
Requirement already satisfied (use --upgrade to upgrade): flask-cache<0.14,>=0.13.1 in /usr/lib/python2.7/site-packages (from airflow)
Requirement already satisfied (use --upgrade to upgrade): flask-login==0.2.11 in /usr/lib/python2.7/site-packages (from airflow)
Requirement already satisfied (use --upgrade to upgrade): flask-swagger==0.2.13 in /usr/lib/python2.7/site-packages (from airflow)
Requirement already satisfied (use --upgrade to upgrade): flask-wtf==0.12 in /usr/lib64/python2.7/site-packages (from airflow)
Requirement already satisfied (use --upgrade to upgrade): funcsigs==1.0.0 in /usr/lib/python2.7/site-packages (from airflow)
Requirement already satisfied (use --upgrade to upgrade): future<0.16,>=0.15.0 in /usr/lib/python2.7/site-packages (from airflow)
Requirement already satisfied (use --upgrade to upgrade): gitpython>=2.0.2 in /usr/lib64/python2.7/site-packages (from airflow)
Requirement already satisfied (use --upgrade to upgrade): gunicorn<19.4.0,>=19.3.0 in /usr/lib/python2.7/site-packages (from airflow)
Requirement already satisfied (use --upgrade to upgrade): jinja2<2.9.0,>=2.7.3 in /usr/lib64/python2.7/site-packages (from airflow)
Requirement already satisfied (use --upgrade to upgrade): lxml<4.0,>=3.6.0 in /usr/lib64/python2.7/site-packages (from airflow)
Requirement already satisfied (use --upgrade to upgrade): markdown<3.0,>=2.5.2 in /usr/lib/python2.7/site-packages (from airflow)
Requirement already satisfied (use --upgrade to upgrade): pandas<1.0.0,>=0.17.1 in /usr/lib64/python2.7/site-packages (from airflow)
Collecting psutil<5.0.0,>=4.2.0 (from airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/6c/49/0f784a247868e167389f6ac76b8699b2f3d6f4e8e85685dfec43e58d1ed1/psutil-4.4.2.tar.gz
Collecting pygments<3.0,>=2.0.1 (from airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/02/ee/b6e02dc6529e82b75bb06823ff7d005b141037cb1416b10c6f00fc419dca/Pygments-2.2.0-py2.py3-none-any.whl
Collecting python-daemon<2.2,>=2.1.1 (from airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/f4/59/816004688f8e8602526553cd96226f34657ce4a86daa2240c3eebb0568a3/python_daemon-2.1.2-py2.py3-none-any.whl
Requirement already satisfied (use --upgrade to upgrade): python-dateutil<3,>=2.3 in /usr/lib/python2.7/site-packages (from airflow)
Collecting python-nvd3==0.14.2 (from airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/30/68/5d5d7c1a46de23f1da3b4a7035ac305b76aba582648d19cd9da89b5bd8f6/python-nvd3-0.14.2.tar.gz
Collecting requests<3,>=2.5.1 (from airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/49/df/50aa1999ab9bde74656c2919d9c0c085fd2b3775fd3eca826012bef76d8c/requests-2.18.4-py2.py3-none-any.whl
Collecting setproctitle<2,>=1.1.8 (from airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/5a/0d/dc0d2234aacba6cf1a729964383e3452c52096dc695581248b548786f2b3/setproctitle-1.1.10.tar.gz
Requirement already satisfied (use --upgrade to upgrade): sqlalchemy>=0.9.8 in /usr/lib/python2.7/site-packages (from airflow)
Collecting tabulate<0.8.0,>=0.7.5 (from airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/a5/8d/86bf900d62216e2be7806d2ff4615cb7da54e13aeb7765549310c355cbae/tabulate-0.7.7-py2.py3-none-any.whl
Collecting thrift<0.10,>=0.9.2 (from airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/ae/58/35e3f0cd290039ff862c2c9d8ae8a76896665d70343d833bdc2f748b8e55/thrift-0.9.3.tar.gz
Collecting zope.deprecation<5.0,>=4.0 (from airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/ee/33/625098914ec59b3006adf2cdf44a721e9671f4836af9eeb8cbe14e485954/zope.deprecation-4.3.0-py2.py3-none-any.whl
Requirement already satisfied (use --upgrade to upgrade): Mako in /usr/lib/python2.7/site-packages (from alembic<0.9,>=0.8.3->airflow)
Requirement already satisfied (use --upgrade to upgrade): python-editor>=0.3 in /usr/lib/python2.7/site-packages (from alembic<0.9,>=0.8.3->airflow)
Requirement already satisfied (use --upgrade to upgrade): Werkzeug>=0.7 in /usr/lib/python2.7/site-packages (from flask<0.12,>=0.11->airflow)
Requirement already satisfied (use --upgrade to upgrade): click>=2.0 in /usr/lib/python2.7/site-packages (from flask<0.12,>=0.11->airflow)
Requirement already satisfied (use --upgrade to upgrade): itsdangerous>=0.21 in /usr/lib/python2.7/site-packages (from flask<0.12,>=0.11->airflow)
Requirement already satisfied (use --upgrade to upgrade): wtforms in /usr/lib/python2.7/site-packages (from flask-admin==1.4.1->airflow)
Requirement already satisfied (use --upgrade to upgrade): PyYAML>=3.0 in /usr/lib64/python2.7/site-packages (from flask-swagger==0.2.13->airflow)
Requirement already satisfied (use --upgrade to upgrade): ordereddict in /usr/lib/python2.7/site-packages (from funcsigs==1.0.0->airflow)
Requirement already satisfied (use --upgrade to upgrade): gitdb2>=2.0.0 in /usr/lib/python2.7/site-packages (from gitpython>=2.0.2->airflow)
Requirement already satisfied (use --upgrade to upgrade): MarkupSafe in /usr/lib64/python2.7/site-packages (from jinja2<2.9.0,>=2.7.3->airflow)
Requirement already satisfied (use --upgrade to upgrade): numpy>=1.7.0 in /usr/lib64/python2.7/site-packages (from pandas<1.0.0,>=0.17.1->airflow)
Requirement already satisfied (use --upgrade to upgrade): pytz>=2011k in /usr/lib/python2.7/site-packages (from pandas<1.0.0,>=0.17.1->airflow)
Requirement already satisfied (use --upgrade to upgrade): setuptools in /usr/lib/python2.7/site-packages (from python-daemon<2.2,>=2.1.1->airflow)
Collecting docutils (from python-daemon<2.2,>=2.1.1->airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/50/09/c53398e0005b11f7ffb27b7aa720c617aba53be4fb4f4f3f06b9b5c60f28/docutils-0.14-py2-none-any.whl
Collecting lockfile>=0.10 (from python-daemon<2.2,>=2.1.1->airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/c8/22/9460e311f340cb62d26a38c419b1381b8593b0bb6b5d1f056938b086d362/lockfile-0.12.2-py2.py3-none-any.whl
Requirement already satisfied (use --upgrade to upgrade): six>=1.5 in /usr/lib/python2.7/site-packages (from python-dateutil<3,>=2.3->airflow)
Collecting python-slugify==1.1.4 (from python-nvd3==0.14.2->airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/2e/13/838ba9b35375ed05bc3b959c81f22ef49e7bdb426063bee888d6f2dc84f0/python-slugify-1.1.4.tar.gz
Collecting certifi>=2017.4.17 (from requests<3,>=2.5.1->airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/40/66/06130724e8205fc8c105db7edb92871c7fff7d31324d7f4405c762624a43/certifi-2017.7.27.1-py2.py3-none-any.whl
Collecting chardet<3.1.0,>=3.0.2 (from requests<3,>=2.5.1->airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl
Collecting idna<2.7,>=2.5 (from requests<3,>=2.5.1->airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/27/cc/6dd9a3869f15c2edfab863b992838277279ce92663d334df9ecf5106f5c6/idna-2.6-py2.py3-none-any.whl
Collecting urllib3<1.23,>=1.21.1 (from requests<3,>=2.5.1->airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/63/cb/6965947c13a94236f6d4b8223e21beb4d576dc72e8130bd7880f600839b8/urllib3-1.22-py2.py3-none-any.whl
Requirement already satisfied (use --upgrade to upgrade): smmap2>=2.0.0 in /usr/lib/python2.7/site-packages (from gitdb2>=2.0.0->gitpython>=2.0.2->airflow)
Collecting Unidecode>=0.04.16 (from python-slugify==1.1.4->python-nvd3==0.14.2->airflow)
Using cached https://pypi.tuna.tsinghua.edu.cn/packages/01/a1/9d7f3138ee3d79a1ab865a2cb38200ca778d85121db19fe264c76c981184/Unidecode-0.04.21-py2.py3-none-any.whl
Installing collected packages: psutil, pygments, docutils, lockfile, python-daemon, Unidecode, python-slugify, python-nvd3, certifi, chardet, idna, urllib3, requests, setproctitle, tabulate, thrift, zope.deprecation, airflow
Running setup.py install for psutil ... done
Running setup.py install for python-slugify ... done
Running setup.py install for python-nvd3 ... done
Found existing installation: idna 2.4
Uninstalling idna-2.4:
Successfully uninstalled idna-2.4
Running setup.py install for setproctitle ... done
Running setup.py install for thrift ... done
Running setup.py install for airflow ... done
Successfully installed Unidecode-0.4.21 airflow-1.8.0 certifi-2017.7.27.1 chardet-3.0.4 docutils-0.14 idna-2.6 lockfile-0.12.2 psutil-4.4.2 pygments-2.2.0 python-daemon-2.1.2 python-nvd3-0.14.2 python-slugify-1.1.4 requests-2.18.4 setproctitle-1.1.10 tabulate-0.7.7 thrift-0.9.3 urllib3-1.22 zope.deprecation-4.3.0
You are using pip version 8.1.2, however version 9.0.1 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.
[root@node1 ~]#