Airflow离线升级(1.10.0-＞2.2.3)文档

Airflow离线升级(1.10.0->2.2.3)文档

钉钉交流群:35701608
钉钉交流群:35701608
钉钉交流群:35701608

计划

离线部署airflow1.10.0 #目前生产环境的版本
升级至1.10.15 #官方建议的桥接版本
升级到2.2.3 

目前环境介绍

python 3.6.5
mysql 5.7
linux (10.2.5.18)单节点

1.部署Airflow1.10

安装python3.6.5

#安装python3.6.5
wget https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tgz
tar -zxvf Python-3.6.5.tgz -C /usr/local/python3
#检查
./configure prefix=/usr/local/python3/Python-3.6.5
#编译
make && make install
#创建软连接
ln -s /usr/local/python3/Python-3.6.5/bin/python3 /usr/bin/python3
ln -s /usr/local/python3/Python-3.6.5/bin/pip3 /usr/bin/pip3
#查看
python3 -V 
pip3 -V
#1.升级pip
pip3 install --upgrade pip

安装mysql5.7

#目前mysql在docker部署的
#创建airflow用户
create database airflow;
create user 'airflow'@'%' identified by 'airflow';
create user 'airflow'@'localhost' identified by 'airflow';
grant all on airflow.* to 'airflow'@'%';
flush privileges;

#进入mysql数据库
show global variables like '%timestamp%';
SET global explicit_defaults_for_timestamp =1; #参数显示on状态为设置成功
SET @@global.innodb_large_prefix = 1; 

安装airflow

#安装所需要的python包
yum -y install gcc #如果这个gcc没有报错可以不用安装
yum -y install python-devel # 或者安装  yum -y install python3-devel
pip3 install suitable
pip3 install marshmallow==2.21.0
pip3 install WTForms==2.3.3
pip3 install wtforms==2.3.1   
pip3 install SQLAlchemy==1.3.23
pip3 install mysql-devel -i 
pip3 install mysql -i 

#配置环境变量
[root@localhost airflow]# vim /etc/profile
	export SLUGIFY_USES_TEXT_UNIDECODE=yes
	#Python环境变量
	export PYTHON_HOME=/usr/local/python3/Python-3.6.5
	export PATH=$PATH:$PYTHON_HOME/bin
	#Airflow环境变量
	export AIRFLOW_HOME=/opt/airflow
	export SITE_AIRFLOW_HOME=/usr/local/python3/lib/python3.6/site-packages/airflow
	export PATH=$PATH:$SITE_AIRFLOW_HOME/bin
[root@localhost airflow]# source /etc/profile

#安装airflow 指定国内镜像 
pip3 install apache-airflow==1.10.0 -i https://pypi.tuna.tsinghua.edu.cn/simple

#执行airflow命令做初始化操作
[root@localhost airflow]# airflow

#此时会在/opt/airflow目录下生成airflow.cfg
修改 load_examples = False

启动airflow

#3001端口启动webserver
	airflow webserver -D -p 3001
#启动schduler
	airflow scheduler -D

创建用户

airflow users create \
    --username airflow \
    --password airflow \
    --firstname pengfei \
    --lastname ma \
    --role Admin \
    --email [email protected]

2.airflow1.10升级至1.15

2.1运行如下脚本升级到1.10.15桥接版本

---

#运行完即可升级到1.10.15
#! /bin/bash
AIRFLOW_VERSION=1.10.15
PYTHON_VERSION="$(python3 -V | cut -d " " -f 2 | cut -d "." -f 1-2)"
CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt"
pip3 install "apache-airflow==${AIRFLOW_VERSION}" --upgrade --constraint "${CONSTRAINT_URL}" -i https://pypi.tuna.tsinghua.edu.cn/simple

#上述脚本执行完命令为 (如果上面手动安装不成功,可以按照下面的命令手动执行)
https://raw.githubusercontent.com/apache/airflow/constraints-1.10.15/constraints-3.6.txt
pip3 install "apache-airflow==1.10.15" --upgrade --constraint "/opt/airflow/constraints-3.6.txt"
#执行完后会获得constraints-3.6.txt文件,文件内容如下:



# Editable install with no version control (apache-airflow==1.10.15)
Babel==2.9.0
Flask-Admin==1.5.4
Flask-AppBuilder==2.3.4
Flask-Babel==1.0.0
Flask-Bcrypt==0.7.1
Flask-Caching==1.3.3
Flask-JWT-Extended==3.25.0
Flask-Login==0.4.1
Flask-OpenID==1.3.0
Flask-SQLAlchemy==2.4.4
Flask-WTF==0.14.3
Flask==1.1.2
GitPython==3.1.11
JPype1==0.7.1
JayDeBeApi==1.2.3
Jinja2==2.11.2
Mako==1.1.3
Markdown==2.6.11
MarkupSafe==1.1.1
PyHive==0.6.3
PyJWT==1.7.1
PyNaCl==1.4.0
PySmbClient==0.1.5
PyYAML==5.3.1
Pygments==2.7.2
SQLAlchemy-JSONField==0.9.0
SQLAlchemy-Utils==0.36.8
SQLAlchemy==1.3.20
Sphinx==3.3.1
Unidecode==1.1.1
WTForms==2.3.3
Werkzeug==0.16.1
adal==1.2.5
aiohttp==3.7.3
alabaster==0.7.12
alembic==1.4.3
amqp==2.6.1
analytics-python==1.2.9
ansiwrap==0.8.4
apispec==1.3.3
appdirs==1.4.4
argcomplete==1.12.2
asn1crypto==1.4.0
astroid==2.4.2
async-generator==1.10
async-timeout==3.0.1
atlasclient==1.0.0
attrs==20.3.0
aws-sam-translator==1.31.0
aws-xray-sdk==2.6.0
azure-common==1.1.26
azure-core==1.9.0
azure-cosmos==3.2.0
azure-datalake-store==0.0.51
azure-identity==1.5.0
azure-keyvault-certificates==4.2.1
azure-keyvault-keys==4.3.0
azure-keyvault-secrets==4.2.0
azure-keyvault==4.1.0
azure-mgmt-containerinstance==1.5.0
azure-mgmt-core==1.2.2
azure-mgmt-datalake-nspkg==3.0.1
azure-mgmt-datalake-store==0.5.0
azure-mgmt-nspkg==3.0.2
azure-mgmt-resource==15.0.0
azure-nspkg==3.0.2
azure-storage-blob==12.6.0
azure-storage-common==2.1.0
azure-storage==0.36.0
backcall==0.2.0
bcrypt==3.2.0
beautifulsoup4==4.7.1
billiard==3.6.3.0
black==20.8b0
blinker==1.4
boto3==1.10.50
boto==2.49.0
botocore==1.13.50
cached-property==1.5.2
cachetools==4.1.1
cassandra-driver==3.20.2
cattrs==1.0.0
celery==4.4.7
certifi==2020.11.8
cffi==1.14.4
cfgv==3.2.0
cfn-lint==0.42.0
cgroupspy==0.1.6
chardet==3.0.4
click==6.7
cloudant==0.5.10
colorama==0.4.4
colorlog==4.0.2
configparser==3.5.3
coverage==5.3
croniter==0.3.36
cryptography==3.2.1
cx-Oracle==8.0.1
dataclasses==0.8
datadog==0.39.0
decorator==4.4.2
defusedxml==0.6.0
dill==0.3.3
distlib==0.3.1
dnspython==1.16.0
docker-pycreds==0.4.0
docker==3.7.3
docopt==0.6.2
docutils==0.15.2
ecdsa==0.14.1
elasticsearch-dsl==5.4.0
elasticsearch==5.5.3
email-validator==1.1.2
entrypoints==0.3
fastavro==1.2.0
filelock==3.0.12
flake8-colors==0.1.9
flake8==3.8.4
flaky==3.7.0
flask-swagger==0.2.14
flower==0.9.5
freezegun==1.0.0
fsspec==0.8.4
funcsigs==1.0.2
future-fstrings==1.2.0
future==0.18.2
gcsfs==0.7.1
gitdb==4.0.5
google-api-core==1.23.0
google-api-python-client==1.12.8
google-auth-httplib2==0.0.4
google-auth-oauthlib==0.4.2
google-auth==1.23.0
google-cloud-bigquery-storage==2.1.0
google-cloud-bigquery==2.4.0
google-cloud-bigtable==1.6.0
google-cloud-container==1.0.1
google-cloud-core==1.4.3
google-cloud-dlp==1.0.0
google-cloud-language==1.3.0
google-cloud-secret-manager==1.0.0
google-cloud-spanner==1.19.1
google-cloud-speech==1.3.2
google-cloud-storage==1.33.0
google-cloud-texttospeech==1.0.1
google-cloud-translate==1.7.0
google-cloud-videointelligence==1.16.1
google-cloud-vision==1.0.0
google-crc32c==1.0.0
google-resumable-media==1.1.0
googleapis-common-protos==1.52.0
graphviz==0.15
grpc-google-iam-v1==0.12.3
grpcio-gcp==0.2.2
grpcio==1.33.2
gunicorn==20.0.4
hdfs==2.5.8
hmsclient==0.1.1
httplib2==0.18.1
humanize==3.1.0
hvac==0.10.5
identify==1.5.10
idna-ssl==1.1.0
idna==2.8
imagesize==1.2.0
importlib-metadata==2.1.0
importlib-resources==1.5.0
inflection==0.5.1
ipdb==0.13.4
ipython-genutils==0.2.0
ipython==7.16.1
iso8601==0.1.13
isodate==0.6.0
itsdangerous==1.1.0
jedi==0.17.2
jira==2.0.0
jmespath==0.10.0
json-merge-patch==0.2
jsondiff==1.1.2
jsonpatch==1.27
jsonpickle==1.4.1
jsonpointer==2.0
jsonschema==3.2.0
junit-xml==1.9
jupyter-client==6.1.7
jupyter-core==4.7.0
kombu==4.6.11
kubernetes==11.0.0
lazy-object-proxy==1.4.3
ldap3==2.8.1
libcst==0.3.14
lockfile==0.12.2
marshmallow-enum==1.5.1
marshmallow-sqlalchemy==0.23.1
marshmallow==2.21.0
mccabe==0.6.1
mock==4.0.2
mongomock==3.21.0
more-itertools==8.6.0
moto==1.3.14
msal-extensions==0.3.0
msal==1.6.0
msrest==0.6.19
msrestazure==0.6.4
multi-key-dict==2.0.3
multidict==5.0.2
mypy-extensions==0.4.3
mypy==0.720
mysqlclient==1.3.14
natsort==7.1.0
nbclient==0.5.1
nbformat==5.0.8
nest-asyncio==1.4.3
networkx==2.5
nodeenv==1.5.0
nteract-scrapbook==0.4.1
ntlm-auth==1.5.0
numpy==1.19.4
oauthlib==3.1.0
oscrypto==1.2.1
packaging==20.7
pandas-gbq==0.14.1
pandas==1.1.4
papermill==2.2.2
parameterized==0.7.4
paramiko==2.7.2
parso==0.7.1
pathspec==0.8.1
pbr==5.5.1
pendulum==1.4.4
pep562==1.0
pexpect==4.8.0
pickleshare==0.7.5
pinotdb==0.1.1
pipdeptree==1.0.0
pluggy==0.13.1
portalocker==1.7.1
pre-commit==2.9.2
presto-python-client==0.7.0
prison==0.1.3
prometheus-client==0.8.0
prompt-toolkit==3.0.8
proto-plus==1.11.0
protobuf==3.14.0
psutil==5.7.3
psycopg2-binary==2.8.6
ptyprocess==0.6.0
py==1.9.0
pyOpenSSL==20.0.0
pyarrow==0.17.1
pyasn1-modules==0.2.8
pyasn1==0.4.8
pycodestyle==2.6.0
pycparser==2.20
pycryptodomex==3.9.9
pydata-google-auth==1.1.0
pydruid==0.5.8
pyflakes==2.2.0
pykerberos==1.2.1
pymongo==3.10.1
pymssql==2.1.5
pyparsing==2.4.7
pyrsistent==0.17.3
pysftp==0.2.9
pytest-cov==2.10.1
pytest-instafail==0.4.2
pytest-timeouts==1.2.1
pytest==5.4.3
python-daemon==2.2.4
python-dateutil==2.8.1
python-editor==1.0.4
python-http-client==3.3.1
python-jenkins==1.7.0
python-jose==3.2.0
python-nvd3==0.15.0
python-slugify==4.0.1
python3-openid==3.2.0
pytz==2020.4
pytzdata==2020.1
pywinrm==0.4.1
pyzmq==20.0.0
qds-sdk==1.16.1
redis==3.5.3
regex==2020.11.13
requests-futures==0.9.4
requests-kerberos==0.12.0
requests-mock==1.8.0
requests-ntlm==1.1.0
requests-oauthlib==1.3.0
requests-toolbelt==0.9.1
requests==2.23.0
responses==0.12.1
rsa==4.6
s3transfer==0.2.1
sasl==0.2.1
sendgrid==5.6.0
sentinels==1.0.0
sentry-sdk==0.19.4
setproctitle==1.2
six==1.15.0
slackclient==1.3.2
smmap==3.0.4
snakebite-py3==3.0.5
snowballstemmer==2.0.0
snowflake-connector-python==2.3.6
snowflake-sqlalchemy==1.2.4
soupsieve==2.0.1
sphinx-argparse==0.2.5
sphinx-autoapi==1.0.0
sphinx-copybutton==0.3.1
sphinx-jinja==1.1.1
sphinx-rtd-theme==0.5.0
sphinxcontrib-applehelp==1.0.2
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-dotnetdomain==0.4
sphinxcontrib-golangdomain==0.2.0.dev0
sphinxcontrib-htmlhelp==1.0.3
sphinxcontrib-httpdomain==1.7.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.4
sshpubkeys==3.1.0
sshtunnel==0.1.5
tabulate==0.8.7
tenacity==4.12.0
text-unidecode==1.3
textwrap3==0.9.2
thrift-sasl==0.4.2
thrift==0.13.0
toml==0.10.2
tornado==5.1.1
tqdm==4.54.0
traitlets==4.3.3
typed-ast==1.4.1
typing-extensions==3.7.4.3
typing-inspect==0.6.0
typing==3.7.4.3
tzlocal==1.5.1
unicodecsv==0.14.1
uritemplate==3.0.1
urllib3==1.25.11
vertica-python==1.0.0
vine==1.3.0
virtualenv==20.2.1
wcwidth==0.2.5
websocket-client==0.54.0
wrapt==1.12.1
xmltodict==0.12.0
yamllint==1.25.0
yarl==1.6.3
zdesk==2.7.1
zipp==3.4.0
zope.deprecation==4.4.0

2.2初始化数据库

[root@localhost airflow]# airflow db upgrade
#重启webserver服务即可安装成功
[root@localhost airflow]# airflow webserver -D -p 3001
#如果是高可用部署,启动worker需要 使用 airflow celery worker 命令

3.airflow从1.15升级到2.2.3

3.1安装升级脚本

#安装“升级检查”脚本。
pip3 install apache-airflow-upgrade-check
#运行升级检查脚本  此脚本会自动校验airflow从当前服务器升级至2.x版本需要修改的地方 `fail`的地方都需要修改
airflow upgrade_check --config=./upgrade-configuration.yaml

#upgrade-configuration.yaml 文件内容如下
ignored_rules:
  - LegacyUIDeprecated
  - ConnTypeIsNotNullableRule
  - PodTemplateFileRule

3.2升级脚本检查问题处理

# 运行过程中会打出相关的报告日志 如:
============================================== STATUS =========================================
Check for latest versions of apache-airflow and checker.................................FAIL
Remove airflow.AirflowMacroPlugin class.................................................SUCCESS
Chain between DAG and operator not allowed..............................................SUCCESS
Connection.conn_id is not unique........................................................SUCCESS
Connection.conn_type is not nullable....................................................SUCCESS
Fernet is enabled by default............................................................SUCCESS
GCP service account key deprecation.....................................................SUCCESS
Changes in import paths of hooks, operators, sensors and others.........................SUCCESS
Users must delete deprecated configs for KubernetesExecutor.............................SUCCESS
Legacy UI is deprecated by default......................................................FAIL
Logging configuration has been moved to new section.....................................SUCCESS
Removal of Mesos Executor...............................................................SUCCESS
Users must set a kubernetes.pod_template_file value......................................SKIPED
SendGrid email uses old airflow.contrib module..........................................SUCCESS
Changes in import path of remote task handlers..........................................SUCCESS
Jinja Template Variables cannot be undefined............................................SUCCESS
Found 3 problems.
 
========================================== RECOMMENDATIONS ====================================

#说明 共发现3处错误
1.Check for latest versions of apache-airflow and checker 
通过查阅源码发现升级插件自动校验版本问题,可忽略此错误

2.Legacy UI is deprecated by default.
airflow 2.x默认开启RBAC角色权限控制,所以需要删除 airflow.cfg中 rbac=False 此项配置或者在运行升级插件的时候
在yaml文件中进行配置忽略

3.Users must set a kubernetes.pod_template_file value
不使用k8s部署,可以忽略此项配置,使用k8s 官网有配置说明

3.3运行升级脚本

#1.修改完以上升级插件所抛出来的错误后,(不断执行airflow upgrade_check --config=./upgrade-configuration.yaml,直到不报错为止)

#2.运行升级脚本  airflow1.10.15升级至2.2.3版本

#! /bin/bash
AIRFLOW_VERSION=2.2.3
PYTHON_VERSION="$(python3 -V | cut -d " " -f 2 | cut -d "." -f 1-2)"
CONSTRAINT_URL="${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt>"
pip3 install "apache-airflow==${AIRFLOW_VERSION}" --upgrade --constraint "${CONSTRAINT_URL}"

#以上脚本运行内容为
https://raw.githubusercontent.com/apache/airflow/constraints-2.2.3/constraints-3.6.txt
pip3 install "apache-airflow==2.2.3" --upgrade --constraint "/opt/airflow/constraints-3.6.txt"


#constraints-3.6.txt内容为: (内网环境需要把此文件拷贝至内网服务器)

#
# This constraints file was automatically generated on 2021-12-13T11:33:11Z
# via "eager-upgrade" mechanism of PIP. For the "v2-2-test" branch of Airflow.
# This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs
# the providers from PIP-released packages at the moment of the constraint generation.
#
# Those constraints are actually those that that regular users use to install released version of Airflow.
# We also use those constraints after "apache-airflow" is released and the constraints are tagged with
# "constraints-X.Y.Z" tag to build the production image for that version.
#
# Editable install with no version control (apache-airflow==2.2.3)
APScheduler==3.6.3
Authlib==0.15.5
Babel==2.9.1
Deprecated==1.2.13
Flask-AppBuilder==3.4.1
Flask-Babel==2.0.0
Flask-Bcrypt==0.7.1
Flask-Caching==1.10.1
Flask-JWT-Extended==3.25.1
Flask-Login==0.4.1
Flask-OpenID==1.3.0
Flask-SQLAlchemy==2.5.1
Flask-WTF==0.14.3
Flask==1.1.4
GitPython==3.1.18
HeapDict==1.0.1
JPype1==1.3.0
JayDeBeApi==1.2.3
Jinja2==2.11.3
Mako==1.1.6
Markdown==3.3.6
MarkupSafe==2.0.1
PyGithub==1.54.1
PyHive==0.6.4
PyJWT==1.7.1
PyNaCl==1.4.0
PyYAML==5.4.1
Pygments==2.10.0
Rx==3.2.0
SQLAlchemy-JSONField==1.0.0
SQLAlchemy-Utils==0.37.9
SQLAlchemy==1.3.24
SecretStorage==3.3.1
Sphinx==3.4.3
Unidecode==1.3.2
WTForms==2.3.3
Werkzeug==1.0.1
adal==1.2.7
aiohttp==3.8.1
aiosignal==1.2.0
alabaster==0.7.12
alembic==1.7.5
aliyun-python-sdk-core==2.13.35
aliyun-python-sdk-kms==2.15.0
amqp==5.0.7
analytics-python==1.4.0
ansiwrap==0.8.4
anyio==3.4.0
apache-airflow-providers-airbyte==2.1.1
apache-airflow-providers-alibaba==1.0.0
apache-airflow-providers-amazon==2.4.0
apache-airflow-providers-apache-beam==3.1.0
apache-airflow-providers-apache-cassandra==2.1.0
apache-airflow-providers-apache-drill==1.0.1
apache-airflow-providers-apache-druid==2.1.0
apache-airflow-providers-apache-hdfs==2.2.0
apache-airflow-providers-apache-hive==2.1.0
apache-airflow-providers-apache-kylin==2.0.1
apache-airflow-providers-apache-livy==2.1.0
apache-airflow-providers-apache-pig==2.0.1
apache-airflow-providers-apache-pinot==2.0.1
apache-airflow-providers-apache-spark==2.0.2
apache-airflow-providers-apache-sqoop==2.0.2
apache-airflow-providers-asana==1.1.0
apache-airflow-providers-celery==2.1.0
apache-airflow-providers-cloudant==2.0.1
apache-airflow-providers-cncf-kubernetes==2.2.0
apache-airflow-providers-databricks==2.0.2
apache-airflow-providers-datadog==2.0.1
apache-airflow-providers-dingding==2.0.1
apache-airflow-providers-discord==2.0.1
apache-airflow-providers-docker==2.3.0
apache-airflow-providers-elasticsearch==2.1.0
apache-airflow-providers-exasol==2.0.1
apache-airflow-providers-facebook==2.1.0
apache-airflow-providers-ftp==2.0.1
apache-airflow-providers-google==6.2.0
apache-airflow-providers-grpc==2.0.1
apache-airflow-providers-hashicorp==2.1.1
apache-airflow-providers-http==2.0.1
apache-airflow-providers-imap==2.0.1
apache-airflow-providers-influxdb==1.1.0
apache-airflow-providers-jdbc==2.0.1
apache-airflow-providers-jenkins==2.0.3
apache-airflow-providers-jira==2.0.1
apache-airflow-providers-microsoft-azure==3.4.0
apache-airflow-providers-microsoft-mssql==2.0.1
apache-airflow-providers-microsoft-psrp==1.0.1
apache-airflow-providers-microsoft-winrm==2.0.1
apache-airflow-providers-mongo==2.2.0
apache-airflow-providers-mysql==2.1.1
apache-airflow-providers-neo4j==2.0.2
apache-airflow-providers-odbc==2.0.1
apache-airflow-providers-openfaas==2.0.0
apache-airflow-providers-opsgenie==2.0.1
apache-airflow-providers-oracle==2.0.1
apache-airflow-providers-pagerduty==2.1.0
apache-airflow-providers-papermill==2.1.0
apache-airflow-providers-plexus==2.0.1
apache-airflow-providers-postgres==2.4.0
apache-airflow-providers-presto==2.0.1
apache-airflow-providers-qubole==2.0.1
apache-airflow-providers-redis==2.0.1
apache-airflow-providers-salesforce==3.4.0
apache-airflow-providers-samba==3.0.1
apache-airflow-providers-segment==2.0.1
apache-airflow-providers-sendgrid==2.0.1
apache-airflow-providers-sftp==2.3.0
apache-airflow-providers-singularity==2.0.1
apache-airflow-providers-slack==4.1.0
apache-airflow-providers-snowflake==2.3.1
apache-airflow-providers-sqlite==2.0.1
apache-airflow-providers-ssh==2.3.0
apache-airflow-providers-tableau==2.1.2
apache-airflow-providers-telegram==2.0.1
apache-airflow-providers-trino==2.0.2
apache-airflow-providers-vertica==2.0.1
apache-airflow-providers-yandex==2.1.0
apache-airflow-providers-zendesk==2.0.1
apache-beam==2.34.0
apispec==3.3.2
appdirs==1.4.4
argcomplete==1.12.3
arrow==1.2.1
asana==0.10.3
asn1crypto==1.4.0
astroid==2.9.0
async-generator==1.10
async-timeout==4.0.1
asynctest==0.13.0
atlasclient==1.0.0
attrs==20.3.0
avro-python3==1.9.2.1
aws-xray-sdk==2.9.0
azure-batch==11.0.0
azure-common==1.1.27
azure-core==1.21.1
azure-cosmos==4.2.0
azure-datalake-store==0.0.52
azure-identity==1.7.1
azure-keyvault-certificates==4.3.0
azure-keyvault-keys==4.4.0
azure-keyvault-secrets==4.3.0
azure-keyvault==4.1.0
azure-kusto-data==0.0.45
azure-mgmt-containerinstance==1.5.0
azure-mgmt-core==1.3.0
azure-mgmt-datafactory==1.1.0
azure-mgmt-datalake-nspkg==3.0.1
azure-mgmt-datalake-store==0.5.0
azure-mgmt-nspkg==3.0.2
azure-mgmt-resource==20.0.0
azure-nspkg==3.0.2
azure-storage-blob==12.8.1
azure-storage-common==2.1.0
azure-storage-file==2.1.0
backcall==0.2.0
backoff==1.10.0
backports.entry-points-selectable==1.1.1
backports.zoneinfo==0.2.1
bcrypt==3.2.0
beautifulsoup4==4.10.0
billiard==3.6.4.0
black==21.12b0
blinker==1.4
boto3==1.18.65
boto==2.49.0
botocore==1.21.65
bowler==0.9.0
cached-property==1.5.2
cachetools==4.2.2
cassandra-driver==3.25.0
cattrs==1.0.0
celery==5.1.2
certifi==2020.12.5
cffi==1.15.0
cfgv==3.3.1
cgroupspy==0.2.1
charset-normalizer==2.0.9
click-didyoumean==0.3.0
click-plugins==1.1.1
click-repl==0.2.0
click==7.1.2
clickclick==20.10.2
cloudant==2.15.0
cloudpickle==1.4.1
colorama==0.4.4
colorlog==4.8.0
commonmark==0.9.1
contextvars==2.4
coverage==6.2
crcmod==1.7
croniter==1.0.15
cryptography==3.4.8
curlify==2.2.1
cx-Oracle==8.3.0
dask==2021.3.0
dataclasses==0.8
datadog==0.43.0
decorator==5.1.0
defusedxml==0.7.1
dill==0.3.1.1
distlib==0.3.4
distributed==2.19.0
dnspython==2.1.0
docker==5.0.3
docopt==0.6.2
docutils==0.16
ecdsa==0.17.0
elasticsearch-dbapi==0.2.7
elasticsearch-dsl==7.4.0
elasticsearch==7.13.4
email-validator==1.1.3
entrypoints==0.3
eventlet==0.33.0
execnet==1.9.0
facebook-business==12.0.1
fastavro==1.4.7
filelock==3.4.0
fissix==21.11.13
flake8-colors==0.1.9
flake8==3.9.2
flaky==3.7.0
flower==1.0.0
freezegun==1.1.0
frozenlist==1.2.0
fsspec==2021.11.1
future==0.18.2
gcsfs==2021.11.1
geomet==0.2.1.post1
gevent==21.12.0
gitdb==4.0.9
github3.py==3.0.0
google-ads==12.0.0
google-api-core==1.31.4
google-api-python-client==1.12.8
google-auth-httplib2==0.1.0
google-auth-oauthlib==0.4.6
google-auth==1.35.0
google-cloud-appengine-logging==1.1.0
google-cloud-audit-log==0.2.0
google-cloud-automl==2.5.2
google-cloud-bigquery-datatransfer==3.4.1
google-cloud-bigquery-storage==2.10.1
google-cloud-bigquery==2.31.0
google-cloud-bigtable==1.7.0
google-cloud-build==3.7.1
google-cloud-container==1.0.1
google-cloud-core==1.7.2
google-cloud-datacatalog==3.6.1
google-cloud-dataproc-metastore==1.3.1
google-cloud-dataproc==3.1.1
google-cloud-dlp==1.0.0
google-cloud-kms==2.10.1
google-cloud-language==1.3.0
google-cloud-logging==2.7.0
google-cloud-memcache==1.0.0
google-cloud-monitoring==2.8.0
google-cloud-os-login==2.5.1
google-cloud-pubsub==2.9.0
google-cloud-redis==2.5.0
google-cloud-secret-manager==1.0.0
google-cloud-spanner==1.19.1
google-cloud-speech==1.3.2
google-cloud-storage==1.43.0
google-cloud-tasks==2.7.1
google-cloud-texttospeech==1.0.1
google-cloud-translate==1.7.0
google-cloud-videointelligence==1.16.1
google-cloud-vision==1.0.0
google-cloud-workflows==1.5.0
google-crc32c==1.3.0
google-resumable-media==2.1.0
googleapis-common-protos==1.54.0
graphviz==0.19.1
greenlet==1.1.2
grpc-google-iam-v1==0.12.3
grpcio-gcp==0.2.2
grpcio==1.42.0
gssapi==1.7.2
gunicorn==20.1.0
h11==0.12.0
hdfs==2.6.0
hmsclient==0.1.1
httpcore==0.13.7
httplib2==0.19.1
httpx==0.19.0
humanize==3.13.1
hvac==0.11.2
identify==2.4.0
idna-ssl==1.1.0
idna==3.3
ijson==3.1.4
imagesize==1.3.0
immutables==0.16
importlib-metadata==4.8.2
importlib-resources==5.4.0
inflection==0.5.1
influxdb-client==1.24.0
iniconfig==1.1.1
ipdb==0.13.9
ipython-genutils==0.2.0
ipython==7.16.2
iso8601==1.0.2
isodate==0.6.0
itsdangerous==1.1.0
jedi==0.17.2
jeepney==0.7.1
jira==3.1.1
jmespath==0.10.0
json-merge-patch==0.2
jsondiff==1.3.0
jsonpath-ng==1.5.3
jsonschema==3.2.0
jupyter-client==7.1.0
jupyter-core==4.9.1
jwcrypto==1.0
keyring==23.4.0
kombu==5.1.0
krb5==0.2.0
kubernetes==11.0.0
kylinpy==2.8.4
lazy-object-proxy==1.4.3
ldap3==2.9.1
libcst==0.3.23
lockfile==0.12.2
lxml==4.7.0
marshmallow-enum==1.5.1
marshmallow-oneofschema==3.0.1
marshmallow-sqlalchemy==0.26.1
marshmallow==3.14.1
mccabe==0.6.1
mongomock==3.23.0
monotonic==1.6
moreorless==0.4.0
moto==2.2.18
msal-extensions==0.3.1
msal==1.16.0
msgpack==1.0.3
msrest==0.6.21
msrestazure==0.6.4
multi-key-dict==2.0.3
multidict==5.2.0
mypy-extensions==0.4.3
mypy==0.770
mysql-connector-python==8.0.27
mysqlclient==2.1.0
nbclient==0.5.9
nbformat==5.1.3
neo4j==4.4.0
nest-asyncio==1.5.4
nodeenv==1.6.0
nox==2020.12.31
ntlm-auth==1.5.0
numpy==1.19.5
oauth2client==4.1.3
oauthlib==3.1.1
openapi-schema-validator==0.1.5
openapi-spec-validator==0.3.1
orjson==3.6.1
oscrypto==1.2.1
oss2==2.15.0
packaging==21.3
pandas-gbq==0.14.1
pandas==1.1.5
papermill==2.3.3
parameterized==0.8.1
paramiko==2.8.1
parso==0.7.1
pathspec==0.9.0
pbr==5.8.0
pdpyras==4.3.0
pendulum==2.1.2
pep562==1.1
pexpect==4.8.0
pickleshare==0.7.5
pinotdb==0.3.8
pipdeptree==2.2.0
platformdirs==2.4.0
pluggy==1.0.0
ply==3.11
plyvel==1.3.0
portalocker==2.3.2
pre-commit==2.16.0
presto-python-client==0.7.0
prison==0.2.1
prometheus-client==0.12.0
prompt-toolkit==3.0.24
proto-plus==1.19.8
protobuf==3.19.1
psutil==5.8.0
psycopg2-binary==2.9.2
ptyprocess==0.7.0
pure-sasl==0.6.2
py4j==0.10.9.2
py==1.11.0
pyOpenSSL==20.0.1
pyarrow==5.0.0
pyasn1-modules==0.2.8
pyasn1==0.4.8
pycodestyle==2.7.0
pycountry==20.7.3
pycparser==2.21
pycryptodome==3.12.0
pycryptodomex==3.12.0
pydata-google-auth==1.3.0
pydot==1.4.2
pydruid==0.6.2
pyenchant==3.2.2
pyexasol==0.23.3
pyflakes==2.3.1
pykerberos==1.2.1
pymongo==3.12.3
pymssql==2.2.2
pyodbc==4.0.32
pyparsing==2.4.7
pypsrp==0.7.0
pyrsistent==0.18.0
pysftp==0.2.9
pyspark==3.2.0
pyspnego==0.3.1
pytest-asyncio==0.16.0
pytest-cov==3.0.0
pytest-forked==1.4.0
pytest-httpx==0.13.0
pytest-instafail==0.4.2
pytest-rerunfailures==9.1.1
pytest-timeouts==1.2.1
pytest-xdist==2.5.0
pytest==6.2.5
python-daemon==2.3.0
python-dateutil==2.8.2
python-http-client==3.3.4
python-jenkins==1.7.0
python-jose==3.3.0
python-ldap==3.4.0
python-nvd3==0.15.0
python-slugify==4.0.1
python-telegram-bot==13.9
python3-openid==3.2.0
pytz-deprecation-shim==0.1.0.post0
pytz==2021.3
pytzdata==2020.1
pywinrm==0.4.2
pyzmq==22.3.0
qds-sdk==1.16.1
redis==3.5.3
redshift-connector==2.0.901
requests-kerberos==0.14.0
requests-mock==1.9.3
requests-ntlm==1.1.0
requests-oauthlib==1.3.0
requests-toolbelt==0.9.1
requests==2.26.0
responses==0.16.0
rfc3986==1.5.0
rich==10.16.0
rsa==4.8
s3transfer==0.5.0
sasl==0.3.1
scramp==1.4.1
scrapbook==0.5.0
semver==2.13.0
sendgrid==6.9.2
sentinels==1.0.0
sentry-sdk==1.5.0
setproctitle==1.2.2
simple-salesforce==1.11.4
six==1.16.0
slack-sdk==3.13.0
smbprotocol==1.8.3
smmap==5.0.0
snakebite-py3==3.0.5
sniffio==1.2.0
snowballstemmer==2.2.0
snowflake-connector-python==2.7.1
snowflake-sqlalchemy==1.2.4
sortedcontainers==2.4.0
soupsieve==2.3.1
sphinx-airflow-theme==0.0.6
sphinx-argparse==0.3.1
sphinx-autoapi==1.0.0
sphinx-copybutton==0.4.0
sphinx-jinja==1.1.1
sphinx-rtd-theme==1.0.0
sphinxcontrib-applehelp==1.0.2
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-dotnetdomain==0.4
sphinxcontrib-golangdomain==0.2.0.dev0
sphinxcontrib-htmlhelp==2.0.0
sphinxcontrib-httpdomain==1.8.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-redoc==1.6.0
sphinxcontrib-serializinghtml==1.1.5
sphinxcontrib-spelling==7.2.1
spython==0.1.17
sqlalchemy-drill==1.1.1
sqlalchemy-redshift==0.8.8
sqlparse==0.4.2
sshtunnel==0.4.0
starkbank-ecdsa==2.0.3
statsd==3.3.0
swagger-ui-bundle==0.0.9
tableauserverclient==0.17.0
tabulate==0.8.9
tblib==1.7.0
tenacity==8.0.1
termcolor==1.1.0
text-unidecode==1.3
textwrap3==0.9.2
thrift-sasl==0.4.3
thrift==0.15.0
toml==0.10.2
tomli==1.2.2
toolz==0.11.2
tornado==6.1
tqdm==4.62.3
traitlets==4.3.3
trino==0.309.0
typed-ast==1.5.1
typing-extensions==3.10.0.2
typing-inspect==0.7.1
typing==3.7.4.3
tzdata==2021.5
tzlocal==4.1
unicodecsv==0.14.1
uritemplate==3.0.1
urllib3==1.26.7
vertica-python==1.0.2
vine==5.0.0
virtualenv==20.10.0
volatile==2.1.0
watchtower==1.0.6
wcwidth==0.2.5
websocket-client==1.2.3
wrapt==1.13.3
xmltodict==0.12.0
yamllint==1.26.3
yandexcloud==0.122.0
yarl==1.7.2
zdesk==2.7.1
zict==2.0.0
zipp==3.6.0
zope.event==4.5.0
zope.interface==5.4.0

3.4重启airflow服务

#脚本执行完成后 运行如下命令即可升级至2.2.3
[root@localhost airflow]# airflow db upgrade
#重启webserver服务即可安装成功
[root@localhost airflow]# airflow webserver -D -p 3001

4.添加用户,访问airflow

#账号密码都为airflow
airflow users create \
    --username airflow \
    --password airflow \
    --firstname pengfei \
    --lastname ma \
    --role Admin \
    --email [email protected]

问题汇总

1.Airflow is used by many users

#报错
Airflow is used by many users, and it is very likely that others had similar problems and you can easily find a solution to your problem.
#执行
[root@localhost airflow]# airflow db upgrade

2.Specified key was too long; max key length is 1000 bytes

SET @@global.innodb_large_prefix = 1; 

3.Global variable explicit_defaults_for_timestamp needs to be on (1) for mysql

3.Exception: Global variable explicit_defaults_for_timestamp needs to be on (1) for mysql

SET global explicit_defaults_for_timestamp =1;

4.airflow: error: the following arguments are required: subcommand

不用理会,官方bug,只要生成airflow.cfg文件即可, 文件生成目录为环境变量配置的airflow安装目录

**5.WebUi: Do not use SequentialExecutor in production **

#官方执行器介绍: https://www.astronomer.io/guides/airflow-executors-explained
#修改airflow.cfg
executor =LocalExecutor

6.connection选项没有mysql

#Connection Type missing? Make sure you're installed the corresponding Airflow Provider Package
pip3 install apache-airflow[mysql]