celery redis mysql_Flask异步任务 celery+redis

backend='redis://:[email protected]:6379/7’, # 返回值存入数据库

broker='redis://:[email protected]:6379/8') # :密码@host/post/db

Windows的Celery只支持到3.1.25

pip install celery==4.3.0

pip install eventlet

celery -A app.tasks.meta.tasks worker -l info -P eventlet

celery -A app.tasks.meta.tasks worker --loglevel=info

celery -A app.celery worker --loglevel=info

tasks.task_name.delay()

t.ready()

t.get()

t.get(timeout=11)

t.get(propagate=False)

t.traceback

CELERY_TIMEZONE='Asia/Shanghai'

CELERY_ENABLE_UTC=True

# 官网推荐消息序列化方式为json

CELERY_ACCEPT_CONTENT=['json']

CELERY_TASK_SERIALIZER='json'

CELERY_RESULT_SERIALIZER='json'

请求耗时(比如大量的数据库插入，发送验证邮件等)

利用Celery来后台处理耗时任务可以保证Flask能够较快响应而且不被阻塞，同时减轻了数据库的高峰写入压力

操作数据库，操作完成后记得释放数据库连接，例如Session.remove

Celery是专注实时处理和任务调度的分布式任务队列。

主要应用场景：

1，web应用，当需要触发事件需要较长时间处理完成，可以交给celery进行异步执行，执行完后返回结果，这段时间不用等待，提高系统的吞吐量和响应时间

2，完成任务时，需要额外的事件处理，如发送邮件等

3，后台定时任务处理，celery可以帮助我们在不同服务器进行定时任务管理

# ----config.py----

class Config(object):

"""项目配置 (基类)"""

# 根目录

BASEDIR = os.path.abspath(os.path.dirname(__file__))

# mysql

SQLALCHEMY_TRACK_MODIFICATIONS = False

SQLALCHEMY_COMMIT_ON_TEARDOWN = True # 请求结束之后进行数据自动提交

SQLALCHEMY_ECHO = False # 查看原始sql语句

# redis

REDIS_HOST = os.environ.get('REDIS_HOST') or "100.69.149.205"

REDIS_PORT = os.environ.get('REDIS_PORT') or 6379 # 6380

REDIS_PASSWORD = os.environ.get('REDIS_PASSWORD') or 'm!N0BII3k'

REDIS_NUM = os.environ.get('REDIS_NUM') or 1 # 缓存

# celery

# CELERY_URL = 'redis://:password@ip/port/db' # 工人:password@ip/port/db

CELERY_BROKER_NUM = os.environ.get('CELERY_BROKER_NUM') or 2 # 工人

CELERY_RESULT_NUM = os.environ.get('CELERY_RESULT_NUM') or 3 # 结果

CELERY_BROKER_URL = 'redis://:{}@{}:{}/{}'.format(REDIS_PASSWORD, REDIS_HOST, REDIS_PORT, CELERY_BROKER_NUM)

CELERY_RESULT_BACKEND = 'redis://:{}@{}:{}/{}'.format(REDIS_PASSWORD, REDIS_HOST, REDIS_PORT, CELERY_RESULT_NUM)

CELERY_TIMEZONE = 'Asia/Shanghai' # 时区

# ----init.py----

from celery import Celery

def make_celery(app):

"""创建celery"""

cele = Celery(app.import_name, broker=app.config['CELERY_BROKER_URL'], backend=app.config['CELERY_RESULT_BACKEND'])

cele.conf.update(app.config)

TaskBase = cele.Task

# 配置Flask上下文

class ContextTask(TaskBase):

abstract = True

def __call__(self, *args, **kwargs):

with app.app_context():

return TaskBase.__call__(self, *args, **kwargs)

cele.Task = ContextTask

return cele

# -------tasks.py---------

# !/usr/bin/env python

# -*- coding: utf-8 -*-

import os

from app.tasks.meta.utiles.inspect_fill_rate_task import inspect_fill_rate

from app.tasks.meta.utiles.inspect_same_task import inspect_same

from app.tasks.meta.utiles.meta_map_task import entrance

from app import make_celery, create_app

cele = make_celery(create_app('development'))

@cele.task(name='update_meta_map')

def update_meta_map():

"""更新元数据地图调用时间:元数据采集完、系统上下线后、删除系统"""

core_system = os.environ.get('CORE_SYSTEM') or 4

entrance(core_system)

@cele.task(name='update_inspect_same')

def update_inspect_same(system_name):

"""更新一致性检核调用时间:点击执行:param system_name::return:"""

inspect_same(system_name)

@cele.task(name='update_fill_rate')

def update_fill_rate(system_name):

"""更新填充率检核调用时间:点击执行:param system_name::return:"""

inspect_fill_rate(system_name)

if __name__ == '__main__':

print(__name__)

# 任务-：

# !/usr/bin/env python

# -*- coding: utf-8 -*-

import itertools

import json

import logging

import os

from urllib.parse import urlparse

from pymysql import connect as mysql_connect

class MetaMap(object):

def __init__(self, system_id):

self.system_id, self.links = system_id, set()

connect_params = os.environ.get(

'DATABASE_URI') or 'mysql+mysqlconnector://deployop:P@[email protected]:3306/metadata'

conn_info = urlparse(connect_params)

db_name = conn_info.path.replace("/", '')

connect_params = {"host": conn_info.hostname, "port": conn_info.port,

"user": conn_info.username, "password": conn_info.password, "database": db_name}

self.conn = mysql_connect(**connect_params)

self.cursor = self.conn.cursor()

self.sql_list = ("SELECT DISTINCT tb_code FROM tbl_info_table WHERE system_id='{}';",

"SELECT DISTINCT tgt_obj_id FROM tbl_data_rel_t WHERE src_obj_id='{}';",

"SELECT DISTINCT system_id FROM tbl_info_table WHERE tb_code='{}';",

"SELECT system_id,system_name FROM tbl_info_system WHERE system_id IN({});")

def son_recursion(self, src_obj_id, system_ids):

"""递归实现影响树 source => target"""

self.cursor.execute(self.sql_list[1].format(src_obj_id))

crs = self.cursor.fetchall()

if not crs:

return

tgt_obj_ids = set(map(lambda x: x[0], crs))

for s in tgt_obj_ids:

self.cursor.execute(self.sql_list[2].format(s))

info = self.cursor.fetchall()

sys_ids = set(map(lambda x: x[0], info))

a_link = {x for x in itertools.product(system_ids, sys_ids) if x[0] != x[1]} # 笛卡尔积

self.links.update(a_link)

self.son_recursion(s, sys_ids)

def map_tree(self):

"""对核心系统下的每个tb_code做影响分析 在让上下层的system_ids 做笛卡尔积:return:"""

try:

try:

self.cursor.execute(self.sql_list[0].format(self.system_id))

tb_codes = self.cursor.fetchall()

except Exception as e:

logging.info(e)

raise Exception('查询系统不存在')

codes = set(map(lambda x: x[0], tb_codes))

codes = filter(lambda x: x.isnumeric(), codes)

for code in codes:

sys_ids = {self.system_id}

try:

self.son_recursion(code, sys_ids)

except Exception as e:

logging.info(e)

raise Exception('递归出错了')

# 组织格式

get_ids = {i for l in self.links for i in l}

system_ids = ','.join(list(map(str, get_ids)))

self.cursor.execute(self.sql_list[3].format(system_ids))

info = self.cursor.fetchall()

nodes = list(map(lambda x: {'id': x[0], 'name': x[1]}, info))

# 降维 {()()()}

rest_ids = {node.get('id') for node in nodes}

drop_ids = get_ids - rest_ids

rest_links = set()

for item in self.links:

if set(item) & drop_ids:

continue

else:

rest_links.add(item)

links = [{"source": l[0], "target": l[1]} for l in rest_links]

meta_map = {"nodes": nodes, "links": links}

# 缓存到redis

from app import redis_store

try:

redis_store.set("meta_map", json.dumps(meta_map))

except Exception as e:

logging.info(e)

raise Exception('连接redis出错了')

except Exception as e:

logging.info(e)

logging.info('更新元数据地图失败')

self.conn.close()

self.cursor.close()

def entrance(core_system=4):

mm = MetaMap(core_system)

mm.map_tree()

logging.info('更新元数据地图成功')

if __name__ == '__main__':

entrance()