【笔记】linux 下python多进程启动程序

导入包

from multiprocessing import Pool, cpu_count

def split(tasks, core,list_count):
    """
    切分任务
    """
    each_part = math.ceil(list_count / core) + 1
    parts = []
    part = []
    for index, task in enumerate(tasks, 1):
        if index % each_part == 0 and index != 0:
            parts.append(part)
            part = []
        part.append((index, task))

    if len(part) > 0:
        parts.append(part)

    return parts

pool = Pool()  //获取进程池
core = cpu_count() //获取cpu数量
data_list = db.xxx.find({"status": '0'}, no_cursor_timeout=True).limit(100000) //获取任务内容列表
if data_list.count() == 0:
    //TODO
parts = split(data_list, core,data_list.count()) //将任务列表切分为和cpu相等份数
for part in parts:
    pool.apply_async(execute_tasks, (part,)) //execte_tasks 为执行任务方法的名称
logging.info('添加任务完毕')
pool.close()
pool.join()