python 多线程，多进程的快速实现 concurrent, joblib, multiprocessing, threading

python 多线程，多进程的快速实现 concurrent, joblib, multiprocessing, threading

Python 界有条不成文的准则： 计算密集型任务适合多进程，IO 密集型任务适合多线程。

通常来说多线程相对于多进程有优势，因为创建一个进程开销比较大，然而因为在 python 中有 GIL 这把大锁的存在，导致执行计算密集型任务时多线程实际只能是单线程。而且由于线程之间切换的开销导致多线程往往比实际的单线程还要慢，所以在 python 中计算密集型任务通常使用多进程，因为各个进程有各自独立的 GIL，互不干扰。

而在 IO 密集型任务中，CPU 时常处于等待状态，操作系统需要频繁与外界环境进行交互，如读写文件，在网络间通信等。在这期间 GIL 会被释放，因而就可以使用真正的多线程。

本文主要介绍一下如何使用python 快速实现多进程及多线程：

多进程

multiprocessing

from multiprocessing import Pool
def asy(sub_f):
    with Pool(processes=6) as p:
        result = []
        for j in range(6):
            a = p.apply_async(sub_f, args=(j,))
            result.append(a)
        res = [j.get() for j in result]
        
def mp(sub_f):
    with Pool(processes=6) as p:
        res = p.map(sub_f, list(range(6)))
    return
  

joblib

from joblib import Parallel, delayed, parallel_backend

def joblib_process(sub_f):
    with parallel_backend("multiprocessing", n_jobs=6):
        res = Parallel()(delayed(sub_f)(j) for j in range(6))
    return

concurrent

from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
def process_pool(sub_f):
    with ProcessPoolExecutor(max_workers=6) as executor:
        res = executor.map(sub_f, list(range(6)))

实现

def showtime(f, sub_f, name):
    start_time = time.time()
    f(sub_f)
    print("{} time: {:.4f}s".format(name, time.time() - start_time))

def main(sub_f):
    showtime(normal, sub_f, "normal")
    print()
    print("------ 多进程 ------")
    showtime(joblib_process, sub_f, "joblib multiprocess")
    showtime(mp, sub_f, "pool")
    showtime(asy, sub_f, "async")
    showtime(process_pool, sub_f, "process_pool")
    print()

多线程

joblib

def joblib_thread(sub_f):
    with parallel_backend('threading', n_jobs=6):
        res = Parallel()(delayed(sub_f)(j) for j in range(6))
    return

thread

def thread(sub_f):
    threads = []
    for j in range(6):
        t = Thread(target=sub_f, args=(j,))
        threads.append(t)
        t.start()
    for t in threads:
        t.join()

Thread pool

from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor

def thread_pool(sub_f):
    with ThreadPoolExecutor(max_workers=6) as executor:
        res = [executor.submit(sub_f, j) for j in range(6)]

实现

print("----- 多线程 -----")
    showtime(joblib_thread, sub_f, "joblib thread")
    showtime(thread, sub_f, "thread")
    showtime(thread_pool, sub_f, "thread_pool")