使用线程池与进程池
Python3.2带来了concurrent.futures模块,这个模块具有线程池和进程池、管理并行编程任务、处理非确定性的执行流程、进程/线程同步等功能。
此模块由一下部分组成:
concurrent.futures.Executor:这是一个虚拟基类,提供了异步执行的方法。
submit(function,argument):调度函数(可调用的对象)的执行,将argument作为参数传入。
map(function,argument):将argument作为参数执行函数,以异步的方式。
shutdown(Wait=True):发出让执行者释放所有资源的信号。
concurrent.futures.Future:其中包括函数的异步执行。Future对象是submit任务(即带有参数的functions)到executor的实例。
Executor是抽象类,可以通过子类访问,即线程或进程的ExecutorPools。
因为,线程或进程的实例是依赖于资源的任务,所以最好以“池”的形式将他们组织在一起,作为可以重用的launcher或executor。
concurrent.Futures模块提供了两种Executor的子类,各自独立操作一个线程池和一个进程池。这两个子类分别是:
cancel():尝试去取消调用。如果调用当前正在执行,不能被取消。这个方法将返回False,否则调用将会被取消,方法将返回True
cancelled():如果调用被成功取消返回True
running():如果当前正在被执行不能被取消返回True
done():如果调用被成功取消或者完成running返回True
result(Timeout = None):拿到调用返回的结果。如果没有执行完毕就会去等待
exception(timeout=None):捕获程序执行过程中的异常
add_done_callback(fn):将fn绑定到future对象上。当future对象被取消或完成运行时,fn函数将会被调用
引用至https://www.cnblogs.com/c-x-a/p/9203313.html
from concurrent.futures import \
ThreadPoolExecutor,\
ProcessPoolExecutor
import concurrent.futures
import time
number_list=[i for i in range(1,11)]
def evaluate_item(x):
result_item=count(x)
return result_item
def count(number):
for i in range(0,10000000):
i=i+1
return i*number
if __name__ == '__main__':
start_time=time.time()
for item in number_list:
s=evaluate_item(item)
print(s)
print(time.time()-start_time)
#单线程5.319959878921509
import concurrent.futures
import time
number_list=[i for i in range(1,11)]
def evaluate_item(x):
result_item=count(x)
return result_item
def count(number):
for i in range(0,10000000):
i=i+1
return i*number
if __name__ == '__main__':
start_time=time.time()
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
futures=[executor.submit(evaluate_item,item) for item in number_list]
for future in concurrent.futures.as_completed(futures):
print(future.result())
print(time.time()-start_time)
#5.515851020812988
import concurrent.futures
import time
number_list=[i for i in range(1,11)]
def evaluate_item(x):
result_item=count(x)
return result_item
def count(number):
for i in range(0,10000000):
i=i+1
return i*number
if __name__ == '__main__':
start_time=time.time()
with concurrent.futures.ProcessPoolExecutor(max_workers=5) as executor:
futures=[executor.submit(evaluate_item,item) for item in number_list]
for future in concurrent.futures.as_completed(futures):
print(future.result())
print(time.time()-start_time)
#1.8779160976409912
import requests
from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor
from lxml import etree
import multiprocessing
def get_link(url):
print('link_html{}'.format(multiprocessing.current_process().pid))
headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
}
html=requests.get(url,headers=headers)
soup=etree.HTML(html.text)
title=soup.xpath('//*[@id="wrapper"]/h1/span/text()')
print(title)
def get_html(url):
headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
}
print('html_pid{}'.format(multiprocessing.current_process().pid))
html=requests.get(url,headers=headers)
soup=etree.HTML(html.text)
links=soup.xpath('//*[@id="subject_list"]/ul/li/div[2]/h2/a/@href')
with ThreadPoolExecutor(max_workers=3) as executor:
for link in links:
executor.submit(get_link,link)
if __name__ == '__main__':
urls=[
'https://book.douban.com/tag/%E5%B0%8F%E8%AF%B4',
'https://book.douban.com/tag/%E9%9A%8F%E7%AC%94',
'https://book.douban.com/tag/%E6%95%A3%E6%96%87'
]
with ProcessPoolExecutor(max_workers=3) as executor:
for url in urls:
executor.submit(get_html,url)
利用基本线程池使用map()
from concurrent.futures import ThreadPoolExecutor
import threading
import time
def task(n):
print('{}: sleeping {}'.format(threading.current_thread().name,n))
time.sleep(n/10)
print('{}:done with {}'.format(threading.current_thread().name,n))
return n/10
ex=ThreadPoolExecutor(max_workers=2)
print('main:starting')
results=ex.map(task,range(5,0,-1))
print('main:unprocessed results {}'.format(results))
print('main:waiting for real results')
real_results=list(results)
print('main:results:{}'.format(real_results))
'''
main:starting
ThreadPoolExecutor-0_0: sleeping 5
main:unprocessed results .result_iterator at 0x000001260BDC5750>
main:waiting for real results
ThreadPoolExecutor-0_1: sleeping 4
ThreadPoolExecutor-0_1:done with 4
ThreadPoolExecutor-0_1: sleeping 3
ThreadPoolExecutor-0_0:done with 5
ThreadPoolExecutor-0_0: sleeping 2
ThreadPoolExecutor-0_0:done with 2
ThreadPoolExecutor-0_0: sleeping 1
ThreadPoolExecutor-0_1:done with 3
ThreadPoolExecutor-0_0:done with 1
main:results:[0.5, 0.4, 0.3, 0.2, 0.1]
'''
用submit调度单个任务
from concurrent.futures import ThreadPoolExecutor
import threading
import time
def task(n):
print('{}:sleeping{}'.format(threading.current_thread().name,n))
time.sleep(n/10)
print('{} done with {}'.format(threading.current_thread().name,n))
return n/10
ex=ThreadPoolExecutor(max_workers=2)
print('main:starting')
f=ex.submit(task,5)
print('main:future{}:'.format(f))
print('main:waiting for results')
result=f.result()
print('main:result:{}'.format(result))
print('main:future after result:{}'.format(f))
'''
main:starting
ThreadPoolExecutor-0_0:sleeping5main:future:
main:waiting for results
ThreadPoolExecutor-0_0 done with 5
main:result:0.5
main:future after result:
'''
按任意顺序等待任务
from concurrent import futures
import random
import time
def task(n):
time.sleep(random.random())
return (n,n/10)
ex=futures.ThreadPoolExecutor(max_workers=2)
print('main:starting')
wait_for=[
ex.submit(task,i)
for i in range(5)
]
for f in futures.as_completed(wait_for):
print('main:result:{}'.format(f.result()))
'''
main:starting
main:result:(1, 0.1)
main:result:(0, 0.0)
main:result:(2, 0.2)
main:result:(3, 0.3)
main:result:(4, 0.4)
'''
回调
from concurrent import futures
import time
def task(n):
print('{}: sleeping'.format(n))
time.sleep(0.5)
print('{}: done'.format(n))
return n/10
def done(fn):
if fn.cancelled():
print('{}: cancelled'.format(fn.arg))
elif fn.done():
error=fn.exception()
if error:
print('{}:error returned: {}'.format(fn.arg,error))
else:
result=fn.result()
print('{}: value returned: {}'.format(fn.arg,result))
if __name__ == '__main__':
ex=futures.ThreadPoolExecutor(max_workers=2)
print('main:starting')
f=ex.submit(task,5)
f.arg=5
f.add_done_callback(done)
result=f.result()
'''
main:starting
5: sleeping
5: done
5: value returned: 0.5
'''