Python学习——多线程和多进程
多线程和多进程
0. 参考：
莫烦Python (https://youtu.be/VJBSkVhHX3g)
1. 多线程
import threading as td
def job(a,b):
print("aaaaa")
t1 = td.Thread(target=job, args=(1,2)) ## 多线程
t1.start() ## start()开始工作
t1.join()
Threading
import threading as td
def main():
print(td.active_count()) ## 查看有多少个激活的线程
print(td.enumerate()) ## 查看具体是哪几个线程被激活
print(td.current_thread()) ## 查看当前所用到的线程
if __name__ == "__main__":
main()
import threading as td
def thread_job():
print("An added thread, num is %s" % td.current_thread())
def main():
added_thread = td.Thread(target=thread_job) ## 增加新的线程
added_thread.start()
if __name__ == "__main__":
main()
join
import threading as td
import time
def thread_job():
print("T1 start\n")
for i in range(10):
time.sleep(0.1)
print("T1 finish\n")
def main():
added_thread = td.Thread(target=thread_job, name="T1")
added_thread.start()
added_thread.join() ## 等待add_thread运行完成之后，再运行下面的代码
print("all done\n")
if __name__ == "__main__":
main()
import threading as td
import time
def thread_job():
print("T1 start\n")
for i in range(10):
time.sleep(0.1)
print("T1 finish\n")
def T2_job():
print("T2 start\n")
print("T2 finish\n")
def main():
added_thread = td.Thread(target=thread_job, name="T1")
thread2 = td.Thread(target=T2_job, name="T2")
added_thread.start()
thread2.start()
added_thread.join() ## 等待add_thread运行完成之后，再运行下面的代码
thread2.join()
print("all done\n")
if __name__ == "__main__":
main()
Queue
import threading as td
from queue import Queue
def job(l,q):
for i in range(len(l)):
l[i] = l[i]**2
q.put(l)
def multi_threading():
q = Queue()
threads = []
data = [[1,2,3],[3,4,5],[4,5,6],[5,6,7]]
for i in range(4):
t = td.Thread(target=job, args=(data[i],q))
t.start()
threads.append(t)
for thread in threads:
thread.join()
results = []
for _ in range(4):
results.append(q.get())
print(results)
if __name__ == "__main__":
multi_threading()
线程锁
共享内存的时候考虑锁的问题。
import threading as td
def job1():
global A, lock
lock.acquire()
for i in range(10):
A += 1
print("job1", A)
lock.release()
def job2():
global A, lock
lock.acquire()
for i in range(10):
A += 10
print("job2", A)
lock.release()
if __name__ == "__main__":
A = 0
lock = td.Lock()
t1 = td.Thread(target=job1)
t2 = td.Thread(target=job2)
t1.start()
t2.start()
t1.join()
t2.join()
2. 多进程
如果用
mp.Process(func, args)的话，它所对应的函数func是不能有return的，需要把结果存放在Queue中，然后等所有进程运行完成之后，在从Queue中将结果依次取出。
import multiprocessing as mp
def job(q):
res = 0
for i in range(1000):
res += i+i**2+i**3
q.put(res) # queue
if __name__ == '__main__':
q = mp.Queue() ## 将每个进程的结果存放到Queue中
p1 = mp.Process(target=job, args=(q,)) ## 多进程
p2 = mp.Process(target=job, args=(q,)) ## 多进程
p1.start()
p2.start()
p1.join()
p2.join()
res1 = q.get()
res2 = q.get()
print("res1: %s" % res1)
print("res2: %s" % res2)
3. 多进程和多线程的耗时对比
# coding=utf-8
import multiprocessing as mp
import threading as td
import time
def job(q):
res = 0
for i in range(1000000):
res += i+i**2+i**3
q.put(res) # queue
def normal():
res = 0
for _ in range(2):
for i in range(1000000):
res += i+i**2+i**3
print("normal: %s", res)
def multithread(): ## 多线程
q = mp.Queue() ## 将每个线程的结果存放到Queue中
p1 = td.Thread(target=job, args=(q,))
p2 = td.Thread(target=job, args=(q,))
p1.start()
p2.start()
p1.join()
p2.join()
res1 = q.get()
res2 = q.get()
print("multitherad: ", res1+res2)
def multicore(): ## 多进程
q = mp.Queue() ## 将每个进程的结果存放到Queue中
p1 = mp.Process(target=job, args=(q,)) ## 多进程
p2 = mp.Process(target=job, args=(q,)) ## 多进程
p1.start()
p2.start()
p1.join()
p2.join()
res1 = q.get()
res2 = q.get()
print("multicore: ", res1+res2)
if __name__ == '__main__':
st = time.time()
normal()
st1 = time.time()
print("normal time: ", st1 - st)
multithread()
st2 = time.time()
print("multithread time: ", st2 - st1)
multicore()
st3 = time.time()
print("multicore time: ", st3 - st2)
结果：
normal: %s 499999666667166666000000
normal time: 1.2420263290405273
multitherad: 499999666667166666000000
multithread time: 1.268148422241211
multicore: 499999666667166666000000
multicore time: 0.7706775665283203
4. 进程池
在
Pool()中用到的func是可以有返回的。
import multiprocessing as mp
def job(x):
return x*x
def multicore():
pool = mp.Pool(processes=2) ## 进程池，processes默认是全部的核，此处手动指定2个核
res = pool.map(job, range(10)) ## range(10)表示job(x)中有x为0-9
print(res）
if __name__ == "__main__":
multicore()
## 如果不想用map()，那么可以用apply_async()，但是只能一次让一个核进行预算，如果要达到map()的效果，需要用列表生成式进行处理。
res = pool.apply_async(job, (2,)) ## 除了用上面的map，还可以用apply_async()，但是只能传入1个值
print(res.get())
multi_res = [pool.apply_async(job, (i,)) for i in range(10)] ## 尽管不能直接传入多个值，但是可以用列表生成式
print([res.get() for res in multi_res])
5. 多进程共享内存
import multiprocessing as mp
value = mp.Value('d',1) ## (类型，数值)
array = mp.Array('i', [1,2,3])
6. 进程锁
import multiprocessing as mp
import time
def job(v,num,l):
l.acquire()
for _ in range(10):
time.sleep(0.1)
v.value += num
print(v.value)
l.release()
def multicore():
l = mp.Lock()
v = mp.Value('i',0)
p1 = mp.Process(target=job, args=(v,1,l))
p2 = mp.Process(target=job, args=(v,3,l))
p1.start()
p2.start()
p1.join()
p2.join()
if __name__ == "__main__":
multicore()
当程序中全局变量涉及共享内存的问题时，需要考虑线程或进程锁的问题。