在python中使用Queue获得多进程处理结果
在 python中可以使用多进程来实现程序的并发执行，这样可以更好的利用多核处理器的优势，更快的完成计算任务。
如果需要在多进程都执行完后，对每个进程的数据处理结果进行 merge操作，则需要解决多进程间的通信问题，尤其是子进程和主进程之间的通信问题。
由于我们是要做 merge操作，所以最好能有一种多生产者单消费者的通信模式。在这里，推荐使用 multiprocessing.Queue，通过queue.put()和 queue.get()可以让我们轻松的获得多进程的处理结果，并在此基础上做 merge操作。下面看一个示例代码：
#!/usr/bin/env python
import json
from multiprocessing import Process, Queue
class FooJob:
def __init__(self, proc_idx, job_idx):
self.proc_idx = proc_idx
self.job_idx = job_idx
self.data = {}
def show_info(self):
info = "proc" + str(self.proc_idx) + "-job" + str(self.job_idx)
info += ": " + json.dumps(self.data)
print(info)
def do_job(self):
self.data[self.job_idx] = self.job_idx
self.data[-self.job_idx] = -self.job_idx
def do_jobs(jobs, q):
for job in jobs:
job.do_job()
q.put(job)
def main():
jobs_count = 100
n_process = 8
jobs_list = []
q = Queue()
for i in range(0, n_process):
jobs_list.append([])
for i in range(0, jobs_count):
jobs_list[i % n_process].append(FooJob(i % n_process, i))
process_list = []
for i in range(0, n_process):
p = Process(target=do_jobs, args=(jobs_list[i], q))
p.start()
process_list.append(p)
res_num = 0
jobs_res = []
while res_num < jobs_count:
jobs_res.append(q.get())
res_num += 1
for p in process_list:
p.join()
for job in jobs_res:
job.show_info()
if __name__ == '__main__':
main()
程序运行结果如下所示：
$ python main.py
proc1-job1: {"1": 1, "-1": -1}
proc1-job9: {"9": 9, "-9": -9}
proc1-job17: {"17": 17, "-17": -17}
proc1-job25: {"25": 25, "-25": -25}
proc1-job33: {"33": 33, "-33": -33}
proc1-job41: {"41": 41, "-41": -41}
proc1-job49: {"49": 49, "-49": -49}
proc1-job57: {"57": 57, "-57": -57}
proc1-job65: {"65": 65, "-65": -65}
proc1-job73: {"73": 73, "-73": -73}
proc1-job81: {"81": 81, "-81": -81}
proc1-job89: {"89": 89, "-89": -89}
proc1-job97: {"97": 97, "-97": -97}
proc3-job3: {"3": 3, "-3": -3}
proc3-job11: {"11": 11, "-11": -11}
proc3-job19: {"19": 19, "-19": -19}
proc3-job27: {"27": 27, "-27": -27}
proc3-job35: {"35": 35, "-35": -35}
proc3-job43: {"43": 43, "-43": -43}
proc3-job51: {"51": 51, "-51": -51}
proc3-job59: {"59": 59, "-59": -59}
proc3-job67: {"67": 67, "-67": -67}
proc3-job75: {"75": 75, "-75": -75}
proc3-job83: {"83": 83, "-83": -83}
proc3-job91: {"91": 91, "-91": -91}
proc3-job99: {"99": 99, "-99": -99}
proc2-job2: {"2": 2, "-2": -2}
proc2-job10: {"10": 10, "-10": -10}
proc2-job18: {"18": 18, "-18": -18}
proc2-job26: {"26": 26, "-26": -26}
proc2-job34: {"34": 34, "-34": -34}
proc2-job42: {"42": 42, "-42": -42}
proc2-job50: {"50": 50, "-50": -50}
proc2-job58: {"58": 58, "-58": -58}
proc2-job66: {"66": 66, "-66": -66}
proc2-job74: {"74": 74, "-74": -74}
proc2-job82: {"82": 82, "-82": -82}
proc2-job90: {"90": 90, "-90": -90}
proc2-job98: {"98": 98, "-98": -98}
proc0-job0: {"0": 0}
proc0-job8: {"8": 8, "-8": -8}
proc0-job16: {"16": 16, "-16": -16}
proc0-job24: {"24": 24, "-24": -24}
proc0-job32: {"32": 32, "-32": -32}
proc0-job40: {"40": 40, "-40": -40}
proc0-job48: {"48": 48, "-48": -48}
proc0-job56: {"56": 56, "-56": -56}
proc0-job64: {"64": 64, "-64": -64}
proc0-job72: {"72": 72, "-72": -72}
proc0-job80: {"80": 80, "-80": -80}
proc0-job88: {"88": 88, "-88": -88}
proc0-job96: {"96": 96, "-96": -96}
proc4-job4: {"4": 4, "-4": -4}
proc4-job12: {"12": 12, "-12": -12}
proc6-job6: {"6": 6, "-6": -6}
proc4-job20: {"20": 20, "-20": -20}
proc6-job14: {"14": 14, "-14": -14}
proc4-job28: {"28": 28, "-28": -28}
proc6-job22: {"22": 22, "-22": -22}
proc4-job36: {"36": 36, "-36": -36}
proc6-job30: {"30": 30, "-30": -30}
proc6-job38: {"38": 38, "-38": -38}
proc6-job46: {"46": 46, "-46": -46}
proc6-job54: {"54": 54, "-54": -54}
proc6-job62: {"62": 62, "-62": -62}
proc6-job70: {"70": 70, "-70": -70}
proc6-job78: {"78": 78, "-78": -78}
proc6-job86: {"86": 86, "-86": -86}
proc6-job94: {"94": 94, "-94": -94}
proc4-job44: {"44": 44, "-44": -44}
proc4-job52: {"52": 52, "-52": -52}
proc4-job60: {"60": 60, "-60": -60}
proc4-job68: {"68": 68, "-68": -68}
proc4-job76: {"76": 76, "-76": -76}
proc4-job84: {"84": 84, "-84": -84}
proc4-job92: {"92": 92, "-92": -92}
proc5-job5: {"5": 5, "-5": -5}
proc5-job13: {"13": 13, "-13": -13}
proc5-job21: {"21": 21, "-21": -21}
proc5-job29: {"29": 29, "-29": -29}
proc5-job37: {"37": 37, "-37": -37}
proc5-job45: {"45": 45, "-45": -45}
proc5-job53: {"53": 53, "-53": -53}
proc5-job61: {"61": 61, "-61": -61}
proc5-job69: {"69": 69, "-69": -69}
proc5-job77: {"77": 77, "-77": -77}
proc5-job85: {"85": 85, "-85": -85}
proc5-job93: {"93": 93, "-93": -93}
proc7-job7: {"7": 7, "-7": -7}
proc7-job15: {"15": 15, "-15": -15}
proc7-job23: {"23": 23, "-23": -23}
proc7-job31: {"31": 31, "-31": -31}
proc7-job39: {"39": 39, "-39": -39}
proc7-job47: {"47": 47, "-47": -47}
proc7-job55: {"55": 55, "-55": -55}
proc7-job63: {"63": 63, "-63": -63}
proc7-job71: {"71": 71, "-71": -71}
proc7-job79: {"79": 79, "-79": -79}
proc7-job87: {"87": 87, "-87": -87}
proc7-job95: {"95": 95, "-95": -95}
参考资料
https://docs.python.org/zh-cn/3/library/multiprocessing.html