分布式进程--错误解决

廖雪峰Python教程–分布式进程

PicklingError：不能pickle <函数在0x02747DB0>：没有找到main 。

解决链接

task_master.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import random, time, queue
from multiprocessing import freeze_support
from multiprocessing.managers import BaseManager

# 发送任务的队列:
task_queue = queue.Queue()
# 接收结果的队列:
result_queue = queue.Queue()

# 从BaseManager继承的QueueManager:
class QueueManager(BaseManager):
    pass

def return_task_queue():
    global task_queue
    return task_queue

def return_result_queue():
    global result_queue
    return result_queue

def test():
    # 把两个Queue都注册到网络上, callable参数关联了Queue对象:
    # QueueManager.register('get_task_queue', callable=lambda: task_queue)
    # QueueManager.register('get_result_queue', callable=lambda: result_queue)
    QueueManager.register('get_task_queue', callable=return_task_queue)
    QueueManager.register('get_result_queue', callable=return_result_queue)

    # 绑定端口5000, 设置验证码'abc':
    manager = QueueManager(address=('127.0.0.1', 5000), authkey=b'abc')
    # 启动Queue:
    manager.start()
    # 获得通过网络访问的Queue对象:
    task = manager.get_task_queue()
    result = manager.get_result_queue()
    # 放几个任务进去:
    for i in range(10):
        n = random.randint(0, 10000)
        print('Put task %d...' % n)
        task.put(n)
    # 从result队列读取结果:
    print('Try get results...')
    for i in range(10):
        r = result.get(timeout=10)
        print('Result: %s' % r)
    # 关闭:
    manager.shutdown()
    print('master exit.')

if __name__ == '__main__':
    freeze_support()
    test()

task_worker.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import time, sys, queue
from multiprocessing.managers import BaseManager

# 创建类似的QueueManager:
class QueueManager(BaseManager):
    pass

# 由于这个QueueManager只从网络上获取Queue，所以注册时只提供名字:
QueueManager.register('get_task_queue')
QueueManager.register('get_result_queue')

# 连接到服务器，也就是运行task_master.py的机器:
server_addr = '127.0.0.1'
print('Connect to server %s...' % server_addr)
# 端口和验证码注意保持与task_master.py设置的完全一致:
m = QueueManager(address=(server_addr, 5000), authkey=b'abc')
# 从网络连接:
m.connect()
# 获取Queue的对象:
task = m.get_task_queue()
result = m.get_result_queue()
# 从task队列取任务,并把结果写入result队列:
for i in range(10):
    try:
        n = task.get(timeout=1)
        print('run task %d * %d...' % (n, n))
        r = '%d * %d = %d' % (n, n, n*n)
        time.sleep(1)
        result.put(r)
    except Queue.Empty:
        print('task queue is empty.')
# 处理结束:
print('worker exit.')

同时打开两个shell。
先运行task_master.py
然后运行task_worker.py。
（间隔时间长会出现Empty Queue）

C:\Users\K\Desktop\x>python task_master.py
Put task 8323...
Put task 2381...
Put task 3692...
Put task 5614...
Put task 8008...
Put task 3851...
Put task 8920...
Put task 4314...
Put task 5676...
Put task 4413...
Try get results...
Result: 8323 * 8323 = 69272329
Result: 2381 * 2381 = 5669161
Result: 3692 * 3692 = 13630864
Result: 5614 * 5614 = 31516996
Result: 8008 * 8008 = 64128064
Result: 3851 * 3851 = 14830201
Result: 8920 * 8920 = 79566400
Result: 4314 * 4314 = 18610596
Result: 5676 * 5676 = 32216976
Result: 4413 * 4413 = 19474569
master exit.

C:\Users\K\Desktop\x>

C:\Users\K\Desktop\x>python task_worker.py
Connect to server 127.0.0.1...
run task 8323 * 8323...
run task 2381 * 2381...
run task 3692 * 3692...
run task 5614 * 5614...
run task 8008 * 8008...
run task 3851 * 3851...
run task 8920 * 8920...
run task 4314 * 4314...
run task 5676 * 5676...
run task 4413 * 4413...
worker exit.

C:\Users\K\Desktop\x>