python 中生产者与消费者 模式

生产者与消费者 模式

摘要:
最近一段时间,写了一些生产者与消费者模型, 特此此文 作为总结. 文中总结了不同的 生产者与消费者的情况. 一个生产者, 多个消费者,与一个生产者,多个消费者 的编程模式.

一. 生产者与消费者

在软件开发的过程中，经常碰到这样的场景：
某些模块负责生产数据，这些数据由其他模块来负责处理（此处的模块可能是：函数、线程、进程等). 产生数据的模块称为生产者,而处理数据的模块称为消费者

二. 生产者与消费者模型的优点:

• 解耦耦合
• 并发
• 闲忙不均

一.多线程实现生产者与消费者模式 实例

1. 一个生产者与一个消费者

# !/usr/bin/env python3
# -*- coding: UTF-8 -*-
"""
@author: Frank
@contact: [email protected]
@file: 18.py
@time: 2018/7/10 下午8:59

一个简单的demon   生产者与消费者

"""
import time
import queue

import threading
import random


class Producer(threading.Thread):
    """
    只负责产生数据
    """
    FINISHED = True

    def __init__(self, name, queue):
        # python3的写法
        super().__init__(name=name)
        self.queue = queue

    def run(self):
        for i in range(10):
            print("%s is producing %d to the queue!" % (self.getName(), i))
            self.queue.put(i)
            time.sleep(random.randint(1, 10) * 0.1)

        # 设置完成的标志位
        self.queue.put(self.FINISHED)
        print("%s finished!" % self.getName())


class Consumer(threading.Thread):
    """
    数据处理,
    写入到数据库里面
    """
    FINISHED = True

    def __init__(self, name, queue):
        super().__init__(name=name)
        self.queue = queue

    def run(self):
        while True:
            value = self.queue.get()
            # 用来退出线程
            if value is self.FINISHED:
                break
            print("{} is consuming. {} in the queue is consumed!".format(self.getName(), value))

        print("%s finished!" % self.getName())


if __name__ == '__main__':
    queue = queue.Queue()
    producer = Producer('producer', queue)

    consumer = Consumer('consumer', queue)

    producer.start()
    consumer.start()

    consumer.join()
    producer.join()

    print('All threads  done')

结果如下:

producer is producing 0 to the queue!
consumer is consuming. 0 in the queue is consumed!
producer is producing 1 to the queue!
consumer is consuming. 1 in the queue is consumed!
producer is producing 2 to the queue!
consumer is consuming. 2 in the queue is consumed!
producer is producing 3 to the queue!
consumer is consuming. 3 in the queue is consumed!
producer is producing 4 to the queue!
consumer is consuming. 4 in the queue is consumed!
producer is producing 5 to the queue!
consumer is consuming. 5 in the queue is consumed!
producer is producing 6 to the queue!
consumer is consuming. 6 in the queue is consumed!
producer is producing 7 to the queue!
consumer is consuming. 7 in the queue is consumed!
producer is producing 8 to the queue!
consumer is consuming. 8 in the queue is consumed!
producer is producing 9 to the queue!
consumer is consuming. 9 in the queue is consumed!
producer finished!
consumer finished!
All threads  done

Process finished with exit code 0

FINISHED 通过这个属性来通知消费者生产者已经完成生产. 相当于一个标志,告诉消费者 生产者已经生产完成.

2. 但是当有多个消费者就会出现问题.
   看下面演示代码,
   消费者就会阻塞在这里 .

#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
"""
@author: Frank 
@contact: [email protected]
@file: 18.py
@time: 2018/7/10 下午8:59

一个简单的demon   生产者与消费者

"""
import time
from datetime import datetime, timedelta
import queue

import threading
import random

# 哨兵
_sentinel = object()


class Producer(threading.Thread):
    """
    只负责产生数据
    """

    def __init__(self, name, queue):
        # python3的写法
        super().__init__(name=name)
        self.queue = queue

    def run(self):
        for i in range(5):
            print("%s is producing %d to the queue!" % (self.getName(), i))
            self.queue.put(i)
            time.sleep(random.randint(1, 10) * 0.1)

        # 设置完成的标志位
        self.queue.put(_sentinel)
        print("%s finished!" % self.getName())


class Consumer(threading.Thread):
    """
    数据处理,
    写入到数据库里面
    """

    def __init__(self, name, queue):
        super().__init__(name=name)
        self.queue = queue

    def run(self):
        while True:
            # 默认是阻塞的等待.
            value = self.queue.get()
            # 用来退出线程
            if value is _sentinel:
                break
            print("{} is consuming. {} in the queue is consumed!".format(self.getName(), value))

        print("%s finished!" % self.getName())


if __name__ == '__main__':
    queue = queue.Queue()
    producer = Producer('producer', queue)

    consumer = Consumer('consumer', queue)
    consumer2 = Consumer('consumer2', queue)

    producer.start()

    consumer.start()
    consumer2.start()

    producer.join()
    consumer.join()
    consumer2.join()

    print('All threads  done')

producer is producing 0 to the queue!
consumer is consuming. 0 in the queue is consumed!
producer is producing 1 to the queue!
consumer is consuming. 1 in the queue is consumed!
producer is producing 2 to the queue!
consumer2 is consuming. 2 in the queue is consumed!
producer is producing 3 to the queue!
consumer is consuming. 3 in the queue is consumed!
producer is producing 4 to the queue!
consumer2 is consuming. 4 in the queue is consumed!
producer finished!
consumer finished!

上面就就阻塞在第二个消费者那里了, 因为第一个消费者把 标志位拿走啦,并且退出了线程. 消费者2 发现队列空了,就在那里等待,
所以 就这样阻塞在这里.

3. 当出现一个生产者和多个消费者 的时候.

解决方法,每个线程消费完成后, 在把标志位放回队列中, 这样其他的消费者线程就可以退出来了.
来看下例子:

#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
"""
@author: Frank 
@contact: [email protected]
@file: 18.py
@time: 2018/7/10 下午8:59

一个简单的demon   生产者与消费者

"""
import time
import queue

import threading
import random

# 哨兵
_sentinel = object()


class Producer(threading.Thread):
    """
    只负责产生数据
    """

    def __init__(self, name, queue):
        # python3的写法
        super().__init__(name=name)
        self.queue = queue

    def run(self):
        for i in range(15):
            print("%s is producing %d to the queue!" % (self.getName(), i))
            self.queue.put(i)
            time.sleep(random.randint(1, 20) * 0.1)

        # 设置完成的标志位
        self.queue.put(_sentinel)
        print("%s finished!" % self.getName())


class Consumer(threading.Thread):
    """
    数据处理,对数据进行消费.
    """

    def __init__(self, name, queue):
        super().__init__(name=name)
        self.queue = queue

    def run(self):
        while True:
            value = self.queue.get()
            # 用来退出线程
            if value is _sentinel:
                # 添加哨兵,让其他线程有机会退出来
                self.queue.put(value)
                break
            print("{} is consuming. {} in the queue is consumed!".format(self.getName(), value))

        print("%s finished!" % self.getName())


if __name__ == '__main__':
    queue = queue.Queue()
    producer = Producer('producer', queue)

    producer.start()

    consumer_threads = []
    for i in range(5):
        consumer = Consumer('consumer_' + str(i), queue)
        consumer_threads.append(consumer)
        consumer.start()

    producer.join()
    for consumer in consumer_threads:
        consumer.join()

    producer.join()

    print('All threads  done')

结果如下:

producer is producing 0 to the queue!
consumer_0 is consuming. 0 in the queue is consumed!
producer is producing 1 to the queue!
consumer_0 is consuming. 1 in the queue is consumed!
producer is producing 2 to the queue!
consumer_1 is consuming. 2 in the queue is consumed!
producer is producing 3 to the queue!
consumer_2 is consuming. 3 in the queue is consumed!
producer is producing 4 to the queue!
consumer_3 is consuming. 4 in the queue is consumed!
producer is producing 5 to the queue!
consumer_4 is consuming. 5 in the queue is consumed!
producer is producing 6 to the queue!
consumer_0 is consuming. 6 in the queue is consumed!
producer is producing 7 to the queue!
consumer_1 is consuming. 7 in the queue is consumed!
producer is producing 8 to the queue!
consumer_2 is consuming. 8 in the queue is consumed!
producer is producing 9 to the queue!
consumer_3 is consuming. 9 in the queue is consumed!
producer is producing 10 to the queue!
consumer_4 is consuming. 10 in the queue is consumed!
producer is producing 11 to the queue!
consumer_0 is consuming. 11 in the queue is consumed!
producer is producing 12 to the queue!
consumer_1 is consuming. 12 in the queue is consumed!
producer is producing 13 to the queue!
consumer_2 is consuming. 13 in the queue is consumed!
producer is producing 14 to the queue!
consumer_3 is consuming. 14 in the queue is consumed!
producer finished!
consumer_4 finished!
consumer_0 finished!
consumer_1 finished!
consumer_2 finished!
consumer_3 finished!
All threads  done

Process finished with exit code 0

当出现 多个生产和 多个消费者模型

二. 多个生产者与多个消费者 模型

1. 第一种 实现方式

#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
"""
@author: Frank
@contact: [email protected]
@file: test_producer_consumer.py
@time: 2018/8/31 上午9:55

一个简单的demon   生产者与消费者模型
多个生产者 与多个消费者

2个生产者 与 多个消费者 模型.

"""

import time
import queue

import threading
import random

from collections import deque

# 哨兵
_sentinel = object()
_sentinel2 = object()


class Producer(threading.Thread):
    """
    只负责产生数据
    """

    def __init__(self, name, queue):
        # python3的写法
        super().__init__(name=name)
        self._queue = queue

    def run(self):
        for i in range(5):
            print("{} is producing {} to the queue!".format(self.getName(), i))

            self._queue.put(i)
            time.sleep(random.randint(1, 20) * 0.1)

        # 设置完成的标志位
        self._queue.put(_sentinel)
        print("%s finished!" % self.getName())


class Producer2(Producer):

    def run(self):
        for i in range(65, 70):
            item = chr(i)

            print("{} is producing {} to the queue!".format(self.getName(), item))
            self._queue.put(item)
            time.sleep(random.randint(1, 20) * 0.8)

        # 设置完成的标志位
        self._queue.put(_sentinel2)
        print("%s finished!" % self.getName())


class Consumer(threading.Thread):
    """
    数据处理
    """

    _deque = deque()

    def __init__(self, name, queue, lock):
        super().__init__(name=name)
        self._queue = queue
        self._lock = lock

    def run(self):
        while True:
            value = self._queue.get(block=True, timeout=10)
            # 用来退出线程
            if value in (_sentinel, _sentinel2):
                with self._lock:
                    if value not in Consumer._deque:
                        Consumer._deque.append(value)

                self._queue.put(value)
                if len(Consumer._deque) == 2:
                    print('Consumer._deque  ==2  break')
                    break
            else:
                print("{} is consuming. {} in the queue is consumed!".format(self.getName(), value))

        print("{} finished!".format(self.getName()))


if __name__ == '__main__':

    q = queue.Queue()

    lock = threading.Lock()

    sentienl_queue = queue.Queue()

    producer = Producer('producer111', q)
    producer2 = Producer2('producer222', q)

    producer2.start()
    producer.start()

    consumer_threads = []
    for i in range(5):
        consumer = Consumer('consumer_' + str(i), q, lock)
        consumer_threads.append(consumer)
        consumer.start()

    for consumer in consumer_threads:
        consumer.join()

    producer.join()
    producer2.join()

    print('All threads  done')

结果如下:

producer222 is producing A to the queue!
producer111 is producing 0 to the queue!
consumer_0 is consuming. A in the queue is consumed!
consumer_0 is consuming. 0 in the queue is consumed!
producer111 is producing 1 to the queue!
consumer_0 is consuming. 1 in the queue is consumed!
producer111 is producing 2 to the queue!
consumer_1 is consuming. 2 in the queue is consumed!
producer111 is producing 3 to the queue!
consumer_2 is consuming. 3 in the queue is consumed!
producer111 is producing 4 to the queue!
consumer_3 is consuming. 4 in the queue is consumed!
producer111 finished!
producer222 is producing B to the queue!
consumer_4 is consuming. B in the queue is consumed!
producer222 is producing C to the queue!
consumer_4 is consuming. C in the queue is consumed!
producer222 is producing D to the queue!
consumer_1 is consuming. D in the queue is consumed!
producer222 is producing E to the queue!
consumer_0 is consuming. E in the queue is consumed!
producer222 finished!
Consumer._deque  ==2  break
consumer_1 finished!
Consumer._deque  ==2  break
consumer_2 finished!
Consumer._deque  ==2  break
consumer_3 finished!
Consumer._deque  ==2  break
consumer_4 finished!
Consumer._deque  ==2  break
consumer_0 finished!
All threads  done

Process finished with exit code 0

2. 第二种 实现方式

#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
"""
@author: Frank
@contact: [email protected]
@file: test_producer_consumer.py
@time: 2018/8/31 上午9:55

一个简单的demon   生产者与消费者模型
多个生产者 与多个消费者


2个生产者 与 多个消费者 模型.

"""
import time
import queue

import threading
import random
from collections import deque

# 两个哨兵  作为结束标志位
_sentinel = object()
_sentinel2 = object()


class Producer(threading.Thread):
    """
    只负责产生数据
    """

    def __init__(self, name, queue):
        # python3的写法
        super().__init__(name=name)
        self._queue = queue

    def run(self):
        for i in range(5):
            print("{} is producing {} to the queue!".format(self.getName(), i))

            self._queue.put(i)
            time.sleep(0.1)

        # 设置完成的标志位
        self._queue.put(_sentinel)
        print("%s finished!" % self.getName())


class Producer2(Producer):

    def run(self):
        for i in range(65, 70):
            item = chr(i)

            print("{} is producing {} to the queue!".format(self.getName(), item))
            self._queue.put(item)
            time.sleep(random.randint(1, 20) * 0.8)

        # 设置完成的标志位
        self._queue.put(_sentinel2)
        print("%s finished!" % self.getName())


class Consumer(threading.Thread):
    """
    数据处理
    """

    _deque = deque()

    def __init__(self, name, queue):
        super().__init__(name=name)
        self._queue = queue

    @staticmethod
    def consume_data(datas):
        """
        消费数据 的方法 ,具体 怎么消费数据, 看如何实现. 比如写入文件, 写入数据库等.
        :param datas:
        :return: int
        """

        return 0

    def run(self):
        while True:
            try:
                datas = self._queue.get(block=True, timeout=10)
            except queue.Empty:
                # print(f"queue  is  empty. ")
                datas = None

            # 用来退出线程
            if datas in (_sentinel, _sentinel2):
                if datas not in Consumer._deque:
                    print(f'put {datas} into the  Consumer._deque')
                    Consumer._deque.append(datas)
                if len(Consumer._deque) == 2:
                    print('Consumer._deque length == 2 ,break, current_thread:{} has finished'.format(self.getName()))
                    break
            else:
                # 消费数据
                if datas:
                    # 消费数据
                    sucess_number = self.consume_data(datas)
                    print("{} is consuming.  datas: {} in the queue is consumed! sucess_number:{}".format(self.getName(), datas, sucess_number))
                else:
                    print('datas is  None...')
                    if len(Consumer._deque) == 2:
                        print('Consumer._deque length == 2 ,break, current_thread:{} has finished'.format(self.getName()))
                        break


if __name__ == '__main__':

    q = queue.Queue()

    producer = Producer('producer111', q)
    producer2 = Producer2('producer222', q)

    producer.start()
    producer2.start()

    consumer_threads = []
    for i in range(5):
        consumer = Consumer('consumer_' + str(i), q)
        consumer_threads.append(consumer)
        consumer.start()

    for consumer in consumer_threads:
        consumer.join()

    producer2.join()
    producer.join()

    print('All threads  done')

运行结果的一部分如下图:

producer222 is producing D to the queue!
consumer_0 is consuming.  datas: D in the queue is consumed! sucess_number:0
datas is  None...
datas is  None...
datas is  None...
datas is  None...
datas is  None...
datas is  None...
datas is  None...
datas is  None...
datas is  None...
producer222 is producing E to the queue!
consumer_0 is consuming.  datas: E in the queue is consumed! sucess_number:0
producer222 finished!
put