Python高效编程实战---7、并发编程核心技术应用进阶

一、如何使用多线程、线程间通信、线程间事件通信

# -*- coding: utf-8 -*-

import csv
from xml.etree.ElementTree import Element, ElementTree
import requests
from StringIO import StringIO
from threading import Thread, Event
from Queue import Queue  # 安全队列，自动加锁
import os, tarfile


def pretty(e, level=0):  # 格式美化
    if len(e) > 0:  # 子元素的个数
        e.text = '\n' + '\t' * (level + 1)  # 如果有子元素的值后面添加回车，在下一行添加制表符
        for child in e:
            pretty(child, level + 1)  # 递归调用
        child.tail = child.tail[:-1]  # 下一层循环结束，进入到上一层级格式时，需要减少一人制表符
    e.tail = '\n' + '\t' * level  # 如果元没有子元素，则在子元素结束符后面添加回车，下一行 添加制表符


class DownloadThread(Thread):  # (I/O)操作
    def __init__(self, sid, queue):
        Thread.__init__(self)
        self.sid = sid
        # self.url = 'http://table.finance.yahoo.com/table.csv?s=%s.sz'  # 测试链接失效
        # self.url %= str(sid).rjust(6, '0')
        self.url = 'http://pythonscraping.com/files/MontyPythonAlbums.csv'  # 暂用此链接代替
        self.queue = queue  # 传入Queue对象，实现两个线程间的通信息的媒介

    def download(self):
        response = requests.get(self.url, timeout=3)
        if response.ok:
            return StringIO(response.content)
            # StringIO经常被用来作为字符串的缓存，应为StringIO有个好处，他的有些接口和文件操作是一致的，
            # 也就是说用同样的代码，可以同时当成文件操作或者StringIO操作

    def run(self):
        data = self.download()  # 下载数据
        self.queue.put((self.sid, data))  # 下载完数据后，将数据压入Queue对象中


class ConverThread(Thread):  # CPU密集型操作
    def __init__(self, queue, cEvent, tEvent):
        Thread.__init__(self)
        self.cEvent = cEvent
        self.tEvent = tEvent
        self.queue = queue  # 传入Queue对象，实现两个线程间的通信息的媒介

    def csvToXml(self, scsv, fxml):  # csv格式转换为xml
        reader = csv.reader(scsv)
        headers = reader.next()
        headers = map(lambda h: h.replace(' ', ''), headers)  # 首行

        root = Element('Data')
        for row in reader:
            eRow = Element('Row')
            root.append(eRow)
            for tag, text in zip(headers, row):
                e = Element(tag)
                e.text = text
                eRow.append(e)
        pretty(root)  # 美化样式
        et = ElementTree(root)  # 生成元素树
        et.write(fxml)

    def run(self):
        count = 0
        while True:
            sid, data = self.queue.get()
            # 调用队列对象的get()方法从队头删除并返回一个项目。可选参数为block，默认为True。
            # 如果队列为空且block为True，get()就使调用线程暂停，直至有项目可用。如果队列为空且block为False，队列将引发Empty异常。
            if sid == -1:
                self.tEvent.set()
                self.cEvent.wait()
                break
            if data:
                with open(str(sid).rjust(6, '0') + '.xml', 'wb') as wf:
                    self.csvToXml(data, wf)  # csv 转换成Xml
                    count += 1
            if count == 3:
                self.tEvent.set()  # 启动Tar压缩进程
                self.cEvent.wait()  # 等待Tar压缩进程cEvent.set()，重新启动进程
                self.cEvent.clear()  # 保证cEvent.wait()重新生效
                count = 0


class TarXml(Thread):
    def __init__(self, cEvent, tEvent):
        Thread.__init__(self)
        self.cEvent = cEvent
        self.tEvent = tEvent
        self.count = 1
        self.setDaemon(True)  # 设置为守护线程，其它线程结束后，自动退出

    def tarXml(self):
        tfname = str(self.count).rjust(3, '0') + '.tgz'
        with tarfile.open(tfname, 'w:gz') as tf:
            for fname in os.listdir('.'):
                if fname.endswith('.xml'):
                    tf.add(fname)
                    os.remove(fname)  # 删除文件
        if not tf.members:
            os.remove(tfname)  # 如果压缩文件为空，则删除
        self.count += 1

    def run(self):
        c=1
        while True:
            self.tEvent.wait()  # 压缩过程，等待转换进程完成一定数量后的tEvent.set()
            self.tarXml()  # 开始压缩
            self.tEvent.clear()  # 让tEvent.wait()进程重新生效
            self.cEvent.set()  # 结束cEvent.wait()
            c+=1


queue = Queue()
cEvent = Event()  # 事件对象，用来在线程间进行事件通知
tEvent = Event()

dThreads = [DownloadThread(i, queue) for i in xrange(1, 8)]  # 生成一个线程的队列
cThread = ConverThread(queue,cEvent,tEvent)  # 生成转换线程
tThread = TarXml(cEvent,tEvent)  # 生成转换线程

for t in dThreads:
    t.start()  # 启动下载线程
cThread.start()  # 启动转换线程
tThread.start()  # 启动转换线程

for t in dThreads:
    t.join()  # 阻塞函数，直到每个子线程都调用结束，才进到程序下一步
queue.put((-1, None))

二、如何使用线程本地数据

import os,cv2,time,struct,threading
from BaseHTTPServer import HTTPServer,BaseHTTPRequestHandler
from SocketServer import TCPServer,ThreadingTCPServer
from threading import Thread,RLock
from select import select

class JpegStreamer(Thread):
    def __init__(self,camera):
        Thread.__init__(self)
        self.cap = cv2.VideoCapture(camera)
        self.lock =RLock()
        self.pipes = {}

    def register(self):
        pr,pw =os.pipe()
        self.lock.acquire()
        self.pipes[pr]=pw
        pw = self.pipes.pop(pr)
        self.lock.release()
        return pr

    def unrgister(self,pr):
        self.lock.acquire()
        self.pips.pop(pr)
        self.lock.release()
        pr.close()
        pw.close()

    def capture(self):
        cap = self.cap
        while cap.isOpened():
            ret,frame = cap.read()
            if ret:
                ret,data =cv2.imencode('.jpg',frame,(cv2.IMWRITE_JPEG_QUALITY,40))
                yield data.tostring()

    def send(self,frame):
        n= struct.pack('l',len(frame))
        self.lock.acquire()
        if len(self.pipes):
            _,pipes,_ =select([],self.pipes.itervalues(),[],1)
            for pipe in pipes:
                os.write(pipe,n)
                os.write(pipe,frame)
        self.lock.release()

    def run(self):
        for frame in self.capture():
            self.send(frame)

class JpegRetriever(object):
    def __init__(self,streamer):
        self.streamer= streamer
        self.pipe =streamer.register()

    def retrieve(self):
        while True:
            ns= os.read(self.pipe,8)
            n=struct.unpack('l',ns)[0]
            data = os.read(self.pipe,n)
            yield data

    def cleanup(self):
        self.streamer.unregister(self.pipe)

class Handler(BaseHTTPRequestHandler):
    retriever=None
    @staticmethod
    def setJpegRetriever(retriever):
        Handler.retriever=retriever

    def do_GET(self):
        if self.retriever is None:
            raise RuntimeError('no retriever')
        if self.path !='/':
            return
        self.send_response(200)
        self.send_header('Content-type','multipart/x-mixed-replace;boundary=abcde')
        self.end_headers()

        for frame in self.retriever.retrieve():
            self.send_frame(frame)

        def send_frame(self,frame):
            self.wfile.write('--abcde\r\n')
            self.wfile.write('Content-Type:image/jpeg\r\n')
            self.wfile.write('Content-Length:%d\r\n\r\n'%len(frame))
            self.wfile.write(frame)


if __name__ == '__main__':
    streamer = JpegRetriever(0)
    streamer.start()
    retriever=JpegRetriever(streamer)
    Handler.setJpegRetriever(retriever)
    print 'start server ...'
    httpd = TCPServer(('',9000),Handler)
    httpd.sever_forever()

五、如何使用线程池

# 待补充

六、如何使用多进程

# 待补充