[ python 线程之间通信 ]

2019独角兽企业重金招聘Python工程师标准>>>

'''
Created on 2018年10月4日

@author: 徐良俊
'''
import requests,logging,csv,time
from functools import wraps
from threading import Thread
from _elementtree import Element
from xml.etree.ElementTree import ElementTree

from queue import Queue



class DownloadThread(Thread):
    '''下载线程
    IO密集性操作
    '''
    def __init__(self,sid,queue):
        Thread.__init__(self)
        self.sid=sid
        self.url="http://hq.sinajs.cn/list=%s" % sid
        self.queue = queue
        

    def download(self,url):
        response=requests.get(url,timeout=3000)
        response.encoding="GBK"
        if response.ok:
            content=response.text
            maincontent_list = content[content.find("\"")+1:content.rfind("\"")].split(",")
#             info="股票名称:%s;今日开盘价:%s;昨日收盘价:%s;日期：%s" % (maincontent_list[0],maincontent_list[1],maincontent_list[2],maincontent_list[30]+" "+maincontent_list[31],)
#             logging.warn(info)
        return maincontent_list
    
    
    def run(self):
        data=self.download(self.url)
        
        print("put %s" % self.sid)    
        self.queue.put((self.sid,data))
        
        
        
        
        
        
class ConvertThread(Thread):
    '''打包线程 
        cpu 密集性操作
    '''
    def __init__(self,queue):
        Thread.__init__(self)     
        self.queue=queue   
        
    def dataToCsv(self,writer):
        
        while True:
            print("get queue")
            csvdata = []
            sid,data=self.queue.get()
            if sid == -1:
                break
            if data and data[0] != '' :
                csvdata.append((data[0],data[1],data[2],data[30],))
                writer.writerows(csvdata)    
        
            
            
    def csvToXml(self,scsv,fxml):
        reader=csv.reader(scsv)
        headers=reader.next()
        headers=map(lambda h:h.replace(' ',''),headers)
        root = Element('Data')
        for row in reader:
            eRow = Element('Row')
            root.append(eRow)
        for tag,text in zip(headers,row):
            e=Element(tag)
            e.text=text
            eRow.append(e)
            
        et=ElementTree(root)
        et.write(fxml)
        
        
    def run(self):
#         fname=str(self.sid).rjust(6,'0')+'.xml'
#         with open(fname,'wb') as wf:
#             csvToXml(data,wf)

        csvFile = open( "fundnav.csv" , "w",encoding='utf-8-sig')
        writer = csv.writer(csvFile,lineterminator='\n')
        csvdata = []
        csvdata.append(("基金名称","开盘价","收盘价","日期",))
        writer.writerows(csvdata)    
        
        self.dataToCsv(writer)
        
        csvFile.close()
            
        
        
  
def handle():
    '''测试
    
    多个生产者：IO生产者
一个消费者：CPU密集消费者，  只要 queque 有数据就处理 
线程之间交互通过 queque 对象
    
    '''      
    start=time.time()       
    
    queue=Queue()
    codes=['sh601006','sh601005','sh601003','sh601002','sh601001','sh601007','sh601008',
           'sh601009','sh601010','sh601011','sh601012','sh601013','sh601014','sh601015','sh601016','sh601017',
           'sh601018','sh601019','sh601020','sh601021','sh601022','sh601023','sh601024','sh601025','sh601026']       
    
        
    ct=ConvertThread(queue)    
    dts=[DownloadThread(codes[i],queue) for i in range(len(codes))]
     
    #线程都启动起来，等待cup 调用 
    for t in dts:
        t.start()
    ct.start()
     
    #只有在t线程执行完，才能执行主线程 
    for t in dts:
        t.join()
      
    queue.put((-1,None)) 
     
    #只有在ct线程执行完，才能执行主线程 
    ct.join() 
     
    print("耗时 %s" % (time.time() - start) )




handle()

 

转载于:https://my.oschina.net/v512345/blog/2223006