python自学-class24-进程线程练习Plus
进程线程练习Plus
1. 数据检索多进程
2. 求东方财富平均市值
3. 多线程统计平均市值
4. 多进程统计平均市值
5. 单线程爬取邮箱
6. 多线程爬取邮箱
7. 多线程简单文件归并
8. 多线程文件归并—多个文件交叉写入
++++++++++++++++++++++++++++++++++++++++++
1.数据检索多进程
昨天写了一个多线程的，逻辑差不太多，也是全部将数据载入内存先；
import multiprocessing
def finddata(datalist,istart,iend,searchstr,lastlist):
for i in range(istart,iend):
line=datalist[i].decode("gbk","ignore")
if line.find(searchstr)!=-1:
print(line)
lastlist.append(line)
def loaddata(datalist):
path = "D:\\Python代码\\class15\\图形化编程\\txm.txt" # 路径
file = open(path, "rb")
datalist.extend(file.readlines()) # 全部读入内存
if __name__=="__main__":
datalist = multiprocessing.Manager().list() #主进程有一份,读取文件
lastlist=multiprocessing.Manager().list() #保存结果
loaddata(datalist)
lines=len(datalist)
N=10
processlist=[]
findstr=input("请输入查询的人名：")
for i in range(0,N-1):
process=multiprocessing.Process(target=finddata,
args=(datalist,
i*(lines//(N-1)),
(i+1)*(lines//(N-1)),
findstr,
lastlist))
process.start()
processlist.append(process)
mylastp=multiprocessing.Process(target=finddata,
args=(datalist,
0,
lines,
findstr,
lastlist))
mylastp.start()
processlist.append(mylastp)
for process in processlist:
process.join() #主进程等待其他进程都干完活
file=open("last.txt","wb") #保存结果
for line in lastlist:
file.write(line.encode("utf-8"))
file.close()
运行效果：
2.求平均市值：
这是从东方财富网爬下来的股票历年数据（爬虫传送门：爬取东方财富历年股票数据），可以看出，市值在第14栏，因此代码如下
import csv
def gettotall(path):
reader=csv.reader(open(path,"r"))
num=0
alldata=0
for item in reader:
if num==0:
pass
else:
alldata+=eval(item[13]) #累加所有市值
num+=1
return alldata/num
path = r"D:\Python代码\class20\down\20201010\0600009.csv"
print(gettotall(path))
运行效果：
3.多线程统计平均市值：
主要用到队列共享数据，可以同时统计多个文件；
import csv
import threading
import queue
import os
myqueue=queue.Queue(0)#0表示无限容量
class getdatatotal(threading.Thread):
def __init__(self,path,queue):
threading.Thread.__init__(self)
self.path=path
self.queue=queue
def run(self):
print("start",self.getName())
reader = csv.reader(open(self.path, "r"))
num = 0
alldata = 0
for item in reader:
if num == 0:
pass
else:
alldata += eval(item[13]) # 累加所有市值
num += 1
print(alldata / num,self.getName())
self.queue.put(alldata / num) #保存结果到队列
path="D:\\Python代码\\class20\\down\\20201010"
filelist=os.listdir(path)
filepathlist=[]
for filename in filelist:
filename=path+"\\"+filename
filepathlist.append(filename)
threadlist=[]
for path in filepathlist:
mythread=getdatatotal(path,myqueue)
mythread.start()
threadlist.append(mythread) #创建线程加入列表
for thread in threadlist:
thread.join()
while not myqueue.empty():
data=myqueue.get()
print(data)
运行效果：
与上一题一致
4.多进程统计平均市值：
import csv
import multiprocessing
import os
def gettotall(path,queue):
reader=csv.reader(open(path,"r"))
num=0
alldata=0
for item in reader:
if num==0:
pass
else:
alldata+=eval(item[13]) #累加所有市值
num+=1
queue.put(alldata/num)
if __name__=="__main__":
queue = multiprocessing.Queue()
path="D:\\Python代码\\class20\\down\\20200201"
filelist=os.listdir(path)
filepathlist=[]
for filename in filelist:
filename=path+"\\"+filename
filepathlist.append(filename)
processlist=[]
for filepath in filepathlist:
p=multiprocessing.Process(target=gettotall,args=(filepath,queue))
p.start()
processlist.append(p)
for process in processlist:
process.join()
while not queue.empty():
data=queue.get()
print(data)
运行效果：
也是一致的
5.单线程爬取邮箱（BFS模式）：
爬取天涯社区的邮箱信息，因为采用了BFS模式，理论上如果从百度爬起可以爬到世界尽头
import urllib.request
import urllib
import re
import queue
import threading
def getallyurl(data):
alllist=[]
mylist1=[]
mylist2=[]
mylist1=gethttp(data)
if len(mylist1)>0:
mylist2=getabsyurl(mylist1[0],data)
alllist.extend(mylist1)
alllist.extend(mylist2)
return alllist
def getabsyurl(url,data):
try:
regex=re.compile("href=\"(.*?)\"",re.IGNORECASE)
httplist=regex.findall(data)
newhttplist=httplist.copy() #深拷贝
for data in newhttplist:
if data.find("http://")!=-1:
httplist.remove(data)
if data.find("javascript")!=-1:
httplist.remove(data)
hostname=gethostname(url)
if hostname!=None:
for i in range(len(httplist)):
httplist[i]=hostname+httplist[i]
return httplist
except:
return ""
def gethostname(httpstr): #抓取主机名称
try:
mailregex = re.compile(r"(http://\S*?)/", re.IGNORECASE)
mylist = mailregex.findall(httpstr)
if len(mylist)==0:
return None
else:
return mylist[0]
except:
return None
def gethttp(data):
try:
mailregex = re.compile(r"(http://\S*?)[\"|>|)]", re.IGNORECASE)
mylist = mailregex.findall(data)
return mylist
except:
return ""
def getallemail(data):
try:
mailregex = re.compile(r"([A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4})", re.IGNORECASE)
mylist = mailregex.findall(data)
return mylist
except:
return ""
def getdata(url):
try:
data = urllib.request.urlopen(url).read().decode("utf-8")
return data
except:
return "" #发生异常返回空
def BFS(urlstr,myqueue):
with sem:
threading.Thread(target=BFSget,args=(urlstr,myqueue)).start()
def BFSget(urlstr,myqueue):
myqueue.put(urlstr)
while not myqueue.empty():
url=myqueue.get() #队列弹出数据
print(url) #打印url连接
pagedata=getdata(url) #获取网页源码
emaillist=getallemail(pagedata) #提取邮箱到列表
if len(emaillist)!=0: #邮箱不为空
for email in emaillist: #打印所有邮箱
print(email)
newurllist=getallyurl(pagedata) #抓取所有的url
if len(newurllist)!=0: #p判断长度
for urlstr in newurllist: #循环处理所有url
myqueue.put(urlstr) #插入
myqueue=queue.Queue(0)
sem = threading.Semaphore(5)
BFS("http://bbs.tianya.cn/post-140-393974-5.shtml",myqueue)
#BFS("http://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&srcqid=4974407339122272475&tn=48020221_29_hao_pg&wd=%E5%B2%9B%E5%9B%BD%E5%A4%A7%E7%89%87%20%E7%95%99%E4%B8%8B%E9%82%AE%E7%AE%B1&oq=%25E5%25A4%25A9%25E6%25B6%25AF%25E5%25A4%25A7%25E5%25AD%25A6%25E8%2580%2581%25E5%25B8%2588%25E9%2582%25AE%25E7%25AE%25B1&rsv_pq=e1e17d5400093975&rsv_t=83fc1KipT0e6dU2l8G8651PAihzqMxhN1tT8Ue1JiKtvBGgKILwuquM4g7%2BKNKKKp6AkBxK7opGg&rqlang=cn&rsv_enter=1&rsv_dl=tb&rsv_sug3=40&rsv_sug1=4&rsv_sug7=100&rsv_sug2=0&rsv_btype=t&inputT=11395&rsv_sug4=11395")
运行效果：
爬取到的第一个邮箱与网页一致
6.多线程爬取邮箱并保存：
import urllib.request
import urllib
import re
import queue
import threading
import time
def getallyurl(data):
alllist=[]
mylist1=[]
mylist2=[]
mylist1=gethttp(data)
if len(mylist1)>0:
mylist2=getabsyurl(mylist1[0],data)
alllist.extend(mylist1)
alllist.extend(mylist2)
return alllist
def getabsyurl(url,data):
try:
regex=re.compile("href=\"(.*?)\"",re.IGNORECASE)
httplist=regex.findall(data)
newhttplist=httplist.copy() #深拷贝
for data in newhttplist:
if data.find("http://")!=-1:
httplist.remove(data)
if data.find("javascript")!=-1:
httplist.remove(data)
hostname=gethostname(url)
if hostname!=None:
for i in range(len(httplist)):
httplist[i]=hostname+httplist[i]
return httplist
except:
return ""
def gethostname(httpstr): #抓取主机名称
try:
mailregex = re.compile(r"(http://\S*?)/", re.IGNORECASE)
mylist = mailregex.findall(httpstr)
if len(mylist)==0:
return None
else:
return mylist[0]
except:
return None
def gethttp(data):
try:
mailregex = re.compile(r"(http://\S*?)[\"|>|)]", re.IGNORECASE)
mylist = mailregex.findall(data)
return mylist
except:
return ""
def getallemail(data):
try:
mailregex = re.compile(r"([A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4})", re.IGNORECASE)
mylist = mailregex.findall(data)
return mylist
except:
return ""
def getdata(url):
try:
data = urllib.request.urlopen(url).read().decode("utf-8")
return data
except:
return "" #发生异常返回空
def BFS(urlstr,myqueue):
with sem:
threading.Thread(target=newBFS,args=(urlstr,myqueue)).start()
def newBFS(urlstr,myqueue):
#print(urlstr)
pagedata = getdata(urlstr) # 获取网页源码
emaillist = getallemail(pagedata) # 提取邮箱到列表
if len(emaillist) != 0: # 邮箱不为空
for email in emaillist: # 打印所有邮箱
print(email)
myqueue.put(email+'\r\n') #队列压入邮箱
newurllist = getallyurl(pagedata) # 抓取所有的url
if len(newurllist) != 0: # p判断长度
for urlstr in newurllist: # 循环处理所有url
with sem:
threading.Thread(target=newBFS, args=(urlstr, myqueue)).start()
def BFSget(urlstr,myqueue):
myqueue.put(urlstr)
while not myqueue.empty():
url=myqueue.get() #队列弹出数据
print(url) #打印url连接
pagedata=getdata(url) #获取网页源码
emaillist=getallemail(pagedata) #提取邮箱到列表
if len(emaillist)!=0: #邮箱不为空
for email in emaillist: #打印所有邮箱
print(email)
newurllist=getallyurl(pagedata) #抓取所有的url
if len(newurllist)!=0: #p判断长度
for urlstr in newurllist: #循环处理所有url
myqueue.put(urlstr) #插入
def saveemail():
global myqueue
file=open("email.txt","wb")
while True:
time.sleep(5)
while not myqueue.empty():
data=myqueue.get()
file.write(data.encode("utf-8")) #实时写入
file.flush()
file.close()
myqueue=queue.Queue(0)
sem = threading.Semaphore(10)
timethread=threading.Timer(5,saveemail)
timethread.start()
BFS("http://bbs.tianya.cn/post-140-393974-5.shtml",myqueue)
#BFS("http://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&srcqid=4974407339122272475&tn=48020221_29_hao_pg&wd=%E5%B2%9B%E5%9B%BD%E5%A4%A7%E7%89%87%20%E7%95%99%E4%B8%8B%E9%82%AE%E7%AE%B1&oq=%25E5%25A4%25A9%25E6%25B6%25AF%25E5%25A4%25A7%25E5%25AD%25A6%25E8%2580%2581%25E5%25B8%2588%25E9%2582%25AE%25E7%25AE%25B1&rsv_pq=e1e17d5400093975&rsv_t=83fc1KipT0e6dU2l8G8651PAihzqMxhN1tT8Ue1JiKtvBGgKILwuquM4g7%2BKNKKKp6AkBxK7opGg&rqlang=cn&rsv_enter=1&rsv_dl=tb&rsv_sug3=40&rsv_sug1=4&rsv_sug7=100&rsv_sug2=0&rsv_btype=t&inputT=11395&rsv_sug4=11395",myqueue)
7.多线程文件归并
归并之前爬取的三个股票信息csv文件，乱序的
import csv
import threading
#缓存，实时刷新
#
def getlist(path):
reader=csv.reader(open(path,"r")) #读取文件
alllist=[]
for item in reader:#读取每一行
alllist.append(item) #加入列表
return alllist
def thread1(csvw,e1,e2,alllist):
for line in alllist:
csvw.writerow(line)
e2.set() #通知e2干活
e1.wait() #e1等待
def thread2(csvw,e2,e3,alllist):
for line in alllist:
e2.wait() # 等待干活
csvw.writerow(line)
e3.set() # 通知e3干活
def thread3(csvw,e3,e1,alllist):
for line in alllist:
e3.wait()
csvw.writerow(line)
e1.set()
#with open("1.csv","w",newline="") as datacsv:
csvw=csv.writer(open("xyz.csv","w",newline=""),dialect=("excel")) #最常用格式excel格式
e1 = threading.Event() # 事件
e2 = threading.Event() # 事件
e3 = threading.Event() # 事件
mylist1=getlist(r"D:\Python代码\class20\down\20201010\0600009.csv")
mylist2=getlist(r"D:\Python代码\class20\down\20201010\0600010.csv")
mylist3=getlist(r"D:\Python代码\class20\down\20201010\0600011.csv")
threading.Thread(target=thread1,args=(csvw,e1,e2,mylist1)).start()
threading.Thread(target=thread2,args=(csvw,e2,e3,mylist2)).start()
threading.Thread(target=thread3,args=(csvw,e3,e1,mylist3)).start()
csvw.writerow([])
运行效果：
8.多线程文件归并-交叉写入
import csv
import threading
#缓存，实时刷新
def getlist(path):
reader=csv.reader(open(path,"r")) #读取文件
alllist=[]
for item in reader:#读取每一行
alllist.append(item) #加入列表
return alllist
def getfilelist(path):
file=open(path,"rb")
alllist=file.readlines()
return alllist
def thread1(alllist,csvw,e1,e2):
for line in alllist:
csvw.write(line)
print(line.decode("utf-8", "ignore"))
e2.set() #通知e2干活
e1.wait(5) #e1等待
e1.clear()
def thread2(alllist,csvw,e2,e3):
for line in alllist:
e2.wait(5) # 等待干活
csvw.write(line)
print(line.decode("utf-8","ignore"))
e2.clear()
e3.set() # 通知e3干活
def thread3(alllist,csvw,e3,e1):
for line in alllist:
e3.wait(5)
csvw.write(line)
print(line.decode("utf-8", "ignore"))
e3.clear()
e1.set()
#with open("1.csv","w",newline="") as datacsv:
#csvw=csv.writer(open("xyz.csv","w",newline=""),dialect=("excel")) #最常用格式excel格式
file=open("xyz.txt",'wb')
e1 = threading.Event() # 事件
e2 = threading.Event() # 事件
e3 = threading.Event() # 事件
mylist1=getfilelist(r"D:\Python代码\class20\down\20201010\1.txt")
mylist2=getfilelist(r"D:\Python代码\class20\down\20201010\2.txt")
mylist3=getfilelist(r"D:\Python代码\class20\down\20201010\3.txt")
myth1=threading.Thread(target=thread1,args=(mylist1,file,e1,e2))
myth2=threading.Thread(target=thread2,args=(mylist2,file,e2,e3))
myth3=threading.Thread(target=thread3,args=(mylist3,file,e3,e1))
myth1.start()
myth2.start()
myth3.start()
myth1.join()
myth2.join()
myth3.join()
print("over")
file.close()
运行效果：
三种股票信息交叉写入
总结
芜湖！！！！！！