python 单线程与多线程爬虫

帮别人写爬虫，先是单线程，太慢，改了多线程

1.单线程

import urllib
import urllib.request
import requests
import xlwt
import re
import string
 
def set_style(name,height,bold=False):
  style = xlwt.XFStyle() # 初始化样式
 
  font = xlwt.Font() # 为样式创建字体
  font.name = name # 'Times New Roman'
  font.bold = bold
  font.color_index = 4
  font.height = height
 
  style.font = font
 
  return style

index=0
r=0
wdk=xlwt.Workbook()       
sheet1=wdk.add_sheet('sheet1',cell_overwrite_ok=True)  #excel单元格名字
row=['招聘职位','月薪','学历要求','工作经验','应聘要求','工作单位']
for i in row:
    sheet1.write(r,index,i)
    index+=1
r+=1
import datetime

starttime = datetime.datetime.now()


for pn in range(0,200,20):
    url = 'http://zhaopin.baidu.com/quanzhi?tid=4139&ie=utf8&oe=utf8&query=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98%E5%B7%A5%E7%A8%8B%E5%B8%88&city_sug=%E9%95%BF%E6%B2%99&&detailmode=close&rn=20&pn='+str(pn)#打开网页
    wp = urllib.request.urlopen(url)
    content = wp.read()
    hrefPatten=r"(href='.{0,200}数据挖掘工程师')"
    href=hrefPatten.encode("utf-8")
    hrefC = re.findall(href, content, re.S)  #返回所有匹配正则表达式的值于列表中
    hre=['']*len(hrefC)
    for i in range (len(hrefC)):
        hre[i]=urllib.parse.unquote(str(hrefC[i].decode('utf-8')).replace("href='","http://zhaopin.baidu.com")[:-1])#将href转化为网址

    for i in range (len(hre)):
        a=requests.get(hre[i])
        
        content1=a.content.decode('utf-8')
        pattern=('(.*?).*?class="salary">(.*?).*?class="xueli">(.*?).*?class="minge">(.*?).*?class="duty duty-box">(.*?)
.*?class="line-clamp2">(.*?)
')

        items=re.findall(pattern, content1, re.S)#抓取招聘信息
        for item in items:
            flag=0
            for _item in item:
                if _item=="":
                    flag=1
            if flag==0:
                for k in range(0,6):
                    sheet1.write(r,k,item[k])
                r+=1

wdk.save('/home/y.xls')
endtime = datetime.datetime.now()

print((endtime - starttime).seconds)

时间我是80s左右

2.多线程

import urllib
import urllib.request
import requests
import xlwt
import re
import string
import threading
from time import ctime,sleep

def set_style(name,height,bold=False):
  style = xlwt.XFStyle() # 初始化样式
 
  font = xlwt.Font() # 为样式创建字体
  font.name = name # 'Times New Roman'
  font.bold = bold
  font.color_index = 4
  font.height = height
 
  style.font = font
 
  return style

import datetime
starttime = datetime.datetime.now()

index=0
global r
r=0
wdk=xlwt.Workbook()       
sheet1=wdk.add_sheet('sheet1',cell_overwrite_ok=True)  #excel单元格名字
row=['招聘职位','月薪','学历要求','工作经验','应聘要求','工作单位']
for i in row:
    sheet1.write(r,index,i)
    index+=1
r+=1
inte=[]
def f(pn):
    url = 'http://zhaopin.baidu.com/quanzhi?tid=4139&ie=utf8&oe=utf8&query=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98%E5%B7%A5%E7%A8%8B%E5%B8%88&city_sug=%E9%95%BF%E6%B2%99&&detailmode=close&rn=20&pn='+str(pn)#打开网页
    wp = urllib.request.urlopen(url)
    content = wp.read()
    hrefPatten=r"(href='.{0,200}数据挖掘工程师')"
    href=hrefPatten.encode("utf-8")
    hrefC = re.findall(href, content, re.S)  #返回所有匹配正则表达式的值于列表中
    hre=['']*len(hrefC)
    for i in range (len(hrefC)):
        hre[i]=urllib.parse.unquote(str(hrefC[i].decode('utf-8')).replace("href='","http://zhaopin.baidu.com")[:-1])#将href转化为网址
    for i in range (len(hre)):
        a=requests.get(hre[i])
        
        content1=a.content.decode('utf-8')
        pattern=('(.*?).*?class="salary">(.*?).*?class="xueli">(.*?).*?class="minge">(.*?).*?class="duty duty-box">(.*?)
.*?class="line-clamp2">(.*?)
')

        items=re.findall(pattern, content1, re.S)#抓取招聘信息
        for item in items:
            flag=0
            for _item in item:
                if _item=="":
                    flag=1
            if flag==0:
                inte.append(item)
    endtime = datetime.datetime.now()
    print((endtime - starttime).seconds)
                
threads = []
for pn in range(0,200,20):
    threads.append(threading.Thread(target=f,args=(pn,)))
    sleep(0.1)

import datetime

starttime = datetime.datetime.now()

if __name__ == '__main__':
    for t in threads:
        t.setDaemon(True)
        t.start()

    sleep(15)
    ind=1
    for item in inte:
        for k in range(0,6):
            sheet1.write(ind,k,item[k])
        ind+=1
    wdk.save('/home/t.xls')
        

15s完事