py爬虫实战

一、糗事百科视频爬取

先找到对应的页面，分析视频的来源，通过正则匹配到链接，然后再通过“美味的汤”来获取对应的视频的标题，进行下载

import requests
import re
from bs4 import BeautifulSoup
url="https://www.qiushibaike.com/video/"
headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36"}
resp=requests.get(url,headers=headers)

content = resp.content.decode('utf-8')
soup=BeautifulSoup(content,'html.parser')
text=resp.text
# print(text)
list=re.findall(r'',text)
divs=soup.find_all('div',class_="content")
title=[]
for item in divs:
    title.append(item.find('span').text.strip())
lst=[]
for item in list:
    lst.append("https:"+item)
count=0
for item in lst:
    resp=requests.get(item,headers=headers)
    with open("video/"+str(title[count])+".mp4","wb") as file:
        file.write(resp.content)
    print("已下载完第"+str(count)+"个")
    count+=1

print("视频下载完毕")

 

二、链家二手房数据

本次主要利用“BeautifulSoup”和mysql数据库

import re
import requests
import mysql.connector
from bs4 import BeautifulSoup

class LianJia():

    mydb=mysql.connector.connect(host='localhost',user='root',password='fengge666',database='hotel')

    mycursor=mydb.cursor()

    #初始化
    def __init__(self):
        self.url="https://bj.lianjia.com/chengjiao/pg{0}/"
        self.headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}

    def send_request(self,url):
        resp=requests.get(url,headers=self.headers)
        if resp:
            self.parse_content(resp)

    def parse_content(self,resp):
        lst=[]
        html=resp.text
        bs=BeautifulSoup(html,'html.parser')
        ul=bs.find('ul',class_='listContent')
        li_lst=ul.find_all('li')
        for item in li_lst:
            title=item.find('div',class_='title').text
            houseInfo=item.find('div',class_='houseInfo').text
            data=item.find('div',class_='dealDate').text
            money=item.find('div',class_='totalPrice').text
            flood=item.find('div',class_='positionInfo').text
            price=item.find('div',class_='unitPrice').text
            span=item.find('span',class_='dealCycleTxt')
            span_lst=span.find_all('span')
            agent=item.find('a',class_='agent_name').text
            lst.append((title,houseInfo,data,money,flood,price,span_lst[0].text,span_lst[1].text,agent))
        #开始存储到数据库
        self.write_mysql(lst)

    def write_mysql(self,lst):
        # print(self.mydb)
        tuple_lst=tuple(lst)
        sql="insert into house (title,houseInfo,data,money,flood,price,current_money,current_data,agent) values (%s,%s,%s,%s,%s,%s,%s,%s,%s)"
        self.mycursor.executemany(sql,tuple_lst)
        self.mydb.commit()

    def start(self):
        for i in range(1,11):
            full_url=self.url.format(i)
            resp=self.send_request(full_url)
            if resp:
                self.parse_content(self,resp)

if __name__ == '__main__':
    lianjia=LianJia()
    lianjia.start()

 

三、爬取招聘职位网信息

主要利用beautifulSoup对其网页进行分析，去除对应的数据，将其保存在xlsx文件里，对其进行分页爬取

import re
import openpyxl
from bs4 import BeautifulSoup
import requests
import time

def send_request(id,page):
    url='https://www.jobui.com/company/{0}/jobs/p{1}'.format(id,page)
    headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36"}
    resp=requests.get(url,headers=headers)
    return resp.text

lst=[]
def parse_html(html):
    bs=BeautifulSoup(html,'html.parser')
    job_lst=bs.find_all('div',class_='c-job-list')
    for item in job_lst:
        name=item.find('h3').text
        div_tag=item.find('div',class_='job-desc')
        span_tag=div_tag.find_all('span')
        url=item.find('a',class_='job-name')['href']
        url='https://www.jobui.com'+url
        lst.append([name,span_tag[0].text,span_tag[1].text,url])


def save(lst):
    wk=openpyxl.Workbook()
    sheet=wk.active
    for item in lst:
        sheet.append(item)
    wk.save('招聘信息.xlsx')

def start(id,pages):
    for page in range(1,pages+1):
        resp_data=send_request(id,page)
        parse_html(resp_data)
        time.sleep(2)
    save(lst)

if __name__ == '__main__':
    id='10375749'
    pages=3
    start(id,pages)

 

四、爬取QQ音乐排行旁信息

这个排行榜的信息封装在json数据里，因此需要我们对json格式进行分析，取出想要的数据，保存到数据库里

import requests
import re
import mysql.connector

def get_request():
    url="https://u.y.qq.com/cgi-bin/musics.fcg?-=getUCGI9057052357882678&g_tk=130572444&sign=zzan8er9xsqr1dg0y3e7df30d14b15a2b335cedcd0d6c6c883f&loginUin=1751520702&hostUin=0&format=json&inCharset=utf8&outCharset=utf-8¬ice=0&platform=yqq.json&needNewCode=0&data=%7B%22detail%22%3A%7B%22module%22%3A%22musicToplist.ToplistInfoServer%22%2C%22method%22%3A%22GetDetail%22%2C%22param%22%3A%7B%22topId%22%3A4%2C%22offset%22%3A0%2C%22num%22%3A20%2C%22period%22%3A%222020-07-23%22%7D%7D%2C%22comm%22%3A%7B%22ct%22%3A24%2C%22cv%22%3A0%7D%7D"
    resp=requests.get(url)
    return resp.json()

def parse_data():
    data=[]
    data_json=get_request()
    lst_song=data_json['detail']['data']['data']['song']
    for item in lst_song:
        data.append((item['rank'],item['title'],item['singerName']))
    return data

def save():
    mydb=mysql.connector.connect(host='localhost',user='root',password='fengge666',database='python_database')
    mycursor=mydb.cursor()
    sql='insert into song_table values(%s,%s,%s)'
    lst=parse_data()
    mycursor.executemany(sql,lst)
    mydb.commit()
    print(mycursor.rowcount,'记录插入成功')

if __name__ == '__main__':
    save()

 

五、爬取12306车次信息

在这里也是对json数据进行解析，唯一有点难度的就是城市的名称和其对应的简称之间的对应，找到对应的城市与简称的json数据报，解析后，得到对应的字典。

然后再对起分析对应的数据，存储到相应的xlsx文件里。

最终的程序模式是：输入起始站，终点站以及出发时间，就会自动帮你找对应的车次以及相应的信息

import requests
import re
import openpyxl

def send_request(begin,end,data):
    lst=getHcity()
    begin=lst[begin]
    end=lst[end]
    url='https://kyfw.12306.cn/otn/leftTicket/query?leftTicketDTO.train_date={0}&leftTicketDTO.from_station={1}&leftTicketDTO.to_station={2}&purpose_codes=ADULT'.format(data,begin,end)
    headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'
             ,'Cookie':'_uab_collina=159514803739447239524678; JSESSIONID=C6B40F3602421EA3F1C176AAAD6D07CD; tk=t4DWOVEUjJCURbdg5YymB_o_T5iBATU7sT2cagOk978qrx1x0; RAIL_EXPIRATION=1595441480862; RAIL_DEVICEID=dCwOxY9htFoUbg-W-ZiiJivIJneE0K0MYpVRFCEGJZVXr2VWjywrVdOvEJ6HKFapkeqFwD82pjGtJv0fB1SeILpr-60WLkdvjz6zV-hcnclaYrz1AcbOskdjaz3e3fJd007cLRkk4OiauQxiu6zjnhVnI4fytM01; BIGipServerpool_passport=283968010.50215.0000; route=6f50b51faa11b987e576cdb301e545c4; _jc_save_fromStation=%u90AF%u90F8%2CHDP; _jc_save_toStation=%u77F3%u5BB6%u5E84%2CSJP; _jc_save_fromDate=2020-07-19; _jc_save_toDate=2020-07-19; _jc_save_wfdc_flag=dc; BIGipServerotn=3990290698.50210.0000; BIGipServerpassport=887619850.50215.0000; uKey=97a13289be6445586b819425f91b9bcbcc15446c5f37bceb8352cc085d1017a4; current_captcha_type=Z'}
    resp=requests.get(url,headers=headers)
    resp.encoding='utf-8'
    # print(resp.text)
    return resp

def pare_json(resp,city):
    json_ticket=resp.json()
    data_lst=json_ticket['data']['result']
    content=data_lst[0].split('|')
    #遍历车次信息
    lst=[]
    lst.append(['车次','起始站','到达站','一等座','二等座','软卧','硬卧','硬座','出行日期','出发时间','到达时间'])
    for item in data_lst:
        d=item.split('|')
        # d[3]为车次
        # d[6]查询起始站
        # d[7]查询到达站
        # d[31]一等座
        # d[30]二等座
        # d[29]硬座
        # d[23]软卧
        # d[28]硬卧
        # d[8] 出发时间
        # d[9] 到达时间
        # d[13]出行时间
        lst.append([d[3],city[d[6]],city[d[7]],d[31],d[30],d[23],d[28],d[29],d[13],d[8],d[9]])
    return lst


def start(begin,end,data):
    lst=pare_json(send_request(begin,end,data),getcity())
    wk = openpyxl.Workbook()
    sheet = wk.active
    for item in lst:
        sheet.append(item)

    wk.save('车票查询.xlsx')

def getcity():
    url="https://kyfw.12306.cn/otn/resources/js/framework/station_name.js?station_version=1.9151"
    headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}
    resp=requests.get(url,headers=headers)
    resp.encoding='utf-8'

    stations=re.findall('([\u4e00-\u9fa5]+)\|([A-Z]+)',resp.text)
    stations_data=dict(stations)
    stations_d={}
    for item in stations_data:
        stations_d[stations_data[item]]=item
    return stations_d

def getHcity():
    url = "https://kyfw.12306.cn/otn/resources/js/framework/station_name.js?station_version=1.9151"
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}
    resp = requests.get(url, headers=headers)
    resp.encoding = 'utf-8'

    stations = re.findall('([\u4e00-\u9fa5]+)\|([A-Z]+)', resp.text)
    stations_data = dict(stations)
    return stations_data

if __name__ == '__main__':
    begin=input('begin:')
    end=input('end:')
    data=input('data:')
    start(begin,end,data)

 

六、笔趣阁小说爬取

闲来无事，回忆当初看过的奇幻小说，想来就盘他吧！！！

import re
import requests
from bs4 import BeautifulSoup
import demjson
import pymysql
import os

headers = {'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36'}#创建头部信息
url='https://www.xsbiquge.com/12_12735/'
ht='https://www.xsbiquge.com'
r=requests.get(url,headers=headers)
content=r.content.decode('utf-8')
soup = BeautifulSoup(content, 'html.parser')
if not os.path.exists('校园纯情霸主小说'):
    os.mkdir('校园纯情霸主小说')
content=soup.find('div',id='list')
Title = str(content.dt.text[:-2]).strip()
print(Title)
#len(content.dl.contents)-1
lst_dd=content.find_all('dd')
for item in lst_dd:
        link=ht+item.a['href']
        name=item.a.text
        r = requests.get(link, headers=headers)
        content = r.content.decode('utf-8')
        soup = BeautifulSoup(content, 'html.parser')
        text=soup.find('div',id='content').text.replace('    ','\n')
        with open('校园纯情霸主小说' + '/' + name + '.txt', 'a',encoding='utf-8')as f:
            f.write(text)
        print("已完成"+name+" 的爬取！！！")

 

七、毒物特征与症状提取

这是和老师正在做的一个项目，需要将毒物的特征描述文字与对应的一些症状进行匹对，然后生成一个表，又该症状则为1，反之则为0

import csv
import re
import pandas as pd
import openpyxl
import xlrd

lst_xlsx=['二级其他检查.xlsx','二级库呼吸.xlsx','二级库尿液检查.xlsx','二级库循环.xlsx','二级库泌尿生殖.xlsx','二级库消化.xlsx','二级库甲状腺.xlsx','二级库皮肤.xlsx','二级库眼部.xlsx','二级库神经.xlsx',
          '二级库粪便检查.xlsx','二级库肌电图.xlsx','二级库肝功能肾功能.xlsx','二级库胃部检查.xlsx','二级库脑电图.xlsx','二级库血液检查.xlsx','二级心电图.xlsx','其它.xlsx']
name_lst = []
content_lst = []
ans_lst=[]
def get_dw():

    wk = xlrd.open_workbook('duwuxiangqing.xls')
    table = wk.sheet_by_name('Sheet1')
    rows = table.nrows
    cols = table.ncols
    for i in range(2, rows):
        name = table.cell(i, 2).value
        str = table.cell(i, 13).value + table.cell(i, 14).value + table.cell(i, 17).value + table.cell(i, 18).value
        content_lst.append(str)
        name_lst.append(name)
    # for i in range(len(content_lst)):
    # print(name_lst[i],content_lst[i])

def get_title():
    lst_title=[]
    lst_title.append("Name")
    for item in lst_xlsx:
        wk = openpyxl.load_workbook(item)
        sheet = wk.active
        rows = sheet.max_row
        cols = sheet.max_column
        for i in range(1, cols + 1):
            for j in range(1, rows + 1):
                size = sheet.cell(j, i).value
                if (size != None):
                    lst_title.append(size)
    return lst_title

def head():
    global flag, num
    lst_title=[]
    lst_title.append("Name")

    #构建毒物名字
    for item in name_lst:
        k=[]
        k.append(item)
        ans_lst.append(k)

    #遍历毒物特征
    p=0
    for cont_item in content_lst:
        cont_item=str(cont_item)
        print("目前正在进行第："+str(p))
        #读取二级库里的特征
        for item in lst_xlsx:
            wk = openpyxl.load_workbook(item)
            sheet = wk.active
            rows = sheet.max_row
            cols = sheet.max_column
            for i in range(1, cols + 1):
                flag=0
                num=0
                for j in range(1, rows + 1):
                    size = sheet.cell(j, i).value
                    if(size!=None):
                        num+=1
                        if cont_item.find(size)>0 :
                            flag=1
                if flag==1:
                    for f in range(num):
                        ans_lst[p].append("1")
                else:
                    for f in range(num):
                        ans_lst[p].append("0")
        p=p+1

    lst_title=get_title()
    write_title(lst_title,ans_lst)


def write_title(title,ans):
    f = open('data.csv', 'w',encoding='utf-8',newline='')
    # 2. 基于文件对象构建 csv写入对象
    csv_writer = csv.writer(f)
    # 3. 构建列表头
    csv_writer.writerow(title)
    for item in ans:
        csv_writer.writerow(item)


if __name__ == '__main__':
    get_dw()
    head()

 

data.csv部分截图：