爬虫训练之：xpath解析的使用-爬取活动信息并存储（单线程）

只做练习，不做他用！

import re
import threading
import requests
import random
from lxml import etree
from fake_useragent import UserAgent

UA=['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36']
headers={'user-agent':random.choice(UA)}

def get_infor(n):
    url=f'https://www.nyato.com/manzhan/?type=expired&p={n}'
    resp=requests.get(url,headers=headers).text
    #获取详情页url
    all_url1=re.compile('
)
    all_url=all_url1.findall(resp)
    #获取详情页的内容信息
    for url1 in all_url:
        resp1=requests.get(url1,headers=headers).text
        #获取名称
        name1=re.compile('
(.*?)
')
        name=name1.findall(resp1)
        # 获取地址
        address1=re.compile('(.*?)')
        address=address1.findall(resp1)
        # 获取票价信息
        price1=re.compile('(.*?)')
        price=price1.findall(resp1)

        #xpath获取评价
        html=etree.HTML(resp1)
        score_url = html.xpath("//div[@class='mt10']/img/@src")[0]
        scrore1=re.compile('//static.nyato.cn/expo-image/stars/star-(.*?).0.png')
        scrore=scrore1.findall(score_url)
        if scrore==[]:
            scrore=['暂无']
        else:
            scrore

        #xpath获取各类信息
        other = ",".join(html.xpath(
            "//span[@class='sf6 f18 fwb ml15']/text()"))

        save_xlsx(name,address,price,scrore,other,url1)


def save_xlsx(name,address,price,scrore,other,url1):
    n=1
    long_str=f'{n},{name[0]},{address[0]},{price[0]},{scrore[0]},{other},{url1}\n'.encode('utf8')
    with open(f'活动数据3.csv','a+',encoding='ANSI',errors='ignore') as f:
        f.write(long_str.decode('utf8'))
        print(name[0],'存储完毕！')
        n+=1

if __name__ == '__main__':
    long_str1 = f'序号,活动名称,活动地点,费用,评分,场馆,交通,热度,时效,url\n'
    with open(f'活动数据3.csv','a+',encoding='ANSI',errors='ignore') as f:
        f.write(long_str1)

    for n in range(1,20):
        get_infor(n)