python bs4 csv requests 爬虫 爬取携程火车票网址信息并保存

目的：爬取携程网址

火车 中的单程与中转

单程
url=“https://trains.ctrip.com/trainbooking/search?tocn=%25e5%258d%2583%25e5%25b2%259b%25e6%25b9%2596&fromcn=%25e6%259d%25ad%25e5%25b7%259e&day=2020-12-31”
中转
url=“https://trains.ctrip.com/pages/booking/hubSingleTrip?ticketType=2&fromCn=%25E6%259D%25AD%25E5%25B7%259E&toCn=%25E5%258D%2583%25E5%25B2%259B%25E6%25B9%2596&departDate=2020-12-31”
采用parse.quote（）进行url转码
采用csv进行数据保存
random.choice进行选择一个User Agent 自认为这是个不错的习惯
携程单程信息在原网页源代码中
携程中转网址火车中中转信息保存在json文件中(js_url)

LET’S GO

url="https://trains.ctrip.com/pages/booking/hubSingleTrip?ticketType=5&fromCn=%25E6%259D%25AD%25E5%25B7%259E&toCn=%25E6%2596%25B0%25E4%25B9%25A1&departDate=2020-12-30" #  携程单程火车原网址 查询参数 fromcn 出发站 tocn 目的站 departDate 日期
#原网页查询参数需要进行两次url编码（注意点1）
#携程单程信息在原网页源代码中
'''
url="https://trains.ctrip.com/pages/booking/hubSingleTrip?ticketType=2&fromCn=%25E6%259D%25AD%25E5%25B7%259E&toCn=%25E5%258D%2583%25E5%25B2%259B%25E6%25B9%2596&departDate=2020-12-31"
js_url="https://trains.ctrip.com/pages/booking/getTransferList?departureStation=%2525E6%25259D%2525AD%2525E5%2525B7%25259E&arrivalStation=%2525E6%252596%2525B0%2525E4%2525B9%2525A1&departDateStr=2020-12-30"    
携程中转网址火车中中转信息保存在json文件中(js_url) 查询参数departureStation arrivalStation departDateStr
类似稍加自己比较即可发现
js_url查询参数需要进行三次url编码（注意点2）
'''

from urllib import parse
import random
from bs4 import BeautifulSoup
import  csv
import os
import requests
# print(parse.unquote((parse.unquote("%25E6%259D%25AD%25E5%25B7%259E"))))
fromArea = input("出发站")
toArea = input("目的站")
date=input("年-月-日 ：")
if not os.path.exists("D:/携程查找练习"):#创建后续保存文件
    os.mkdir("D:/携程查找练习")
class NewsByTransfer():#该类用于爬取中转的信息
    def __init__(self):#初始化
        self.fromArea=fromArea
        self.toArea=toArea
        self.date=date
    def getOneJsUrl(self,fromArea,toArea,date):#进行js_url拼接
        fromArea=parse.quote(parse.quote(fromArea))
        departureStation=parse.quote(fromArea)
        toArea=parse.quote(parse.quote(toArea))
        arrivalStation=parse.quote(toArea)
        url="https://trains.ctrip.com/pages/booking/hubSingleTrip?ticketType=5&fromCn="+fromArea+"&toCn="+toArea #原网页
        js_url="https://trains.ctrip.com/pages/booking/getTransferList?departureStation="+departureStation+"&arrivalStation="+arrivalStation
        js_url=js_url+"&departDateStr="+date
        # print(url)
        print(js_url)
        return js_url
    def getOneNews(self,js_url):#爬取js_url信息
        UA = [
            "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
            "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
            'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
            'Opera/9.25 (Windows NT 5.1; U; en)',
            'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
            'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
            'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',
            'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',
            "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",
            "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0 "
        ]
        user_agent = random.choice(UA)
        text=requests.get(js_url,headers={
     "User-Agent":user_agent}).json() #获取json字符串用于python字典处理
        transferList=text["data"]["transferList"]#第一次定位主要信息列表
        csvList=[]#创建csv后续dictwriter写入 保存列表
        for oneTransfer in  transferList:
            # print(oneTransfer)
            tranDict={
     }
            tranDict["总出发站"] = oneTransfer["departStation"]
            tranDict["总目的站"] = oneTransfer["arriveStation"]
            tranDict["总信息"] = oneTransfer["transferStation"] + "换乘 停留" + oneTransfer["transferTakeTime"] + " 全程" + \
                              oneTransfer["totalRuntime"] + " 价格" + oneTransfer['showPriceText']
            trainTransferInfosList=oneTransfer["trainTransferInfos"]
            for trainTransferInfos in trainTransferInfosList:
                tranDict[f"班次列车号{trainTransferInfos['sequence']}"]=trainTransferInfos['trainNo']
                tranDict[f"发车时间-到站时间{trainTransferInfos['sequence']}"]=trainTransferInfos['departDate']+" "+  \
                    trainTransferInfos['departTime']+"---"+trainTransferInfos['arriveDate']+" "+trainTransferInfos['arriveTime']
                tranDict[f"发车站-目的站{trainTransferInfos['sequence']}"]=trainTransferInfos[ 'departStation']+"---"  +\
                    trainTransferInfos["arriveStation"]
            csvList.append(tranDict)
            print(csvList)
        return csvList
    def mkcsv(self,csvlist):#创建csv文件
        with open(f"D:/携程查找练习/{csvlist[0]['总出发站']}到{csvlist[0]['总目的站']}转站查找.csv","w+",newline="",encoding="utf-8") as f:
                writer = csv.DictWriter(f, list(csvlist[0].keys()))
                writer.writeheader()
                writer.writerows(csvlist)
    def main(self):
        js_url = self.getOneJsUrl(self.fromArea, self.toArea, self.date)
        csvList = self.getOneNews(js_url)
        self.mkcsv(csvList)
        print(csvList)

class NewsBySingle():#爬取单程信息
    def __init__(self):
        self.fromArea=fromArea
        self.toArea=toArea
        self.date=date
    def getOneUrl(self,fromArea,toArea,date):
        fromArea=parse.quote(parse.quote(fromArea))
        toArea=parse.quote(parse.quote(toArea))
        url="https://trains.ctrip.com/trainBooking/search?ticketType=0&fromCn="+fromArea+"&toCn="+toArea+"&day="+self.date+"&mkt_header=&orderSource="
        # print(url)
        print(url)
        return url
    def getOneNews(self,url):
        UA = [
            "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
            "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
            'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
            'Opera/9.25 (Windows NT 5.1; U; en)',
            'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
            'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
            'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',
            'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',
            "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",
            "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0 "
        ]
        user_agent = random.choice(UA)
        text=requests.get(url,headers={
     "User-Agent":user_agent}).content.decode("utf-8")
        print(text)#获取源代码为后续bs4解析
        soup=BeautifulSoup(text,"lxml")
        oneTripList=soup.select("div.railway_list")
        print(len(oneTripList))
        oneTripNewList=[]
        for oneTrip in oneTripList:
            oneTripDict={
     }
            print(oneTrip)
            oneTripDict["班次列车号"]=oneTrip.select("strong")[0].string
            oneTripDict["出发站名称"]=oneTrip.select("span")[0].string
            oneTripDict["出发站时间"]=oneTrip.select("strong")[1].string
            oneTripDict["中途时间"]=list(oneTrip.select("div.haoshi")[0].stripped_strings)[0]
            oneTripDict["目的站名称"]=oneTrip.select("span")[1].string
            oneTripDict["到站时间"]=oneTrip.select("strong")[2].string
            print(oneTripDict)
            oneTripNewList.append(oneTripDict)
            print("---"*60)
        print(oneTripNewList)
        return oneTripNewList
    def mkcsv(self,oneTripNewList):
        with open(f"D:/携程查找练习/{oneTripNewList[0]['出发站名称']}到{oneTripNewList[0]['目的站名称']}单程查找.csv","w+",newline="",encoding="utf-8") as f:
                writer = csv.DictWriter(f, list(oneTripNewList[0].keys()))
                writer.writeheader()
                writer.writerows(oneTripNewList)
    def main(self):
        url=self.getOneUrl(self.fromArea, self.toArea, self.date)
        oneTripNewList=self.getOneNews(url)
        self.mkcsv(oneTripNewList)
NewsByTransfer().main()
NewsBySingle().main()

总结一下还是不难的，基础打牢，python类函数运用 bs4解析（感觉比xpath直观）然后浏览器network中xhr就简单看看写一下就行，给个点赞好不，创造不易，无需关注，给赞就行