爬取淘宝商品信息（正则版）

爬取淘宝商品信息（正则版）

• 确定目标信息的存在形式，可以看到价格前面有标签"view_price"，标题前面有标签"title"

• 程序分为四部分，分别为提取网页HTML文本、解析页面对目标信息进行提取、打印信息以及主函数。

具体如下：

import requests
import re

def getHTMLText(url):
    """提取页面HTML代码，并返回HTML文本"""
    kv = {
     'cookie':'t=5c749e8d453e7e3664735746f5eb5de8; cna=BrXNFDenbXUCAXggNKX9H1bo; thw=cn; tg=0; enc=5LMRHD8305w3oo8X0agYVFUDa7Ox%2F4RBf34oCWap48nRHy%2B%2B1RZCWZJ7ebn%2Fpy7vRNIL8xpS%2Ba0oNFXG5nSu8w%3D%3D; hng=CN%7Czh-CN%7CCNY%7C156; cookie2=10dbf1309bd9a2d5bc9cabe562965aee; _tb_token_=ee67e1a3ee0e5; alitrackid=www.taobao.com; swfstore=308730; v=0; unb=2448224718; sg=%E6%BB%A18d; _l_g_=Ug%3D%3D; skt=d53506c42f2db259; cookie1=BxUHGXuG%2B5Y4Iw7vZCvjLJ0zDvFL2Xy3mjxT%2FRPtFeA%3D; csg=4246b77f; uc3=vt3=F8dByEzfiHo1%2FUIk8VY%3D&id2=UUwU0bQkq1tYDQ%3D%3D&nk2=CN5OZuI3Xv2%2BlbVX&lg2=W5iHLLyFOGW7aA%3D%3D; existShop=MTU1MDU2MTUyMQ%3D%3D; tracknick=king%5Cu4E36%5Cu5C0F%5Cu4E30%5Cu6EE1; lgc=king%5Cu4E36%5Cu5C0F%5Cu4E30%5Cu6EE1; _cc_=VFC%2FuZ9ajQ%3D%3D; dnk=king%5Cu4E36%5Cu5C0F%5Cu4E30%5Cu6EE1; _nk_=king%5Cu4E36%5Cu5C0F%5Cu4E30%5Cu6EE1; cookie17=UUwU0bQkq1tYDQ%3D%3D; lastalitrackid=login.taobao.com; mt=ci=5_1; x=e%3D1%26p%3D*%26s%3D0%26c%3D0%26f%3D0%26g%3D0%26t%3D0%26__ll%3D-1%26_ato%3D0; uc1=cookie14=UoTZ5OXqJxXKdA%3D%3D&lng=zh_CN&cookie16=W5iHLLyFPlMGbLDwA%2BdvAGZqLg%3D%3D&existShop=false&cookie21=UIHiLt3xThH8t7YQouiW&tag=8&cookie15=UIHiLt3xD8xYTw%3D%3D&pas=0; JSESSIONID=F99B5E66516B99D5E7C9F431E402713F; l=bBNU0zKPvJ9oGfuLBOCNZuI8LN_OGIRYjuPRwCfMi_5B46JhzLQOllv3_FJ6Vj5RsK8B4z6vzNp9-etki; isg=BDg4VI5GkPAaMvx83RJGSPCNCeYKCZ0m9uCVOHKp6XNmjdh3GrU6uo2vQcWY5lQD; whl=-1%260%260%261550562673185',
          'user-agent':'Mozilla/5.0'}
    try:
        r = requests.get(url, headers=kv,timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        print("页面提取错误")
        return ""

def parsePage(infoList, html):
    """解析页面，将[价格，名字] 存入列表"""
    try:
        #分析网页源码解析出价格
        price = re.findall(r'\"view_price\":\"\d+\.\d+?"',html)
        title = re.findall(r'\"raw_title\":\".+?\"',html)
        #test this re
        #print(price)
        #print(title)
        for i in range(len(price)):
            _price = eval(price[i].split(':')[1])
            _title = eval(title[i].split(':')[1])
            infoList.append([_price, _title])
    except:
        print('页面解析错误')

def printGoodsList(infoList):
    """将解析好的商品页面打印出来"""
    tplt = '{:^4}\t{:^6}\t{:^10}'
    print(tplt.format("数量","价格", "名字"))
    count = 0
    for goods in infoList:
        count+=1
        print(tplt.format(count,goods[0], goods[1]))        

def main():
    keyword = "书包"
    deep = 1 #页数
    url = 'https://s.taobao.com/search?q=' + keyword
    infoList = []
    for i in range(deep):
        _url = url + '&s=' + str(i*44)
        text = getHTMLText(_url)
        parsePage(infoList, text)
        printGoodsList(infoList)

main()