python网络爬虫笔记二

一、搜索淘宝商品名称和价格信息--定向爬虫

import requests
import re
from bs4 import BeautifulSoup
import bs4

# 获取网页内容
def getHTMLText(url):
    try:
        r = requests.get(url, timeout = 30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        print("getHTMLText failed")
        return ""

# 从网页内容获取商品的价格和名称
def parserPage(ilt, html):
    try:
        # 正则表达式匹配价格名称，淘宝网页中价格和名称的存放方式为键值对
        # "view_price" : "109.00"       "raw_title" : "abcdef"
        plt = re.findall(r'\"view_price\"\:\"[\d\.]*\"', html)
        tlt = re.findall(r'\"raw_title\"\:\".*?\"', html)
        for i in range(len(plt)):
            # eval函数可去掉双引号
            price = eval(plt[i].split(':')[1])
            title = eval(tlt[i].split(':')[1])
            ilt.append([price, title])
    except:
        print("parserPage failed")

def printGoodsList(ilt):
    # 输出格式
    tplt = "{:^4}\t{:^8}\t{:^16}"      
    print(tplt.format("序号","价格","商品名称"))
    count = 0
    for g in ilt:
        count = count + 1
        print(tplt.format(count, g[0], g[1]))

def main():
    goods = "书包"
    depth = 2
    start_url = 'https://s.taobao.com/search?q=' + goods
    infoList = []
    for i in range(depth):
        try:
            # 网页的格式，每页44个商品，翻页url的变化
            url = start_url + '&s=' + str(44*i)     
            html = getHTMLText(url)
            parserPage(infoList,html)
        except:
            continue
    printGoodsList(infoList)

二、股票数据的定向爬虫

import requests
import re
import traceback
from bs4 import BeautifulSoup

def getHTMLText(url):
    try:
        r = requests.get(url, timeout = 30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        print("getHTMLText failed")
        return ""

def getStockList(lst, stockURL):
    html = getHTMLText(stockURL)
    soup = BeautifulSoup(html, 'html.parser')
    a = soup.find_all('a')
    for i in a:
        try:
            href = i.attrs['href']
            lst.append(re.findall(r"[s][hz]\d{6}", href)[0])
        except:
            continue

def getStockInfo(lst, stockURL, fpath):
    # 显示进度
    count = 0
    for stock in lst:
        url = stockURL + stock + ".html"
        html = getHTMLText(url)
        try:
            if html == "":
                continue
            infoDict = {}
            soup = BeautifulSoup(html, 'html.parser')
            stockInfo = soup.find('div', attrs={'class': 'stock-bets'})

            name = stockInfo.find_all(attrs = {'class': 'bets-name'})[0]
            infoDict.update({'股票名称' : name.text.split()[0]})

            keyList = stockInfo.find_all('dt')
            valueList = stockInfo.find_all('dd')
            for i in range(len(keyList)):
                key = keyList[i].text
                value = valueList[i].text
                infoDict[key] = value

            with open(fpath, 'a', encoding='utf-8') as f:
                count = count + 1
                print('\r当前速度: {:.2f}%'.format((count * 100) / len(lst)), end='')
                f.write(str(infoDict) + '\n')
        except:
            count = count + 1
            print('\r当前速度: {:.2f}%'.format((count * 100) / len(lst)), end='')
            # traceback.print_exc()
            continue

def main():
    stock_list_url = 'http://quote.eastmoney.com/stocklist.html'
    stock_info_url = 'https://gupiao.baidu.com/stock/'
    output_file = 'H://Stock.txt'
    slist = []
    getStockList(slist, stock_list_url)
    getStockInfo(slist, stock_info_url, output_file)