在当当和亚马逊中搜书并输出最低价格

这两天想买几本关于Python的书，自然是到各网上书店里找，比较哪家最便宜的下手了。可是发现很麻烦，需要在每个网站里每本书都要搜一遍，搜完还得计个总价格，看看谁家便宜。所以想到了用Python的爬虫技术，做一个工具，到各网上书店里找书并计算总价。

京东的搜索很烂，结果是一大堆无关的东西，未找到好方法解决，只好先放弃。目前实现了当当网和亚马逊搜书并找出最低价和各书的地址，将其保存在results.txt中，并显示最低总价。

最好是做成一个web页面，可以接受输入书名，并且在页面中直观的显示各网站书的总价，还要能一键放入购物书。

代码如下：

# -*- coding:utf-8 -*-
"""在当当和亚马逊中找书，输出最低价格"""
import requests, datetime, threading
from urllib.request import quote
from lxml import etree

books = ('流畅的python', 'Python编程快速上手 让繁琐工作自动化', '编写高质量Python代码的59个有效方法')


def d(book, book_ifos):
    """当当网"""
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3'}
    d_url = 'http://search.dangdang.com/?key={}&act=input&sort_type=sort_xlowprice_asc#J_tab'
    search_url = d_url.format(quote(book, encoding='gbk'))

    r = requests.get(search_url, headers=headers)
    root = etree.HTML(r.text)
    results = root.xpath('//li[starts-with(@class,"line")]')
    """若是有results，则找到书了"""
    if results:
        book_sub = book.lower().split(' ')
        for result in results:
            title = result.xpath('a')[0].attrib['title'].strip()
            """判断书名中是否含有旧书，有则跳过"""
            if '旧书' in title:
                continue
            """书名按空格折分，并在title中匹配，全匹配才是找对书"""
            hit = False
            for s in book_sub:
                if s in title.lower():
                    hit = True
                else:
                    hit = False
                    continue
            """取得价格和地址，添加到book_ifos中"""
            if hit:
                a = result.xpath('p/span[@class="search_now_price"]')
                if len(a) != 0:
                    price = float(a[0].text[1:])
                else:
                    continue
                url = result.xpath('a')[0].attrib['href']
                book_ifos['dangdang'].append({'title': title, 'price': price, 'url': url})
                break
    else:
        book_ifos['dangdang'].append({'title': book, 'price': 0, 'url': ''})


def z(book, book_ifos):
    """亚马逊"""
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3'}
    z_url = 'https://www.amazon.cn/s/ref=nb_sb_ss_ime_c_1_5?__mk_zh_CN=%E4%BA%9A%E9%A9%AC%E9%80%8A%E7%BD%91%E7%AB%99&url=search-alias%3Daps&field-keywords={}'

    search_url = z_url.format(book)
    r = requests.get(search_url, headers=headers)
    r.encoding = 'utf-8'
    root = etree.HTML(r.text)
    results = root.xpath('//li[starts-with(@id,"result_")]')
    """若是有results，则找到书了"""
    if results:
        book_sub = book.lower().split(' ')
        book_ifo = []
        for result in results:
            a = result.xpath('div/div/div/a/h2/..')[0]
            title = a.attrib['title'].strip()
            """书名按空格折分，并在title中匹配，全匹配才是找对书"""
            hit = False
            for s in book_sub:
                if s in title.lower():
                    hit = True
                else:
                    hit = False
                    continue
            if hit:
                price_str = result.xpath('div/div/a/span')[1].text
                """
                获取到的价格为：￥222.222，所以只提取数字部分，并转为float
                若是电子书，则取不到价格，跳过
                """
                if price_str:
                    price = float(price_str[1:])
                else:
                    continue
                url = a.attrib['href']
                """获取最低价格"""
                if len(book_ifo) == 0:
                    book_ifo = [title, price, url]
                elif book_ifo[1] > price:
                    book_ifo = [title, price, url]
        book_ifos['Amazon'].append({'title': book_ifo[0], 'price': book_ifo[1], 'url': book_ifo[2]})
    else:
        book_ifos['Amazon'].append({'title': book, 'price': 0, 'url': ''})


if __name__ == '__main__':
    start_time = datetime.datetime.now()
    book_ifos = {'dangdang': [], 'Amazon': []}
    threads = []

    for book in books:
        """当当网价格查询"""
        t = threading.Thread(target=d, args=(book, book_ifos))
        t.start()
        threads.append(t)
        """亚马逊网价格查询"""
        t = threading.Thread(target=z, args=(book, book_ifos))
        t.start()
        threads.append(t)
    """等待线程运行结束"""
    for t in threads:
        t.join()
    """统计各网站的总价格"""
    for site in book_ifos:
        total_price = 0.0
        for book in book_ifos[site]:
            total_price += book['price']
        print(site, '\t', round(total_price, 2))

    print('spend time:', str(datetime.datetime.now() - start_time)[:10])
    with open('results.txt', 'w') as f:
        f.write(str(book_ifos))

---

如果本文对您有帮助，请给我留个言。谢谢！