熊猫学猿--python爬虫某商城所有商品的名称，价格，下载详情图片

用python爬了1000多个商品，把商品标题及价格整理进Excel列表，多线程下载商品详情图5万多张。

import sys
import urllib
import json
import re
import time
import os
import json
import requests
from bs4 import BeautifulSoup
import threading
import socket
import concurrent.futures
import xlwt
socket.setdefaulttimeout(30)
def getpic(img, path, j):
    print(path)
    urllib.request.urlretrieve(img, '%s/%s.jpg' % (path, j))
    try:
        urllib.request.urlretrieve(img, '%s/%s.jpg' % (path, j))
    except socket.timeout:
        count = 1
        while count <= 5:
            try:
                urllib.request.urlretrieve(img, '%s/%s.jpg' % (path, j))
                break
            except socket.timeout:
                err_info = 'Reloading for %d time' % count if count == 1 else 'Reloading for %d times' % count
                print(err_info)
                count += 1
        if count > 5:
            print("downloading picture fialed!")




f = xlwt.Workbook()
sheet1 = f.add_sheet('商品', cell_overwrite_ok=True)
f.save('goods.xls')
headers = {}
headers[
    "User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"
for page in range(2):
    print(page)
    data = {"token": "264FE0", "t": 1534401238, "cate_id": 0, "cate_id_1": 0, "zhe": 0,
            "sort": 0,
            "order": 'Desc', "page": page, "limit": 20, "tokenId": "90708F15467", "c": 0}
    url = "http://www.test.shop.com/goods_list"
    resp = requests.post(url=url, data=data, headers=headers)
    goods = resp.text
    goods = json.loads(goods)
    goods = goods['data']
    threads = []
    thnum = 0
    for index in range(len(goods)):
        headers = {}
        row = {}
        headers[
            "User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"
        url = goods[index]['url']
        print(url)
        row[0] = goods[index]['goods_id']
        row[1] = goods[index]['goods_name']
        row[2] = goods[index]['retail_price']
        row[3] = goods[index]['b2b_price']
        row[4] = goods[index]['enjoy_price']

        request = urllib.request.Request(url=url, headers=headers)
        response = urllib.request.urlopen(request)
        html = response.read()
        html = BeautifulSoup(html, "html5lib", from_encoding='utf-8')
        cmplist = html.select('#imageMenu ul li img ')
        cntlist = html.select('.con0 img ')
        # title = html.select('.sel_bar h2 ')[0].get_text()
        # oldprice = html.select('.zx_lsj ')[0].get_text()
        # price = html.select('.zx_cgj')[0].get_text()
        # litleprice = html.select('.zx_cgj .text_obscure')[0].get_text()
        # if(price.find('litleprice')):
        #     price=price.replace(litleprice,"."+litleprice)
        # price2 = html.select('#enjoy_price')[0].get_text()
        # litleprice2 = html.select('#enjoy_price .text_obscure')[0].get_text()
        # if(price2.find('litleprice2')):
        #     price2=price2.replace(litleprice2,"."+litleprice2)
        path = './goods/' + goods[index]['goods_id']
        if not os.path.exists(path):
            os.makedirs(path)
        row[5] = '%s/%s.jpg' % (path, "defult_img")
        if not os.path.exists( '%s/%s.jpg' % (path, "defult_img")):
           getpic(goods[index]['default_image'], path, "defult_img")
        info = '
'
        if len(cntlist) > 0:
            for j in range(len(cntlist)):
                img = cntlist[j]['src']
                info += ""
                if (img.find('http://www.test.shop.com') == -1):
                    img = "http://www.test.shop.com" + cntlist[j]['src']
                if (img):
                    if not os.path.exists('%s/%s.jpg' % (path, j)):
                        threads.append(threading.Thread(target=getpic, args=(img, path, j)))
        info += '
'
        row[6] = info
        for i in range(0, len(row)):
            sheet1.write(index, i, row[i])
        f.save('goods.xls')
    if len(threads) > 0:
        for t in threads:
            t.setDaemon(True)
            t.start()
            t.join()
    print("finish")