使用Python爬取京东商品晒单图

本文我们将尝试下载如下所示的京东商品晒单图。

来到京东首页，在搜索框中输入关键字，比如：手机。打开浏览器的开发者工具(Ctrl+Shift+I)，切换到Network选项。

点击按钮搜索之前输入的关键词，可以看到有如下请求：

请求带到服务器的参数主要有：

keyword: 手机
enc: utf-8

响应内容为一个HTML文件，包含了搜索到的商品的URL。

访问搜索到的某个商品，同样打开浏览器的开发者工具，切换到Network选项，可以看到有如下请求(没有看到的话刷新一下页面)：

返回的数据包含了评论相关的数据，晒单图也在其中。

至此，我们将整个过程分析完毕，用代码实现上述过程即可。

import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import re
import os
import json


def parse_goods_url(keyword):
    headers = {
        "User-Agent":
        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36"
    }
    params = {"keyword": keyword, "enc": "utf-8"}
    base_url = "https://search.jd.com/Search"
    response = requests.get(base_url, params=params, headers=headers)
    soup = BeautifulSoup(response.text, "lxml")
    goods_url_list = soup.select("#J_goodsList .gl-item .p-name a")
    for good in goods_url_list:
        yield urljoin("https://item.jd.com", good.get("href"))


def get_good_comment_image_url(good_url, page_num=1):
    headers = {
        "User-Agent":
        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36",
        "Referer":
        good_url
    }
    productId = re.findall("(\d+)", good_url)[0]
    base_url = "https://sclub.jd.com/comment/productPageComments.action"
    for page in range(page_num):
        params = {
            "callback": "fetchJSON_comment98vv3487",
            "productId": productId,
            "score": 0,
            "sortType": 5,
            "page": page,
            "pageSize": 10,
            "isShadowSku": 0,
            "fold": 1
        }
        response = requests.get(base_url, params=params, headers=headers)
        origin_data = re.findall("(\{.*\})", response.text)[0]
        comments = json.loads(origin_data)["comments"]
        for comment in comments:
            for image in comment.get("images"):
                yield urljoin(base_url, image.get("imgUrl"))


def download_image(image_url, good_url):
    file_name = image_url.split('/')[-1]
    fold_name = re.findall("(\d+)", good_url)[0]
    if not os.path.exists(fold_name):
        os.makedirs(fold_name)
    headers = {
        "User-Agent":
        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36",
        "Referer":
        good_url
    }
    response = requests.get(image_url, headers=headers)
    with open("{}/{}".format(fold_name, file_name), "wb") as f_obj:
        f_obj.write(response.content)


if __name__ == '__main__':
    keyword = "手机"
    for url in parse_goods_url(keyword):
        print("商品：{}".format(url))
        for image_url in get_good_comment_image_url(url):
            print("下载：{}".format(image_url))
            download_image(image_url, url)

运行结果如下：