爬虫--汽车之家

spider的代码：

# -*- coding: utf-8 -*-
import scrapy
from AD.items import AdItem


class AdImgSpider(scrapy.Spider):
    name = 'AD_img'
    allowed_domains = ['car.autohome.com.cn']
    start_urls = ['https://car.autohome.com.cn/pic/brand-33.html#pvareaid=2042194']

    def parse(self, response):
        uiboxs = response.xpath("//div[@class='uibox']")
        for uibox in uiboxs:
            title = uibox.xpath(".//span[@class='fn-left']/a/@title").get()
            # print(title)
            urls = uibox.xpath(".//ul/li/a/img/@src").getall()
            # for url in urls:
            #     url = response.urljoin(url)
                # print(url)
            urls = list(map(lambda url: response.urljoin(url), urls))
            item = AdItem(title=title, urls=urls)
            yield item

pipelines的代码：

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
import os
from urllib import request
from scrapy.pipelines.images import ImagesPipeline
from AD import settings


class AdPipeline(object):
    def __init__(self):
        self.path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "images")
        if not os.path.exists(self.path):
            os.mkdir(self.path)

    def process_item(self, item, spider):
        title = item["title"]
        urls = item["urls"]

        title_path = os.path.join(self.path, title)
        if not os.path.exists(title_path):
            os.mkdir(title_path)

        for url in urls:
            image_name = url.split("_")[-1]
            request.urlretrieve(url, os.path.join(title_path, image_name))

        return item