scrapy+xpath爬取不可描述网站

今天来爬一个让人很有动力的网站，网址就不便放上来了，看看有没有有缘人能得知了
还是先来items.py

import scrapy


class AvmooItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    name=scrapy.Field()
    birthday=scrapy.Field()
    age=scrapy.Field()
    height=scrapy.Field()
    cup=scrapy.Field()
    bust=scrapy.Field()
    waistline=scrapy.Field()
    hipline=scrapy.Field()
    birthplace=scrapy.Field()
    Avatar=scrapy.Field()
    designations=scrapy.Field()
    des_imgs=scrapy.Field()
    des_urls=scrapy.Field()

各位施主从这些字段应该就可以看出来了吧
接下来就是主爬取程序了
spider.py

# -*- coding:utf-8 -*-

import scrapy
from AVMOO.items import AvmooItem
import os
import requests

class AvmooSpider(scrapy.Spider):
    name='AVMOO'
    allowed_domains=['xxx.xx','jp.netcdn.space/digital/video/']#,'xxxx.xx']
    start_urls=['https://xxx.xx/cn/actresses/']
    base_url='https://xxx.xx'
    des_imgs=[]


    def parse(self,response):
        star_urls=response.xpath('//a[@class="avatar-box text-center"]/@href').extract()
        for star_url in star_urls:
            yield scrapy.Request(star_url,callback=self.star_item)
        next_url=self.base_url+response.xpath('//a[@name="nextpage"]/@href').extract()[0]
        if next_url:
            yield scrapy.Request(next_url,callback=self.parse)

    def star_item(self,response):
        item=AvmooItem()
        item['name']=response.xpath('//span[@class="pb-10"]/text()').extract()[0]
        item['age']=response.xpath('//p/text()')[1].extract()
        item['birthday']=response.xpath('//p/text()')[0].extract()
        item['height']=response.xpath('//p/text()')[2].extract()
        item['cup']=response.xpath('//p/text()')[3].extract()
        item['bust']=response.xpath('//p/text()')[4].extract()
        item['waistline']=response.xpath('//p/text()')[5].extract()
        item['hipline']=response.xpath('//p/text()')[6].extract()
        item['birthplace']=response.xpath('//p/text()')[7].extract()
        self.mkdir('/home/shitfly/webspider/AVMOO/avmoo',item['name'])
        f=open(item['name']+'.txt','w+')
        print '写入个人信息'
        self.save_txt(item,f)
        movie_urls=response.xpath('//a[@class="movie-box"]/@href').extract()
        for movie_url in movie_urls:
            yield scrapy.Request(movie_url,meta={'item1':item},callback=self.movie_item)
        next_url=self.base_url+response.xpath('//a[@name="nextpage"]/@href').extract()[0]
        if next_url:
            yield scrapy.Request(next_url,meta={'item1':item},callback=self.star_item2)

    def star_item2(self,response):
        item1=response.meta['item1']
        movie_urls=response.xpath('//a[@class="movie-box"]/@href').extract()
        for movie_url in movie_urls:
            yield scrapy.Request(movie_url,meta={'item1':item1},callback=self.movie_item)
        try:
            next_url=self.base_url+response.xpath('//a[@name="nextpage"]/@href').extract()[0]
        except:
            pass
        if next_url:
            yield scrapy.Request(next_url,meta={'item1':item1},callback=self.star_item2)

    def movie_item(self,response):
        item=AvmooItem()
        item2=response.meta['item1']
        item['name']=item2['name']
        item['designations']=response.xpath('//span[@style="color:#CC0000;"]/text()').extract()[0]
        base_path='/home/shitfly/webspider/AVMOO/avmoo'+'/'+item['name']
        self.mkdir(base_path,item['designations'])
        cover=response.xpath('//a[@class="bigImage"]/img/@src').extract()[0]
        self.des_imgs=[]
        self.des_imgs.append(cover)
        des_imgs=response.xpath('//a[@class="sample-box"]/div[@class="photo-frame"]/img/@src').extract()
        for des_img in des_imgs:
            s=des_img.split('-')
            s[0]=s[0]+str('jp-')
            des_img=''.join(i for i in s)
            self.des_imgs.append(des_img)
        item['des_imgs']=self.des_imgs
        yield item


    def mkdir(self,base_path,title): #title是文件夹名
        title=title.strip()
        isExists=os.path.exists(os.path.join(base_path,title))
        if not isExists:
            print "新建",title,"文件夹"
            os.makedirs(os.path.join(base_path,title))
            os.chdir(os.path.join(base_path,title))
        else:
            print title,'已存在'
            os.chdir(os.path.join(base_path,title))
        return True

    def save_txt(self,item,f):
        it=[]
        inf=['age','birthday','height','cup','bust','waistline','hipline','birthplace']
        for i in inf:
            it.append(item[i])
        for it_e in it:
            it_e=it_e+'\n'
            f.write(it_e.encode('utf-8'))

网址已被和谐，这里希望图片保存到本地，所以还是在磁盘中建立文本文件和文件夹。想要传递字段的话，使用meta={‘item1’:item}，下一个函数接收是item1=response.meta[‘item1’]。

保存图片的话在管道中来进行保存
pipelines.py

import requests

class AvmooPipeline(object):
    def process_item(self, item, spider):
        des_imgs=item['des_imgs']
        path='/home/shitfly/webspider/AVMOO/avmoo'+'/'+item['name']+'/'+item['designations']
        for i in range(len(des_imgs)):
            image=requests.get(des_imgs[i])
            path_i=path+'/'+str(i)+'.jpg'
            f=open(path_i,'wb')
            f.write(image.content)
            f.close()
            print u'正在保存图片：',des_imgs[i]
            print u'图片路径：',path_i
            print u'文件：',path


        return item

需要在settings.py中使用这个管道

ITEM_PIPELINES = {
    'AVMOO.pipelines.AvmooPipeline': 300,
}

来重写一下下载中间件编写您自己的下载器中间件
middlewares.py

import random

class RandomUserAgentMiddleware(object):

    def __init__(self,agents):
        self.agents=agents

    @classmethod
    def from_crawler(cls,crawler):
        return cls(crawler.settings.getlist('USER_AGENTS'))

    @classmethod
    def from_settings(cls, settings):
        return cls(settings.getlist('USER_AGENTS'))

    def process_request(self,request,spider):
        us=random.choice(self.agents)
        print us
        request.headers.setdefault('User-Agent',us)

在settings.py中加入一个USER_AGENTS = […]的列表

DOWNLOADER_MIDDLEWARES = {
'AVMOO.middlewares.RandomUserAgentMiddleware': 543,}
就可以随机使用一个USER_AGENTS了

在settings.py中添加

CONCURRENT_ITEMS=100#默认Item并发数
CONCURRENT_REQUESTS=16#默认Request并发数
CONCURRENT_REQUESTS_PER_DOMAIN = 16#默认每个域名的并发数
CONCURRENT_REQUESTS_PER_IP = 0#每个ip的最大并发数，0表示忽略 

settings

未完待续。。。