pyspider创建淘女郎图片爬虫任务-源码解析

源码：
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2016-03-29 10:59:36
# Project: taonvlang

from pyspider.libs.base_handler import *

DIR_PATH = 'E:/taonvlang'


class Handler(BaseHandler):
    crawl_config = {
    }

def __init__(self):
self.base_url = 'https://mm.taobao.com/json/request_top_list.htm?page='
self.pages= 10
self.deal = Deal()

    @every(minutes=24 * 60)
def on_start(self):
#爬取多页
for ipage in range(1, self.pages):
            url = self.base_url + str(ipage)
print url
self.crawl(url, callback=self.index_page, validate_cert=False)

    @config(age=10 * 24 * 60 * 60)
def index_page(self, response):
for each in response.doc('.lady-avatar').items():
self.crawl(each.attr.href, callback=self.detail_page, validate_cert=False)

    @config(priority=2)
def detail_page(self, response):
        name = response.doc('.mm-p-model-info-left-top dd > a').text()
        dir_path = self.deal.mkDir(name)
#先抓大再抓小
brief = response.doc('.mm-aixiu-content').text()
        imgs = response.doc('.mm-aixiu-content img').items()
        count = 1
self.deal.saveBrief(brief, dir_path, name)
for img in imgs:
            url = img.attr.src
if url:
print url
                count += 1
extension = self.deal.getExtension(url)
                file_name = name + str(count) + '.' + extension
self.crawl(img.attr.src, callback=self.save_img, save={'dir_path': dir_path, 'file_name': file_name}, validate_cert=False)

return {
"name":name,
"url": response.url,
"title": response.doc('title').text(),
        }

#抓到图片后保存
def save_img(self, response):
        content = response.content
        dir_path = response.save['dir_path']
        file_name = response.save['file_name']
        file_path = dir_path + '/' + file_name
self.deal.saveImg(content, file_path)

import os

class Deal:
def __init__(self):
self.path = DIR_PATH
if not self.path.endswith('/'):
self.path = self.path + '/'
if not os.path.exists(self.path):
            os.makedirs(self.path)

def mkDir(self, path):
        path = path.strip()
        dir_path = self.path + path
        exists = os.path.exists(dir_path)
if not exists:
            os.makedirs(dir_path)
return dir_path
else:
return dir_path

def saveImg(self, content, path):
        f = open(path, 'wb')
        f.write(content)
        f.close()

def saveBrief(self, content, dir_path, name):
        file_name = dir_path + "/" + name + ".txt"
f = open(file_name, "w+")
        f.write(content.encode('utf-8'))

def getExtension(self, url):
extension = url.split('.')[-1]
return extension

源码参考：http://cuiqingcai.com/2652.html

相比原来源码优化了一个步骤，因此更简单点。