Python简单爬虫

简单爬虫实现，主要用到BeautifulSoup,re,urlparse, urllib2库

项目主要结构如下：

• crawler_main.py 项目启动程序
• url_manager.py url管理器
• html_downloader.py html内容下载器
• html_parser.py html解析器
• html_outputer.py html输出器

crawler_main.py(爬虫主程序)

import html_downloader＃html内容下载器
import html_parser
import url_manager
import html_outputer

class SpiderMain(object):
    def __init__(self):
        self.urls = url_manager.UrlManager()
        self.downloader = html_downloader.HtmlDownloader()
        self.parser = html_parser.HtmlParser()
        self.outputer = html_outputer.HtmlOutputer()

    def craw(self, root_url):
        count = 1
        self.urls.add_new_url(root_url)＃将开始url添加到url管理中
        while self.urls.has_new_url():＃查询url管理器中是否还存在新的url
            try:
                new_url = self.urls.get_new_url()#从url管理器中获取新的url
                print 'craw %d : %s' %(count, new_url)
                html_content = self.downloader.download(new_url)#从下载器中获取网页内容
                new_urls, new_data = self.parser.parser(new_url, html_content)＃将网页内容解析成我们想要获取的数据，此处为新的new_urls(链接集合), new_data(字典数据)
                self.urls.add_new_urls(new_urls)#将解析后的new_urls添加到url管理器中
                self.outputer.collect_data(new_data)#将解析后的内容添加到输出器

                if count == 100:
                    break
                count = count + 1
            except BaseException, e:
                print e.message
                print 'craw fail'

        self.outputer.output_html()#最后将内容以网页的内容输出

if __name__=="__main__":
    root_url = "http://baike.baidu.com/item/%E8%9C%98%E8%9B%9B/8135707"
    obj_crawler = SpiderMain()
    obj_crawler.craw(root_url)

url_manager.py(url管理器)

# coding=utf-8
class UrlManager(object):
    '''
    url管理器
    此处采取set(),set中不能添加相同的value
    new_urls 未爬取数据的url集合
    old_urls 已爬取数据的url集合
    '''
    def __init__(self):
        self.new_urls = set()
        self.old_urls = set()

    def add_new_url(self, root_url):
        if root_url is None:
            return
        #添加的url不存在new_urls中且不在old_urls中
        if root_url not in self.new_urls and root_url not in self.old_urls:
            self.new_urls.add(root_url)

    def has_new_url(self):
        return len(self.new_urls) > 0

    def get_new_url(self):
        new_url = self.new_urls.pop()＃从新集合中移除并获取一条url
        self.old_urls.add(new_url)
        return new_url

    def add_new_urls(self, new_urls):
        if new_urls is None or len(new_urls) == 0:
            return
        for url in new_urls:
            self.add_new_url(url)

html_downloader.py(html下载管理器)

import urllib2

'''从指定的url中获取网页内容'''
class HtmlDownloader(object):
    def download(self, new_url):
        if new_url is None:
            return None

        response = urllib2.urlopen(new_url)

        if response.getcode() != 200:
            return None

        return response.read()

html_parser.py (html解析器)

# coding=utf-8
import re
from bs4 import BeautifulSoup
import urlparse

class HtmlParser(object):
    def parser(self, new_url, html_content):
        if new_url is None or html_content is None:
            return

        soup = BeautifulSoup(html_content, 'html.parser', from_encoding='utf-8')#初始化网页解析器
        new_urls = self._get_new_urls(new_url, soup)#从内容中提取爬取链接
        new_data = self._get_new_data(new_url, soup)＃从内容中提取想要的内容

        return new_urls, new_data

    def _get_new_urls(self, new_url, soup):
        new_urls = set()
        links = soup.find_all('a', href=re.compile(r"/item/"))＃通过指定的正则表达式来提取符合条件的标签
        for link in links:
            url = link['href']#获取href标签内容
            new_full_url = urlparse.urljoin(new_url, url)＃通过urlparse.urljoin来拼接完整路径的url
            new_urls.add(new_full_url)

        return new_urls

    '''从网页内容中提取想要的内容'''
    def _get_new_data(self, new_url, soup):
        res_data = {}

        res_data['url'] = new_url
        #< dd class ="lemmaWgt-lemmaTitle-title" > < h1 > 网络爬虫 < / h1 >
        title_node = soup.find('dd', class_="lemmaWgt-lemmaTitle-title").find("h1")#采用class_来过滤，class是关键词
        res_data['title'] = title_node.get_text()＃获取提取到标签的内容

        summary_node = soup.find('div', class_="lemma-summary")
        res_data['summary'] = summary_node.get_text()

        return res_data

html_outputer.py (html输出器)

'''此处采用数组来存储抓取到的数据'''
class HtmlOutputer(object):
    def __init__(self):
        self.datas = []

    def collect_data(self, new_data):
        if new_data is None:
            return
        self.datas.append(new_data)

    '''自定义生成网页来展示抓取到的数据'''
    def output_html(self):
        fout = open('output.html', 'w')

        fout.write('')
        fout.write('')
        fout.write("")
        fout.write('')
        fout.write('')
        fout.write('')

        for data in self.datas:
            fout.write('')
            fout.write("" % data['url'])
            fout.write("" % data['title'].encode('utf-8'))#由于存在中文，需要指定编码格式
            fout.write("" % data['summary'].encode('utf-8'))
            fout.write('')

        fout.write('
%s
%s
%s
')
        fout.write('')
        fout.write('')

        fout.close()