python 简单爬虫

以爬取百度百科与python相关的1000个词条为例
https://baike.baidu.com/item/Python/407313

注：来源于https://www.imooc.com/learn/563

HTML下载器 (html_downloader.py)

from urllib import request

class HtmlDownloader(object):

    def download(self, url):
        if url is None:
            return None

        response = request.urlopen(url)

        if response.getcode() != 200:
            return None

        return response.read().decode("utf-8")

Html解析器（html_parser.py）

from bs4 import BeautifulSoup
import re
from urllib import parse

class HtmlParser(object):
    def _get_new_urls(self, page_url, soup):

        new_urls = set()

        links = soup.find_all('a', href = re.compile(r'/item/'))
        for link in links:
            new_url = link['href']
            new_urls.add(parse.urljoin(page_url, new_url))

        return new_urls

    def _get_new_data(self, page_url, soup):
        res_data = {}

        title_node = soup.find('dd', class_ = "lemmaWgt-lemmaTitle-title").find("h1")
        res_data['title'] = title_node.get_text()

        summary_node = soup.find('div', class_ ="lemma-summary")
        res_data['content'] = summary_node.get_text()

        return res_data

    def parser(self, page_url, html_cont):
        if page_url is None or html_cont is None:
            return
        
        soup = BeautifulSoup(html_cont, 'html.parser', from_encoding = "utf-8")
        new_urls = self._get_new_urls(page_url, soup)
        new_data = self._get_new_data(page_url, soup)

        return new_urls, new_data

url管理器 （url_manager.py）

class UrlManager(object):
    def __init__(self):
        self.new_urls = set()
        self.old_urls = set()

    def add_new_url(self, url):
        if url is None:
            return

        if url not in self.new_urls and url not in self.old_urls:
            self.new_urls.add(url)

    def add_new_urls(self, urls):
        if urls is None or len(urls) == 0:
            return

        for url in urls:
            self.add_new_url(url)

    def has_new_url(self):
        if len(self.new_urls) != 0:
            return len(self.new_urls)
        else:
            return None

    def get_new_url(self):
        new_url = self.new_urls.pop()
        self.old_urls.add(new_url)

        return new_url

html输出器 (html_outputer.py)

class HtmlOutputer(object):
    def __init__(self):
        self.datas = []

    def collect_data(self, data):
        if data is None:
            return

        self.datas.append(data)

    def output_html(self):
       f = open('putput.html', 'w', encoding = "utf-8")

       f.write("")
       f.write("")
       f.write("")

       for data in self.datas:
           f.write("")
           f.write("" % data['url'])
           f.write("" % data['title'])
           f.write("" % data['summary'])
           f.write("")

       f.write("
 %s 
 %s 
 %s 
")
       f.write("")
       f.write("")

       f.close()

调度程序

from baike_apider import html_downloader, html_parser, url_manager, html_outputer
class SpiderMain(object):
    def __init__(self):
        self.urls = url_manager.UrlManager()
        self.download = html_downloader.HtmlDownloader()
        self.parser = html_parser.HtmlParser()
        self.outputer = html_outputer.HtmlOutputer()

    def craw(self, root_url):
        count = 1

        self.urls.add_new_url(root_url)

        while self.urls.has_new_url():
            try:
                new_url = self.urls.get_new_url()
                print('craw %d : %s' % (count, new_url))
                html_cont = self.download.download(new_url)
                new_urls, new_data = self.parser.parser(new_url, html_cont)

                self.urls.add_new_urls(new_urls)
                self.outputer.collect_data(new_data)

                if count >= 2:
                    break
                count = count + 1
            except:
                print('craw fail')
                
        self.outputer.output_html()



if __name__ == '__main__':
    root_url = 'https://baike.baidu.com/item/Python/407313'
    obj_spider = SpiderMain()
    obj_spider.craw(root_url)