Python爬虫实战(十)：爬取Linux公社资源站的所有电子资源

#coding=utf-8
import re
import requests
from tenacity import retry, stop_after_attempt

@retry(stop=stop_after_attempt(3))
def get_html(url):
    '''获取页面源代码'''
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0'}
    page = requests.get(url,headers = headers)
    html = page.text
    return html

def get_real_url(url,reg):
    '''获取真实的页面地址'''
    mainurl = 'http://linux.linuxidc.com/'
    html = get_html(url)
    items = re.findall(reg,html)
    for item in items:
        realurl = '{}{}'.format(mainurl,item)
        yield realurl

def get_year_url():
    '''获取**年的页面地址'''
    url = 'http://linux.linuxidc.com/index.php'
    reg = re.compile(r'href="(.*?)">\d+年资料')
    year_urls = get_real_url(url,reg)
    for year_url in year_urls:
        yield year_url

def get_month_url():
    '''获取**月的页面地址'''
    reg = re.compile(r'href="(.*?)">\d+月')
    year_urls = get_year_url()
    for year_url in year_urls:
        month_urls =  get_real_url(year_url,reg)
        for month_url in month_urls:
            yield month_url

def get_day_url():
    '''获取**日的页面地址'''
    reg = re.compile(r'href="(.*?)">\d+日')
    month_urls = get_month_url()
    for month_url in month_urls:
        day_urls =  get_real_url(month_url,reg)
        for day_url in day_urls:
            yield day_url

def get_books_urls(urls):
    '''获取资料名称及下载页'''
    for url in urls:
        reg = re.compile(r'href="(index.*?)">(.*?)

') html = get_html(url) items = re.findall(reg,html) for item in items: yield item def get_other_url(url): '''获取其他链接的资料''' reg = re.compile(r'href="(index.*?)">.*?

') all_urls = get_real_url(url,reg) for all_url in all_urls: yield all_url def print_book_url(book,book_url): '''打印可下载的书籍及链接''' url = 'http://linux.linuxidc.com/' book = book.lower() item = book_url if len(item) == 2: bookname = item[1].lower() if book in bookname: print ('\n'+ item[1]) print ("资料下载链接:") dlurl = '{}{}'.format(url,item[0]) reg = re.compile(r'href="(linuxconf/download.php.*?)">.*?

') download_urls = get_real_url(dlurl,reg) for download_url in download_urls: print (download_url) def get_download_url(book): url_2011 = 'http://linux.linuxidc.com/index.php?folder=MjAxMcTq18rBzw==' all_urls = get_other_url(url_2011) books_urls = get_books_urls(all_urls) for book_url in books_urls: print_book_url (book,book_url) day_urls = get_day_url() books_urls = get_books_urls(day_urls) for book_url in books_urls: print_book_url (book,book_url) if __name__=='__main__': book = input ("请输入资料名称:") get_download_url(book)

说明：待添加。