完整代码
# encoding:UTF-8
# from bs4 import BeautifulSoup
import urlparse
import urllib2
import re
import robotparser
import datetime
import time
import itertools
import Queue # 同步的、线程安全的队列类
import lxml.html
import lxml.cssselect
import csv
def crawl_sitemap(url, scrape_callback=None):
"""
1.通过robots文件记录的链接爬虫
:param url:
:return:如果有回调函数，返回回调结果
"""
sitemap = urllib2.urlopen(url).read()
links = re.findall('
if scrape_callback:
for link in links:
html = urllib2.urlopen(link).read()
scrape_callback(link, html)
def crawl_id(url, scrape_callback=None):
"""
2.通过ID爬虫,连续发生多次下载错误才会退出
:param url:含有ID的链接的公共部分,没有/
:return:
"""
max_error = 5
num_error = 0
throttle = Throttle(5)
for page in itertools.count(1): # 迭代器，从1开始
link = url + ("/-%d" % page)
html = urllib2.urlopen(link).read()
if html is None:
num_error += 1
if num_error == max_error:
break
else: # 网页存在
throttle.wait(link)
scrape_callback(link, html)
num_error = 0
def link_crawler(seed_url, link_regex=None, delay=-1, max_depth=1, max_urls=-1,
headers=None, user_agent="wswp", proxy=None, num_retries=2, scrape_callback=None):
"""
3.通过链接爬虫,深度优先，禁用某些功能可将其参数设为负数
待爬虫队列存在，逐一判断robots访问权限，在等待一定时间后进行下载，
并根据访问深度决定是否继续进行访问。如继续，根据正则表达式匹配获
取链接集，逐一规范化后，若某链接没有被访问过，且域名和种子网址域名相同，
则归入待爬虫队列。每完成一次访问，链接总数+1
:param seed_url:种子链接
:param link_regex:目标链接识别正则表达式
:param user_agent:用户代理
:return:爬虫结果
"""
crawl_queue = Queue.deque([seed_url])
seen = {seed_url: 0}
num_urls = 0
rp = get_robots(seed_url)
throttle = Throttle(delay)
headers = headers or {}
if user_agent:
headers['User-agent'] = user_agent
while crawl_queue:
url = crawl_queue.pop()
depth = seen[url]
if rp.can_fetch(user_agent, url):
throttle.wait(url)
html = download(url, headers, proxy=proxy, num_retries=num_retries)
links = []
if scrape_callback:
# links.extend(scrape_callback(url, html) or [])
scrape_callback(url, html)
if depth != max_depth:
# 获取链接
if link_regex:
links.extend(link for link in get_links(html) if re.match(link_regex, link))
# 如果没有被访问过，且域名相同，归入连接诶队列
for link in links:
link = normalize(seed_url, link)
if link not in seen:
seen[link] = depth + 1
if same_domain(seed_url, link):
crawl_queue.append(link)
num_urls += 1
if num_urls == max_urls:
break
else:
print("Blocked by robots.txt:", url)
class Throttle:
"""
在两次下载之间添加时间延迟
"""
def __init__(self, delay):
self.delay = delay # 延迟多长时间
self.domains = {} # 字典记录域名最后一次被访问的时间地图
def wait(self, url):
"""
功能：页面休眠
urlparse将url（http：//开头）解析成组件
组件：协议(scheme)、位置(netloc)、路径(path)、可选参数(parameters)、查询(query)、片段(fragment)
:param url:
:return:
"""
domain = urlparse.urlparse(url).netloc
last_accessed = self.domains.get(domain)
if self.delay > 0 and last_accessed is not None:
sleep_secs = self.delay - (datetime.datetime.now() - last_accessed).seconds
if sleep_secs > 0:
time.sleep(sleep_secs)
self.domains[domain] = datetime.datetime.now()
class ScrapeCallback:
def __init__(self):
self.writer = csv.writer(open('countries.csv', 'w'))
self.fields = ('area', 'population', 'iso', 'country', 'capital',
'continent', 'tld', 'currency_code', 'currency_name',
'phone', 'postal_code_format', 'postal_code_regex', 'languages')
self.writer.writerow(self.fields)
def __call__(self, url, html):
"""
:param url:判断是否是目标链接
:param html:下载数据的页面
:return:
"""
if re.search('/view/', url):
tree = lxml.html.fromstring(html)
row = []
for field in self.fields:
row.append(tree.cssselect('table>tr#places_{}__row>td.w2p_fw'.format(field))[0].text_content())
self.writer.writerow(row)
def download(url, headers, proxy, num_retries, data=None):
"""
设置一般的请求后，根据爬虫代理参数选择是否使用特定处理器来获取链接
若遇到50X网页暂时无法访问的情况，尝试多次后无果则退出
:param url:链接
:param user_agent:用户代理
:param proxy:协议，ip端口
:param num_retries:出错是尝试访问多少次
:return: 整个网页的源代码
"""
print("Downloading:", url)
request = urllib2.Request(url, data, headers)
opener = urllib2.build_opener() # 用特定处理器来获取urls
if proxy:
proxy_params = {urlparse.urlparse(url).scheme: proxy}
opener.add_handler(urllib2.ProxyHandler(proxy_params))
try:
html = urllib2.urlopen(request).read()
# # 数据获取方式1：正则表达式（C、快、使用困难、灵活性差）
# result = re.findall('
# if result:
# print(result[1])
# # 数据获取方式2：通过beautifulsoup（Python、慢、安装简单）
# soup = BeautifulSoup(html, 'html.parser')
# tr = soup.find(attrs={'id': 'places_area__row'})
# if tr:
# td = tr.find(attrs={'class': 'w2p_fw'})
# area = td.text
# print(area)
# # 数据获取方式3：通过lxml（快、大量数据抓取效果更明显、安装相对困难）
# tree = lxml.html.fromstring(html)
# td = tree.cssselect('tr#places_neighbours__row > td.w2p_fw')
# if td:
# area = td[0].text_content()
# print(area)
except urllib2.URLError as e:
print("Download error:", e.reason)
html = None
if num_retries > 0:
if hasattr(e, 'code') and 500 <= e.code <= 600:
return download(url, headers, proxy, num_retries - 1, data)
return html
def get_links(html):
"""
提取网页中的所有链接
:param html:
:return:
"""
webpage_regex = re.compile('
re.IGNORECASE) # re.compile()函数将字符串形式的正则表达式转换成模式
return webpage_regex.findall(html)
def get_robots(url):
"""
:param url:
:return: 包含robots信息的对象
"""
rp = robotparser.RobotFileParser()
rp.set_url(urlparse.urljoin(url, '/robots.txt'))
rp.read()
return rp
def normalize(seed_url, link):
"""
链接规范化，相对路径转化成绝对路径
:param seed_link:
:param link:
:return:
"""
link, _ = urlparse.urldefrag(link) # 去掉碎部（链接#后的部分）
return urlparse.urljoin(seed_url, link)
def same_domain(url1, url2):
"""
两个链接的域名相同，为True
:param url1:
:param url2:
:return:
"""
return urlparse.urlparse(url1).netloc == urlparse.urlparse(url2).netloc
def main():
## 1.通过robots文件记录的链接爬虫
# crawl_sitemap('http://example.webscraping.com/sitemap.xml', scrape_callback=ScrapeCallback())
# # 2.通过ID爬虫
# crawl_id('http://example.webscraping.com/places/default/view',scrape_callback=ScrapeCallback())
# 3.通过链接爬虫
link_crawler('http://example.webscraping.com', '/places/default/(view|index)',
delay=0, num_retries=5, max_depth=2, user_agent='GoodCrawler', scrape_callback=ScrapeCallback())
main()