python urllib2的使用案例
使用urllib2，太强大了
试了下用代理登陆拉取cookie，跳转抓图片......
文档：http://docs.python.org/library/urllib2.html
直接上demo代码了
包括：直接拉取，使用Reuqest(post/get),使用代理，cookie,跳转处理
#!/usr/bin/python
# -*- coding:utf-8 -*-
# urllib2_test.py
# author: wklken
# 2012-03-17 [email protected]
import urllib,urllib2,cookielib,socket
url = "http://www.testurl....."#change yourself
#最简单方式
def use_urllib2():
try:
f = urllib2.urlopen(url, timeout=5).read()
except urllib2.URLError, e:
print e.reason
print len(f)
#使用Request
def get_request():
#可以设置超时
socket.setdefaulttimeout(5)
#可以加入参数 [无参数，使用get，以下这种方式，使用post]
params = {"wd":"a","b":"2"}
#可以加入请求头信息，以便识别
i_headers = {"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1) Gecko/20090624 Firefox/3.5",
"Accept": "text/plain"}
#use post,have some params post to server,if not support ,will throw exception
#req = urllib2.Request(url, data=urllib.urlencode(params), headers=i_headers)
req = urllib2.Request(url, headers=i_headers)
#创建request后，还可以进行其他添加,若是key重复，后者生效
#request.add_header('Accept','application/json')
#可以指定提交方式
#request.get_method = lambda: 'PUT'
try:
page = urllib2.urlopen(req)
print len(page.read())
#like get
#url_params = urllib.urlencode({"a":"1", "b":"2"})
#final_url = url + "?" + url_params
#print final_url
#data = urllib2.urlopen(final_url).read()
#print "Method:get ", len(data)
except urllib2.HTTPError, e:
print"Error Code:", e.code
except urllib2.URLError, e:
print"Error Reason:", e.reason
def use_proxy():
enable_proxy = False
proxy_handler = urllib2.ProxyHandler({"http":"http://proxyurlXXXX.com:8080"})
null_proxy_handler = urllib2.ProxyHandler({})
if enable_proxy:
opener = urllib2.build_opener(proxy_handler, urllib2.HTTPHandler)
else:
opener = urllib2.build_opener(null_proxy_handler, urllib2.HTTPHandler)
#此句设置urllib2的全局opener
urllib2.install_opener(opener)
content = urllib2.urlopen(url).read()
print"proxy len:",len(content)
class NoExceptionCookieProcesser(urllib2.HTTPCookieProcessor):
def http_error_403(self, req, fp, code, msg, hdrs):
return fp
def http_error_400(self, req, fp, code, msg, hdrs):
return fp
def http_error_500(self, req, fp, code, msg, hdrs):
return fp
def hand_cookie():
cookie = cookielib.CookieJar()
#cookie_handler = urllib2.HTTPCookieProcessor(cookie)
#after add error exception handler
cookie_handler = NoExceptionCookieProcesser(cookie)
opener = urllib2.build_opener(cookie_handler, urllib2.HTTPHandler)
url_login = "https://www.yourwebsite/?login"
params = {"username":"user","password":"111111"}
opener.open(url_login, urllib.urlencode(params))
for item in cookie:
print item.name,item.value
#urllib2.install_opener(opener)
#content = urllib2.urlopen(url).read()
#print len(content)
#得到重定向 N 次以后最后页面URL
def get_request_direct():
import httplib
httplib.HTTPConnection.debuglevel = 1
request = urllib2.Request("http://www.google.com")
request.add_header("Accept", "text/html,*/*")
request.add_header("Connection", "Keep-Alive")
opener = urllib2.build_opener()
f = opener.open(request)
print f.url
print f.headers.dict
print len(f.read())
if __name__ == "__main__":
use_urllib2()
get_request()
get_request_direct()
use_proxy()
hand_cookie()
递归的抓取需要的数据：
抓取某个网站下图片
可定义 图片保存路径，最小图片大小域值，遍历深度，是否遍历到外站，抓取并下载图片
#!/usr/bin/python
# -*- coding:utf-8 -*-
# author: wklken
# 2012-03-17 [email protected]
#1实现url解析 #2实现图片下载 #3优化重构
#4多线程 尚未加入
import os,sys,urllib,urllib2,urlparse
from sgmllib import SGMLParser
img = []
class URLLister(SGMLParser):
def reset(self):
SGMLParser.reset(self)
self.urls=[]
self.imgs=[]
def start_a(self, attrs):
href = [ v for k,v in attrs if k=="href"and v.startswith("http")]
if href:
self.urls.extend(href)
def start_img(self, attrs):
src = [ v for k,v in attrs if k=="src"and v.startswith("http") ]
if src:
self.imgs.extend(src)
def get_url_of_page(url, if_img = False):
urls = []
try:
f = urllib2.urlopen(url, timeout=1).read()
url_listen = URLLister()
url_listen.feed(f)
if if_img:
urls.extend(url_listen.imgs)
else:
urls.extend(url_listen.urls)
except urllib2.URLError, e:
print e.reason
return urls
#递归处理页面
def get_page_html(begin_url, depth, ignore_outer, main_site_domain):
#若是设置排除外站 过滤之
if ignore_outer:
ifnot main_site_domain in begin_url:
return
if depth == 1:
urls = get_url_of_page(begin_url, True)
img.extend(urls)
else:
urls = get_url_of_page(begin_url)
if urls:
for url in urls:
get_page_html(url, depth-1)
#下载图片
def download_img(save_path, min_size):
print"download begin..."
for im in img:
filename = im.split("/")[-1]
dist = os.path.join(save_path, filename)
#此方式判断图片的大小太浪费了
#if len(urllib2.urlopen(im).read()) < min_size:
# continue
#这种方式先拉头部，应该好多了，不用再下载一次
connection = urllib2.build_opener().open(urllib2.Request(im))
if int(connection.headers.dict['content-length']) < min_size:
continue
urllib.urlretrieve(im, dist,None)
print"Done: ", filename
print"download end..."
if __name__ == "__main__":
#抓取图片首个页面
url = "http://www.baidu.com/"
#图片保存路径
save_path = os.path.abspath("./downlaod")
ifnot os.path.exists(save_path):
os.mkdir(save_path)
#限制图片最小必须大于此域值 单位 B
min_size = 92
#遍历深度
max_depth = 1
#是否只遍历目标站内，即存在外站是否忽略
ignore_outer = True
main_site_domain = urlparse.urlsplit(url).netloc
get_page_html(url, max_depth, ignore_outer, main_site_domain)
download_img(save_path, min_size)
后续可以优化
1.使用多线程优化下载，目前多层遍历不够速度
2.使用BeautifulSoup写一个版本
3.加入图形界面......