【用Python写爬虫】获取html的方法【二】：使用pycurl

# Pycurl参考地址：http://pycurl.sourceforge.net/

# Pycurl下载地址：http://pycurl.sourceforge.net/download/pycurl-7.18.1.tar.gz

# -*-coding: UTF-8 -*-

importpycurl

importStringIO

 

defgetURLContent_pycurl(url):   

    c = pycurl.Curl()

    c.setopt(pycurl.URL,url)

    b = StringIO.StringIO()

    c.setopt(pycurl.WRITEFUNCTION, b.write)

    c.setopt(pycurl.FOLLOWLOCATION, 1)

    c.setopt(pycurl.MAXREDIRS, 5)

    # 代理

    #c.setopt(pycurl.PROXY, 'http:// 11.11.11 .11:8080')

    #c.setopt(pycurl.PROXYUSERPWD, 'aaa:aaa')

    c.perform()

    returnb.getvalue()

 

url = 'http://blog.csdn.net'

content =getURLContent_pycurl(url)

print content