爬虫--人人网-简易爬取3

from urllib import  request,parse
from urllib.error import URLError,HTTPError

#导入包  ，保存cookie
from http import cookiejar


class Session (object):
    def __init__(self):
        # 通过对象保存cookie
        cookie = cookiejar.CookieJar()
        # handler 对应着一个操作
        handler = request.HTTPCookieProcessor(cookie)
        # opener 遇到有cookie的response的时候，调用handler内的一个函数 存储cookie到 objec中
        self.opener = request.build_opener(handler)

    def get(self ,url, headers=None, opener=None):
        return get(url, headers=headers, opener=opener)

    def post(self ,url, form, headers=None, opener=None):
        return post(url, form, headers=headers, opener=opener)


def get(url, headers=None, opener = None):
    return urlrequsets(url, headers=headers, opener = opener)

def post(url, form, headers=None, opener = None):
    return urlrequsets(url, form, headers=headers, opener=opener)
import json
# 1.传入url
# 2.user_agent
# 3.geaders
# 4.定义Request
# 5.urlopen
# 6.返回byte数组
def urlrequsets(url, form = None,headers=None, opener = None):

    user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36'
    if headers ==None:
        headers = {
            'User-Agent' : user_agent
        }
    html_byte = b''
    try:
        #
    # 判断
        if form:
            # POST
            #转str
            print(form)
            form_str = parse.urlencode(form)
            print(form_str)

            # 转bytes
            form_bytes = form_str.encode('utf-8')
            req = request.Request(url, data=form_bytes, headers=headers)
        else:
            # GET
            req = request.Request(url,  headers=headers)
        if opener:
            response = opener.open(req)
        else:
            response = request.urlopen(req)
        # print(999)
        # print(response)
        html_byte = response.read()
        print(html_byte)
    except HTTPError as h:
        print(h)
    except URLError as u:
        print(u)


    return html_byte
if __name__ == '__main__':
    # url = 'http://fanyi.baidu.com'
    # form = {
    #     'kw' :'你好'
    # }
    # # print(111)
    # html_byte = urlrequsets(url)
    # # html_byte = json.loads(html_byte)
    # print(html_byte)

    url = 'http://www.baidu.com'
    html_byte = get(url)
    print(html_byte)
    # url='http://fanyi.baidu.com/sug'
    # data ={
    #     'kw':'哈哈'
    # }
    # a= post(url,data)
    # print(a)

调用上面的内容 ，

from tuozhan_all import Session,post,get

import json




# 1.url
url = 'http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=2018722359875'
# 2. form
form = {
    'email': '17600015762',
    'icode': '',
    'origURL': 'http://www.renren.com/home',
    'domain': 'renren.com',
    'key_id': '1',
    'captcha_type': 'web_login',
    'password': '000e2c3c99f8a73a61287bcaaa16d53f11dea4f308438ff687eb64c557a14962',
    'rkey': 'ccfaa6b14a7da2899fccd0a15cbd7b13',
    'f': 'http%3A%2F%2Fwww.renren.com%2F966927992',
}


s = Session()
html_byte = s.post(url, form)
print(html_byte)
#html_bytes = post(url, form=form)
# 打印结果
#print(html_bytes)
# 通过json获取一个字典类型
res_dict = json.loads(html_byte.decode('utf-8'))

home_url = res_dict['homeUrl']

# 访问页面
html_byte = get(home_url)
print(html_byte)