使用代理请求拉勾网接口

前面试过使用selenium爬取拉勾网，但是效率特别慢，今天试了一下请求数据传输接口，不需要多线程速度就特别快了。

主要实现原理：
	1.实例化session随机添加代理UA和IP
	2.建立与拉钩网的持续会话
	3.post请求数据传输API

相当于每次重新请求接口之前，重新实例化一个session。提交的form data 里面提交的数据，pn就是页数，kd就是搜索的职位。然后就是我本地的UA和IP都死网上爬取的，具体可见我前面的一篇博客。

还有要注意的地方就是，写入数据，爬取的数据中有的字符写入csv是非法的，所以爬取一个数据就写入一个，其实可以加一个判断条件，是否写入csv是非法的。

代码如下：

import requests
import json
import csv
import time
import random

class LaGouSpider(object):
    def __init__(self):
        self.url = 'https://www.lagou.com/jobs/list_python%E7%88%AC%E8%99%AB?labelWords=&fromSearch=true&suginput='
        self.post_url = 'https://www.lagou.com/jobs/positionAjax.json?city=%E6%AD%A6%E6%B1%89&needAddtionalResult=false'
        self.user_agent = self.read_user_agent()
        self.proxies = self.read_proxies()
        self.creat_file()

    # 读取本地UA
    def read_user_agent(self):
        with open(r'文件名', 'r') as f:
            headers = []
            datas = csv.reader(f)
            for index, data in enumerate(datas):
                headers.append(data)
                if index == 100:
                    break
            return headers

    # 读取本地Proxy
    def read_proxies(self):
        with open(r'文件名', 'r') as f:
            proxies = []
            datas = csv.reader(f)
            for index, data in enumerate(datas):
                proxies.append(data)
            return proxies

    # 创建一个CSV文件用于保存数据
    def creat_file(self):
        with open('数据结果.csv', 'w') as f:
            w = csv.writer(f)
            w.writerow(['companyFullName', 'positionName', 'salary', 'workYear', 'education', 'famousCompany'])

    def get_json(self, page_num=3):
        session = requests.session()
        session.headers = {'User-Agent': '%s' % random.choice(self.user_agent)[0],
                                'Referer': 'https://www.lagou.com/jobs/list_python%E7%88%AC%E8%99%AB?labelWords=&fromSearch=true&suginput='}
        proxy = random.choice(self.proxies)
        session.proxies = {proxy[0]: '{}:{}'.format(proxy[1], proxy[2])}
        session.get(self.url)
        # print(session.proxies, session.cookies)

        data = {'first': 'false',
                'pn': '{}'.format(page_num),
                'kd': 'python爬虫',
                'sid': '2f99d51db5fb43949ed168e0237dbd99'}
        print(data)

        r = session.post(self.post_url, data=data)
        datas = json.loads(r.text)
        return datas

    def parse_data(self, datas):
        try:
            for data in datas['content']['positionResult']['result']:
                positionName = data['positionName']
                companyFullName = data['companyFullName']
                salary = data['salary']
                workYear = data['workYear']
                education = data['education']
                famousCompany = data['famousCompany']
                result = [companyFullName, positionName, salary, workYear, education, famousCompany]
                self.save_data(result)
        except:
            print('页面数据读取错误')
        else:
            print('-' * 100)

    def save_data(self, result):
        try:
            with open('数据结果.csv', 'a') as f:
                w = csv.writer(f)
                w.writerow(result)
        except:
            print(result, 'write failed!')
        else:
            print(result, 'write success!')

    def run(self):
        for i in range(1, 24):
            datas = self.get_json(i)
            self.parse_data(datas)
            time.sleep(random.random())

if __name__ == '__main__':
    lagou = LaGouSpider()
    lagou.run()

实现效果如下：