selenium爬取拉勾网职位招聘信息

selenium爬取拉勾网职位招聘信息

# encoding:utf-8

from selenium import webdriver
from lxml import etree
from pyquery import PyQuery as pq
# 引入显式等待
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

from selenium.webdriver.common.by import By

import time


class LagouSpider(object):
    driver_path = r'D:\chromedriver\chromedriver.exe'

    def __init__(self):
        self.driver = webdriver.Chrome(executable_path=self.driver_path)
        self.url = 'https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput='
        self.positions = []

    def run(self):
        '''
        程序开始函数
        :return:
        '''
        self.driver.get(self.url)
        while True:
            source = self.driver.page_source
            self.parse_list_page(source)
            # 显示等待，直到指定的元素被加载成功，若没有加载成功超过指定的时间后会抛出异常
            WebDriverWait(driver=self.driver, timeout=10).until(
                EC.presence_of_all_elements_located((By.XPATH, "//div[@class='pager_container']/span[last()]"))
            )
            next_btn = self.driver.find_element_by_xpath("//div[@class='pager_container']/span[last()]")
            # 模拟点击下一页， 并判断，若有下一页点击没有的话，停止
            if 'pager_next_disabled' in next_btn.get_attribute("class"):
                break
            else:
                next_btn.click()
            time.sleep(2)

    def parse_list_page(self, source):
        '''
        解析列表详情页面
        :param source: 列表详情页面源代码
        :return:
        '''
        html = etree.HTML(source)
        links = html.xpath('//a[@class="position_link"]/@href')
        for link in links:
            self.request_detail_page(link)

    def request_detail_page(self, url):
        '''
        请求工作详情页面
        :param url: 工作详情页面的url
        :return:
        '''
        # self.driver.get(url)
        self.driver.execute_script("window.open('%s')" % url)
        self.driver.switch_to_window(self.driver.window_handles[1])
        WebDriverWait(driver=self.driver, timeout=10).until(
            EC.presence_of_all_elements_located((By.CLASS_NAME, 'company'))
        )
        source = self.driver.page_source
        self.parse_detail_page(source)
        time.sleep(3)
        self.driver.close()
        self.driver.switch_to_window(self.driver.window_handles[0])

    def parse_detail_page(self, source):
        '''
        解析工作详情页面
        :param source:工作详情页面的源代码
        :return:
        '''
        html = pq(source)
        company = html('.company').text()
        print(company)
        position = {
            'company': company
        }
        self.positions.append(position)


if __name__ == '__main__':
    spider = LagouSpider()
    spider.run()