赶集网招聘信息数据分析与可视化

一、使用python编写爬虫——使用urllib库下载网页，使用xpath解析

提取页面中我们所需的信息，公司名称、薪资待遇、工作地点、工作经验、最低学历、招聘人数、公司规模，然后结构化输出到txt文件中，以备之后的分析使用。

#! /usr/bin/env python3
# -*- coding:utf-8 -*-


from urllib.request import Request
import string
import urllib
import lxml.html


def getHtml(url):
    headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36'}
    url = urllib.parse.quote(url, safe=string.printable)
    req = Request(url, headers=headers)
    page = urllib.request.urlopen(req)
    html = page.read()
    return html


def getImg(html):
    flag = 0
    tree = lxml.html.fromstring(html)
    src0 = tree.xpath('//div[@class="s-btop s-bb1 "]//p/text()')
    src1 = tree.xpath('//div[@class="s-butt s-bb1"]//li[1]/text()')
    src2 = tree.xpath('//div[@class="s-butt s-bb1"]//li[2]/text()')
    src3 = tree.xpath('//div[@class="s-butt s-bb1"]//li[3]/text()')
    src4 = tree.xpath('//div[@class="s-butt s-bb1"]//li[4]/text()')
    src5 = tree.xpath('//div[@class="s-butt s-bb1"]//li[5]/text()')
    src6 = tree.xpath('//div[@class="s-butt s-bb1"]//li[6]/text()')
    while flag < len(src0):
        with open('ganji_info.txt', 'ab') as jb:
            jb.write(src0[flag].encode('utf8'))
            jb.write("\t".encode('utf8'))
            jb.write(src1[flag].strip().encode('utf8'))
            jb.write("\t".encode('utf8'))
            jb.write(src2[flag].encode('utf8'))
            jb.write("\t".encode('utf8'))
            jb.write(src3[flag].encode('utf8'))
            jb.write("\t".encode('utf8'))
            jb.write(src4[flag].encode('utf8'))
            jb.write("\t".encode('utf8'))
            jb.write(src5[flag].encode('utf8'))
            jb.write("\t".encode('utf8'))
            jb.write(src6[flag].encode('utf8'))
            jb.write("\r\n".encode('utf8'))
        flag += 1


if __name__ == '__main__':
    str = "http://bj.ganji.com/zpruanjianhulianwang/o{}/"
    pn = 1
    while pn <= 300:  # 爬取300页
        print('爬取第', pn, '页')
        url = str.format(pn)
        html = getHtml(url)
        getImg(html)
        pn += 1
    print('爬取完成')