一、爬取数据

Python版本是3.6，爬取后保存在MySQL中，版本是5.5。

51job搜索位置的链接是【数据分析师招聘，求职】-前程无忧

首先是可以在ide中运行scrapy的文件run.py：

from scrapy.cmdline import execute

execute(['scrapy', 'crawl', 'job51'])

需要爬取并存储的字段item.py：

import scrapy

class Job51Item(scrapy.Item):

# define the fields for your item here like:

# name = scrapy.Field()

zhiwei = scrapy.Field()

gongsi = scrapy.Field()

didian = scrapy.Field()

xinzi = scrapy.Field()

gongsileixing = scrapy.Field()

guimo = scrapy.Field()

hangye = scrapy.Field()

jingyan = scrapy.Field()

xueli = scrapy.Field()

fuli = scrapy.Field()

zhiweiyaoqiu = scrapy.Field()

lianjie = scrapy.Field()

爬虫的入口 job51.py：

import re

import scrapy

from bs4 import BeautifulSoup

from items import Job51Item

class Myspider(scrapy.Spider):

name = 'job51'

allowed_domains = ['51job.com']

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'}

def start_requests(self):

for i in range(1, 208):

url_1 = 'http://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE%25E5%2588%2586%25E6%259E%2590%25E5%25B8%2588,2,'

url_2 = '.html?lang=c&stype=1&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=1&dibiaoid=0&address=&line=&specialarea=00&from=&welfare='

url = url_1 + str(i) + url_2

yield scrapy.Request(url, headers=self.headers, callback=self.parse)

#获取每个职位的详细信息入口url

def parse(self, response):

soup = BeautifulSoup(response.text, 'lxml')

tls = soup.find_all('p', class_='t1 ')

for tl in tls:

url = tl.find('a', target='_blank')['href']

yield scrapy.Request(url, callback=self.get_content, meta={'url': url})

#用BeautifulSoup爬取详细页字段

def get_content(self, response):

soup = BeautifulSoup(response.text, 'lxml')

item = Job51Item()

item['zhiwei'] = soup.find('h1').get_text().replace('\xa0', '')

item['gongsi'] = soup.find('p', class_='cname').find('a', target='_blank').get_text().replace('\xa0', '')

item['didian'] = soup.find('span', class_='lname').get_text().replace('\xa0', '')

item['xinzi'] = soup.find('div', class_='cn').find('strong').get_text().replace('\xa0', '')

gongsixinxi = soup.find('p', class_='msg ltype').get_text().replace('\t', '').replace('\r', '').replace('\n', '').replace('\xa0', '')

item['gongsileixing'] = gongsixinxi.split('|')[0]

item['guimo'] = gongsixinxi.split('|')[1]

item['hangye'] = gongsixinxi.split('|')[2]

zhaopinyaoqiu = soup.find('div', class_='t1').get_text().replace('\xa0', '')

item['jingyan'] = zhaopinyaoqiu.split('\n')[1]

try:

item['xueli'] = re.findall(r'(.*?)', response.text)[0]

except:

item['xueli'] = '无'

try:

item['fuli'] = soup.find('p', class_='t2').get_text().replace('\n', ' ').replace('\xa0', '')

except:

item['fuli'] = '无'

item['zhiweiyaoqiu'] =

re.findall(r'< div class="bmsg job_msg inbox">(.*?)
', response.text, re.I|re.S|re.M)[0].replace('\r', '').replace('\n', '').replace('\t', '').replace('\xa0', '').replace('
', '').replace('
', '')

item['lianjie'] = response.meta['url']

yield item

SQL部分sql.py：

import pymysql.cursors

import settings

MYSQL_HOSTS = settings.MYSQL_HOSTS

MYSQL_USER = settings.MYSQL_USER

MYSQL_PASSWORD = settings.MYSQL_PASSWORD

MYSQL_PORT = settings.MYSQL_PORT

MYSQL_DB = settings.MYSQL_DB

cnx = pymysql.connect(

host=MYSQL_HOSTS,

port=MYSQL_PORT,

user=MYSQL_USER,

passwd=MYSQL_PASSWORD,

db=MYSQL_DB,

charset='gbk')

cur = cnx.cursor()

class Sql:

@classmethod

def insert_job51(cls, zhiwei, gongsi, didian, xinzi, gongsileixing, guimo, hangye, jingyan, xueli, fuli, zhiweiyaoqiu, lianjie):

sql = 'INSERT INTO job51(zhiwei,gongsi,didian,xinzi,gongsileixing,guimo,hangye,jingyan,xueli,fuli,zhiweiyaoqiu,lianjie) ' \

'VALUES(%(zhiwei)s,%(gongsi)s,%(didian)s,%(xinzi)s,%(gongsileixing)s,%(guimo)s,%(hangye)s,%(jingyan)s,%(xueli)s,%(fuli)s,%(zhiweiyaoqiu)s,%(lianjie)s)'

value = {'zhiwei': zhiwei,

'gongsi': gongsi,

'didian': didian,

'xinzi': xinzi,

'gongsileixing': gongsileixing,

'guimo': guimo,

'hangye': hangye,

'jingyan': jingyan,

'xueli': xueli,

'fuli': fuli,

'zhiweiyaoqiu': zhiweiyaoqiu,

'lianjie': lianjie,}

cur.execute(sql, value)

cnx.commit()

管道存储pipelines.py：

from .sql import Sql

from items import Job51Item

class Job51Pipeline(object):

def process_item(self, item, spider):

zhiwei = item['zhiwei']

gongsi = item['gongsi']

didian = item['didian']

xinzi = item['xinzi']

gongsileixing = item['gongsileixing']

guimo = item['guimo']

hangye = item['hangye']

jingyan = item['jingyan']

xueli = item['xueli']

fuli = item['fuli']

zhiweiyaoqiu = item['zhiweiyaoqiu']

lianjie = item['lianjie']

Sql.insert_job51(zhiwei, gongsi, didian, xinzi, gongsileixing, guimo, hangye, jingyan, xueli, fuli, zhiweiyaoqiu, lianjie)

print('写入职位信息')

在MySQL中建立新表，包含所有字段：

CREATE TABLE `job51` (

`id` int(10) NOT NULL AUTO_INCREMENT,

`zhiwei` varchar(255) DEFAULT NULL,

`gongsi` varchar(255) DEFAULT NULL,

`didian` varchar(255) DEFAULT NULL,

`xinzi` varchar(255) DEFAULT NULL,

`gongsileixing` varchar(255) DEFAULT NULL,

`guimo` varchar(255) DEFAULT NULL,

`hangye` varchar(255) DEFAULT NULL,

`jingyan` varchar(255) DEFAULT NULL,

`xueli` varchar(255) DEFAULT NULL,

`fuli` varchar(255) DEFAULT NULL,

`zhiweiyaoqiu` text,

`lianjie` text,

PRIMARY KEY (`id`)

) ENGINE=InnoDB DEFAULT CHARSET=gbk

配置settings.py：

BOT_NAME = 'job51'

SPIDER_MODULES = ['job51.spiders']

NEWSPIDER_MODULE = 'job51.spiders'

MYSQL_HOSTS = 'localhost'

MYSQL_USER = 'root'

MYSQL_PASSWORD = '123456'

MYSQL_PORT = 3306

MYSQL_DB = 'job51'

ROBOTSTXT_OBEY = True

ITEM_PIPELINES = {

'job51.pipelines.Job51Pipeline': 300,

}

HTTPCACHE_ENABLED = True

HTTPCACHE_EXPIRATION_SECS = 0

HTTPCACHE_DIR = 'httpcache'

HTTPCACHE_IGNORE_HTTP_CODES = []

HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'