爬虫学习记录
获取58同城上的招聘信息
爬虫的意义
我们编写爬虫就是把网页中的关键信息爬取下来，然后做分析，现在是数据时代，所以数据是很重要的资源。爬虫可以帮助我们获取这些资源。
本文的目的
现在的爬虫技术很多，但是以python为主，作为初学者我建议不要使用太多现成的工具，这样无法学习到里面的技术，比如你在使用scrapy时都很难了解它在内部调用了什么，这篇文章也将用urllib2+beautifulSoup+mysql来获取58同城上的招聘信息，最关键的是分析网页源代码，找到需要的信息。
获取网页源码
url = "http://hz.58.com/tech/" + "pn"+str(start)+"/"
request = urllib2.Request(url=url,headers=headers)
response = urllib2.urlopen(request,timeout=60)
html = response.read().decode('utf-8')
soup = BeautifulSoup(html,'lxml')
获取58的列表信息
for item in all_dl:
job = item.find('dt').find('a')
info = getdatas.getInfo(job['href'])
if info != 0:
count += insertmysql.insertMysql(info)
print "现在的数据量为%d"%(count)
time.sleep(5)
start = start + 1
其中的每一个item就是一条招聘信息，然后进入这个二级地址，获取相关的招聘信息
二级网址
在这个部分首先也要获取网页源代码，然后用beautifulSoup来匹配关键信息，beautifulSoup的用法可以在官网看看。
def getInfo(url):
headers = {}
headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36"
try:
# proxies = {'http': proxy_ip}
request = urllib2.Request(url=url, headers=headers)
# request.set_proxy(proxy_ip, 'http')
response = urllib2.urlopen(request)
html = response.read().decode('utf-8')
# html = requests.get(url, headers=headers, proxies=proxies)
html = BeautifulSoup(html, 'lxml')
info = {}
info['id'] = uuid.uuid4()
info['title'] = html.find('div', class_='item_con pos_info').find('span', class_='pos_name').get_text()
temp = html.find('div', class_='pos_base_info').find('span', class_='pos_salary').get_text()
info['salary_min'] = 0+int(re.findall(r"(\d+)\-", temp)[0])
info['salary_max'] = 0 + int(re.findall(r"\-(\d+)", temp)[0])
info['company'] = html.find('div', class_='item_con company_baseInfo').find('p',class_='comp_baseInfo_title').find('a', class_='baseInfo_link').get_text()
temp = html.find('div', class_='item_con company_baseInfo').find('p', class_='comp_baseInfo_scale').get_text()
info['scale_min'] = 0+int(re.findall(r"(\d+)\-", temp)[0])
info['scale_max'] = 0+int(re.findall(r"\-(\d+)", temp)[0])
info['address'] = html.find('div', class_='item_con work_adress').find('p', class_='detail_adress').get_text()
return info
except Exception, e:
return 0
我用uuid作为主键，爬取了招聘信息中的主要内容，薪水，公司规模，公司地址等信息，但是58里的招聘页面有些不是按照这个标准设置的，所以如果想要更加完整的信息，就需要在分类讨论一下。
存储数据库
这里选择的数据库是mysql，python连接mysql也很容易：
db = MySQLdb.connect(host='localhost', user='root', passwd='123', db='58city', port=3306,charset='utf8')
cursor = db.cursor()
然后将相关的信息放到mysql中：
cursor.execute(
'insert into jobs(id,title,salary_min,salary_max,company,scale_min,scale_max,address) values(%s,%s,%s,%s,%s,%s,%s,%s)',
(id,title,salary_min,salary_max,company,scale_min,scale_max,address))
db.commit()
db.close()
cursor.close()
我们在写代码的时候会肯定会有bug，所以使用try catch 的方法最好。
except Exception, e:
print e.message+"数据库报错"+e.message+e.args[0]
return 0
反爬的策略
我们可以做个ip代理，防止地址被封，并且设置休眠时间，以免爬取太快
被网站察觉。
这里提供源代码
# coding:utf8
import random
import urllib2
import time
from bs4 import BeautifulSoup
import getdatas
import insertmysql
import requests
ISOTIMEFORMAT = '%Y-%m-%d %X'
headers = {}
headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36"
import getip
# 获取tag
# start
print
"********** START **********"
print
time.strftime(ISOTIMEFORMAT, time.localtime())
try:
start = 33
count = 0
# proxy_list = getip.get_ips()
while True:
try:
# proxy_ip = random.choice(proxy_list)
# proxies = {'http': proxy_ip}
#
url = "http://hz.58.com/tech/" + "pn"+str(start)+"/"
request = urllib2.Request(url=url,headers=headers)
# request.set_proxy(proxy_ip,'http')
response = urllib2.urlopen(request,timeout=60)
html = response.read().decode('utf-8')
# html = requests.get(url, headers=headers, proxies=proxies)
soup = BeautifulSoup(html,'lxml')
all_dl = soup.find('div',id='infolist').findAll('dl')
if len(all_dl) == 0:
break
for item in all_dl:
job = item.find('dt').find('a')
info = getdatas.getInfo(job['href'])
if info != 0:
count += insertmysql.insertMysql(info)
print "现在的数据量为%d"%(count)
time.sleep(5)
start = start + 1
print start
time.sleep(5)
# print info_list['director']
except Exception, e:
print e.message + "1"
except Exception, e:
print e.message +'2'
# coding:utf8
import urllib2
import urllib
import json
import time
import re
import random
import uuid
import requests
from bs4 import BeautifulSoup
def getInfo(url):
headers = {}
headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36"
try:
# proxies = {'http': proxy_ip}
request = urllib2.Request(url=url, headers=headers)
# request.set_proxy(proxy_ip, 'http')
response = urllib2.urlopen(request)
html = response.read().decode('utf-8')
# html = requests.get(url, headers=headers, proxies=proxies)
html = BeautifulSoup(html, 'lxml')
info = {}
info['id'] = uuid.uuid4()
info['title'] = html.find('div', class_='item_con pos_info').find('span', class_='pos_name').get_text()
temp = html.find('div', class_='pos_base_info').find('span', class_='pos_salary').get_text()
info['salary_min'] = 0+int(re.findall(r"(\d+)\-", temp)[0])
info['salary_max'] = 0 + int(re.findall(r"\-(\d+)", temp)[0])
info['company'] = html.find('div', class_='item_con company_baseInfo').find('p',class_='comp_baseInfo_title').find('a', class_='baseInfo_link').get_text()
temp = html.find('div', class_='item_con company_baseInfo').find('p', class_='comp_baseInfo_scale').get_text()
info['scale_min'] = 0+int(re.findall(r"(\d+)\-", temp)[0])
info['scale_max'] = 0+int(re.findall(r"\-(\d+)", temp)[0])
info['address'] = html.find('div', class_='item_con work_adress').find('p', class_='detail_adress').get_text()
return info
except Exception, e:
return 0
# -*- coding:utf-8 -*-
import MySQLdb
import MySQLdb.cursors
import getCity
def insertMysql(info):
if info == None:
print "there is no infomation"
return 0
else:
try:
db = MySQLdb.connect(host='localhost', user='root', passwd='123', db='58city', port=3306,charset='utf8')
cursor = db.cursor()
id = info['id']
title = info['title']
salary_min = info['salary_min']
salary_max = info['salary_max']
company = info['company']
scale_min = info['scale_min']
scale_max = info['scale_max']
address = info['address']
cursor.execute(
'insert into jobs(id,title,salary_min,salary_max,company,scale_min,scale_max,address) values(%s,%s,%s,%s,%s,%s,%s,%s)',
(id,title,salary_min,salary_max,company,scale_min,scale_max,address))
db.commit()
db.close()
cursor.close()
return 1
except Exception, e:
print e.message+"数据库报错"+e.message+e.args[0]
return 0