java爬取前程无忧(51job)
什么是Jsoup
jsoup是一款优秀的Java的HTML解析器，主要用来对HTML解析，就是dom的操作,有很多和js方法一样,如getElementById，select等,使用起来非常的方便,不清楚的朋友可以去学习下这里就不强调了。
爬取前程无忧准备工作
1.创建一个普通的maven工程
2.在 pom.xml文件中引入依赖
<dependencies>
<dependency>
<groupId>org.apache.httpcomponentsgroupId>
<artifactId>httpclientartifactId>
<version>4.5.9version>
dependency>
<dependency>
<groupId>org.apache.httpcomponentsgroupId>
<artifactId>httpcoreartifactId>
<version>4.4.11version>
dependency>
<dependency>
<groupId>org.jsoupgroupId>
<artifactId>jsoupartifactId>
<version>1.12.1version>
dependency>
dependencies>
分析51job网站
如何获取所有的城市和代码？
开始爬取51job
观察51job的字段创建对应的实体类表
创建Jobs实体类
public class Jobs {
private Integer jobId;//自动增长id
private String jobName; //岗位
private String companyName;//公司名
private String workAddr;//公司地址
private String salary;//薪水
private String pushDate;//发布日期
private String jobKey;//对应城市的id
public Integer getJobId() {
return jobId;
}
public void setJobId(Integer jobId) {
this.jobId = jobId;
}
public String getJobName() {
return jobName;
}
public void setJobName(String jobName) {
this.jobName = jobName;
}
public String getCompanyName() {
return companyName;
}
public void setCompanyName(String companyName) {
this.companyName = companyName;
}
public String getWorkAddr() {
return workAddr;
}
public void setWorkAddr(String workAddr) {
this.workAddr = workAddr;
}
public String getSalary() {
return salary;
}
public void setSalary(String salary) {
this.salary = salary;
}
public String getPushDate() {
return pushDate;
}
public void setPushDate(String pushDate) {
this.pushDate = pushDate;
}
public String getJobKey() {
return jobKey;
}
public void setJobKey(String jobKey) {
this.jobKey = jobKey;
}
创建JobParse
public class JobParse {
public static List<Jobs> getData(String entity){
/**
* 读取mybatis配置文件
*/
String resource = "mybatis-config.xml";
InputStream inputStream = null;
try {
inputStream = Resources.getResourceAsStream(resource);
} catch (IOException e) {
e.printStackTrace();
}
/**
* 得到连接对象注册sqlsession
*/
SqlSessionFactory sqlSessionFactory = new SqlSessionFactoryBuilder().build(inputStream);
SqlSession sqlSession = sqlSessionFactory.openSession();
JobsMapper jobsMapper = sqlSession.getMapper(JobsMapper.class);
List<Jobs> data = new ArrayList<Jobs>();
Document doc = Jsoup.parse(entity);
Elements elements = doc.select("div.el");
Elements title = elements.select("p.t1").select("span").select("a"); //标题
Elements complany = elements.select("span.t2").select("a"); //公司
Elements address = elements.select("span.t3");//地址
Elements salary = elements.select("span.t4");//薪水
Elements datas = elements.select("span.t5");//发布日期
Elements SrcId = elements.select("p.t1").select("input.checkbox");//招聘信息对应的id
Jobs jobs = new Jobs();
if (title !=null || title.equals("")) {
for (Element element : title) {
jobs.setJobName(element.text());
}
}
if (complany !=null || complany.equals("")) {
for (Element element : complany) {
jobs.setCompanyName(element.text());
}
}
if (address !=null || address.equals("")) {
for (Element element : address) {
jobs.setWorkAddr(element.text());
}
}
if (salary !=null || salary.equals("")) {
for (Element element : salary) {
jobs.setSalary(element.text());
}
}
if (datas !=null || datas.equals("")) {
for (Element element : datas) {
jobs.setPushDate(element.text());
}
}
if (SrcId !=null || SrcId.equals("")) {
for (Element element : SrcId) {
jobs.setJobKey(element.attr("value"));
}
}
jobsMapper.insert(jobs);
sqlSession.commit();
data.add(jobs);
return data;
}
创建爬取主启动程序JobMain
public class JobMain {
public static void main(String[] args) {
System.out.println("正在生成客户端...");
HttpClient client = null;
System.out.println("客户端生成完毕.");
String[] city = {"阿坝", "阿克苏", "阿拉尔", "阿拉善盟", "阿勒泰", "阿里", "鞍山",
"安康", "安庆", "安顺", "安阳", "巴彦淖尔", "巴音郭楞", "巴中",
"白城", "白沙", "白山", "白银", "百色", "蚌埠", "包头",
"保定", "保山", "保亭", "宝鸡", "北海", "北京", "本溪",
"毕节", "滨州", "博尔塔拉", "亳州", "沧州", "昌都", "昌吉",
"昌江", "常德", "常熟", "常州", "长春", "长沙", "长治",
"朝阳", "潮州", "郴州", "成都", "澄迈", "承德", "池州",
"赤峰", "崇左", "滁州", "楚雄", "重庆", "达州", "大理", "大连", "大庆", "大同", "大兴安岭", "丹东",
"丹阳", "德宏", "德阳", "德州", "邓州", "迪庆", "定安",
"定西", "东方", "东营", "东莞", "儋州", "鄂尔多斯", "鄂州",
"恩施", "防城港", "佛山", "福州", "抚顺", "抚州", "阜新",
"阜阳", "甘南", "甘孜", "赣州", "固原", "广安", "广元",
"广州", "桂林", "贵港", "贵阳", "果洛", "哈尔滨", "哈密", "海北", "海东", "海口", "海南", "海宁",
"海西", "邯郸", "汉中", "杭州", "菏泽", "和田", "合肥",
"河池", "河源", "鹤壁", "鹤岗", "贺州", "黑河", "衡水",
"衡阳", "红河州", "呼和浩特", "呼伦贝尔", "葫芦岛", "湖州", "怀化",
"淮安", "淮北", "淮南", "黄冈", "黄南", "黄山", "黄石",
"惠州", "鸡西", "吉安", "吉林", "济南", "济宁", "济源", "嘉兴",
"嘉峪关", "佳木斯", "江门", "焦作", "揭阳", "金昌", "金华",
"锦州", "晋城", "晋中", "荆门", "荆州", "景德镇", "靖江",
"九江", "酒泉", "喀什地区", "开封", "开平", "克拉玛依", "克孜勒苏柯尔克孜",
"昆明", "昆山", "拉萨", "莱芜", "来宾", "兰州", "廊坊", "乐山", "丽江",
"丽水", "连云港", "凉山", "聊城", "辽阳", "辽源", "林芝",
"临沧", "临汾", "临高", "临夏", "临沂", "陵水", "柳州",
"六安", "六盘水", "龙岩", "陇南", "娄底", "吕梁", "洛阳",
"泸州", "漯河", "马鞍山", "茂名", "梅州", "眉山", "绵阳",
"牡丹江", "那曲", "南昌", "南充", "南京", "南宁", "南平",
"南通", "南阳", "内江", "宁波", "宁德", "怒江", "攀枝花", "盘锦", "萍乡", "平顶山", "平凉", "莆田", "普洱",
"濮阳", "七台河", "齐齐哈尔", "黔东南", "黔南", "黔西南", "潜江",
"钦州", "秦皇岛", "青岛", "清远", "庆阳", "琼海", "琼中",
"曲靖", "泉州", "衢州", "日喀则", "日照", "三门峡", "三明", "三沙", "三亚", "山南", "汕头", "汕尾",
"商洛", "商丘", "上海", "上饶", "韶关", "邵阳", "绍兴",
"深圳", "神农架", "沈阳", "十堰", "石河子", "石家庄", "石嘴山",
"双鸭山", "朔州", "四平", "松原", "苏州", "宿迁", "宿州",
"随州", "绥化", "遂宁", "塔城", "台州", "泰安", "泰兴",
"泰州", "太仓", "太原", "唐山", "天津", "天门", "天水",
"铁岭", "通化", "通辽", "铜川", "铜陵", "铜仁", "图木舒克",
"吐鲁番", "屯昌", "万宁", "威海", "潍坊", "渭南", "温州", "文昌", "文山",
"乌海", "乌兰察布", "乌鲁木齐", "无锡", "芜湖", "梧州", "吴忠",
"武汉", "武威", "五家渠", "五指山", "西安", "西昌", "西宁",
"西双版纳", "锡林郭勒盟", "厦门", "仙桃", "咸宁", "咸阳", "襄阳",
"湘潭", "湘西", "孝感", "新乡", "新余", "忻州", "信阳",
"兴安盟", "邢台", "雄安新区", "徐州", "许昌", "宣城", "乐东", "雅安", "烟台", "盐城", "延安", "延边", "延吉",
"燕郊开发区", "杨凌", "扬州", "洋浦经济开发区", "阳江", "阳泉", "伊春",
"伊犁", "宜宾", "宜昌", "宜春", "义乌", "益阳", "银川",
"鹰潭", "营口", "永州", "榆林", "玉林", "玉树", "玉溪",
"岳阳", "云浮", "运城", "枣庄", "湛江", "漳州", "张家港",
"张家界", "张家口", "张掖", "昭通", "肇庆", "镇江", "郑州",
"中山", "中卫", "舟山", "周口", "珠海", "株洲", "驻马店",
"资阳", "淄博", "自贡", "遵义", "广东省", "江苏省", "浙江省", "四川省", "海南省", "福建省", "山东省",
"江西省", "广西", "安徽省", "河北省", "河南省", "湖北省", "湖南省",
"陕西省", "山西省", "黑龙江省", "辽宁省", "吉林省", "云南省", "贵州省",
"甘肃省", "内蒙古", "宁夏", "西藏", "新疆", "青海省", "香港",
"澳门", "台湾", "国外"};
String[] value = {
"092200","310600","310900","281500","311300",
"300800","230400","201000","150400","260500","170900","280900","311800","092000","241000","101800","240900","270800","141100","150600","280400","160400","251200","101700","200400","140500","010000","231000","260700","121500","311900","151800",
"160800","300600","311200","101900","190700","070700","070500","240200","190200","210600","231400","032000","190900","090200","101300","161000","151500","280300","141400","150900","251700","060000",
"091700","250500","230300","220500","210400","221400","230800","072100","251600","090600","121300","172000","252000","101100","271100","100900","121000","030800","100800","280800","181000","181800","140800",
"030600","110200","230600","131100","231500","150700","271500","092100","130800","290600","091300","091600","030200","140300","141000","260200","320800","220200","310700","320500","320300","100200","320700","081600","320400","160700","200900","080200",
"121400","311600","150200","141200","032100","171700","221000","141500","221200","161200","190500","251000","280200","281100","230900","080900","191100","071900","151700","151100","181100","320600","151000","180400","030300",
"220900","130900", "240300","120200","120900","171900","080700","270400","220800","031500","170500","032200","270300","080600","230700","210700","211000","180800","180700","130400","072500","130300","270500","310400","170400","032700","310300","311700","250200","070600",
"300200","121800","141300","270200","160300","090400","250600","081000","071200","092300","121700","231100","240400","300400","251800","210500","101400","271400","120800","102100","140400","151200","260400","111000","271200","191200","211200",
"170300","090500","171500","150500","032300","032600","091200","090300","220700","300700","130200","091100","070200","140200","110800","070900","170600","090900","080300","110900","251900",
"091000","231300","130500","171000","271000","110600","251100","171600","221300","220600","260900","261000","260800","181500","140900","160600","120300","031900","271300","100600","101600","250300","110400","081200","300300","121200",
"171800","110700","101500","100300","300500","030400","032400","201100","171300","020000","131200","031400","191000","080500","040000","181700","230200","180600","310800","160200","290500","221100","210900","240600","240700","070300","072000","151600","181200","220400","091500","311500",
"080800","121100","072300","071800","071600","210200","160500","050000","181600","270600","231200","240500","280700","200500","150800","260600","311100","311400","101200","100700","120600","120500","200700","080400","100500","251400","281000","281200",
"310200", "070400", "150300", "140700", "290300", "180200", "270700", "311000", "101000", "200200", "091900", "320200", "251500", "281400", "110300", "181400", "181300", "200300", "180500", "190400", "191500", "180900", "170700", "130600", "211100", "171200", "281300", "161100", "160100", "071100", "171100",
"151400","102000","091800","120400","071300","200600","241100","240800","161300","201200","070800","100400","032800","210800","220300","310500","090700","180300","131000","081400","190800","290200","130700","230500","191300","200800","140600","320900","250400","190600","032900","210300","121600","031700","110500","071400","191400","160900","270900","251300","031800","071000","170200","030700",
"290400","081100","170800","030500","190300","171400","091400","120700","090800","260300","030000","070000","080000","090000","100000","110000","120000","130000","140000","150000","160000","170000","180000","190000","200000","210000",
"220000","230000","240000","250000","260000","270000","280000","290000","300000","310000","320000","330000","340000","350000","360000"
};
int pagesize = 1;
boolean splider = true;
for (int num = 0; num <410; num ++) {
while (splider) {
String url = "https://search.51job.com/list/"+ value[num] + ",000000,0000,00,9,99," + city[num] + ",2," + pagesize++ + ".html";
System.out.println(url);
List<Jobs> jobsList = null;
System.out.println("正在生成客户端...");
client = HttpClientBuilder.create().build();
System.out.println("客户端生成完毕.");
//开始解析
try {
System.out.println("开始响应客户端...");
try {
Thread.sleep(200);
jobsList = URLHandle.urlParser(client, url);
if (jobsList.iterator().next().getJob_name().equals("")){
pagesize = 1;
break;
}
} catch (InterruptedException e) {
e.printStackTrace();
}
System.out.println("响应完成.");
} catch (ParseException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("开始输出结果...");
for (Jobs job : jobsList) {
System.out.println(job.getJob_name());
}
System.out.println("整个结果输出完毕，程序结束.");
}
}
}
其中value和city是从51job获取的全部城名和城市代码,这里使用的是while循环,当jobsList中的某一个字段为空的时候说明后面已经没有信息了,里面结束当前循环,并且pagesize初始化为1,下一个城市从第一页开始。就这样反复的轮训city和value数组的城市名和代码来进行爬取,然后通过mybatis插入到数据库。
数据库配置
mapper
public interface JobsMapper {
void insert(Jobs jobs);
List<Jobs> findAll();
}
jdbc.properties
driver=com.mysql.jdbc.Driver
url=jdbc:mysql://localhost:3306/job51?ServerTimezone=GMT%2B8
username=root
password=root
mybatis-config.xml
<configuration>
<properties resource="jdbc.properties">properties>
<environments default="development">
<environment id="development">
<transactionManager type="JDBC"/>
<dataSource type="POOLED">
<property name="driver" value="${driver}"/>
<property name="url" value="${url}"/>
<property name="username" value="${username}"/>
<property name="password" value="${password}"/>
dataSource>
environment>
environments>
<mappers>
<mapper resource="JobsMapper.xml"/>
mappers>
configuration>
mapper.xml
<mapper namespace="cn.com.scitc.mapper.JobsMapper">
<resultMap id="JobsMapperMap" type="cn.com.scitc.model.Jobs">
<id column="job_id" property="jobId" jdbcType="INTEGER"/>
<id column="job_name" property="jobName" jdbcType="VARCHAR"/>
<id column="company_name" property="companyName" jdbcType="VARCHAR"/>
<id column="work_addr" property="workAddr" jdbcType="VARCHAR"/>
<id column="salary" property="salary" jdbcType="VARCHAR"/>
<id column="push_date" property="pushDate" jdbcType="VARCHAR"/>
<id column="job_key" property="jobKey" jdbcType="VARCHAR"/>
resultMap>
<insert id="insert" keyColumn="jobId" useGeneratedKeys="true" parameterType="cn.com.scitc.model.Jobs">
insert into jobs (job_name,company_name,work_addr,salary,push_date,job_key) values (#{jobName},#{companyName},#{workAddr},#{salary},#{pushDate},#{jobKey} )
insert>
<select id="findAll" resultMap="JobsMapperMap">
SELECT * FROM jobs
select>
mapper>
项目地址
github
总结
爬取51job很简单,这里有个小坑就是该网站的编码是gbk,你没有听错就是gbk,转换成utf-8还不行,还是会乱码,所以这里使用gbk编码。