Java爬虫入门到精通（十三）——WebMagic爬虫小案例
废话不多说，因为相较于HttpClient的爬虫会比较简单，直接上代码，代码中会有注释
项目是爬去51job.com有关java的招聘信息
目录结构：
pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.5.RELEASE</version>
</parent>
<groupId>org.example</groupId>
<artifactId>crawler-51job</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<!--SpringMVC-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!--SpringData Jpa-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<!--MySQL连接包-->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<!--WebMagic-->
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-core</artifactId>
<version>0.7.3</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<!--WebMagic扩展-->
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-extension</artifactId>
<version>0.7.3</version>
</dependency>
<!--工具包-->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<!--不知道这个干嘛用的，但是没这个会报错-->
<!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>28.2-jre</version>
</dependency>
</dependencies>
</project>
application.properties
#DB Configuration
spring.datasource.driverClassName=com.mysql.cj.jdbc.Driver
spring.datasource.url=jdbc:mysql://127.0.0.1:3306/crawler
spring.datasource.username=root
spring.datasource.password=******
#JPA Configuration
spring.jpa.database=mysql
spring.jpa.show-sql=true
pojo实体类
JobInfo.java
package com.zzdreamz.job.pojo;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
@Entity
public class JobInfo {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
private String companyName;
private String companyAddr;
private String companyInfo;
private String jobName;
private String jobAddr;
private String jobInfo;
private String salary;
private String url;
private String time;
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getCompanyName() {
return companyName;
}
public void setCompanyName(String companyName) {
this.companyName = companyName;
}
public String getCompanyAddr() {
return companyAddr;
}
public void setCompanyAddr(String companyAddr) {
this.companyAddr = companyAddr;
}
public String getCompanyInfo() {
return companyInfo;
}
public void setCompanyInfo(String companyInfo) {
this.companyInfo = companyInfo;
}
public String getJobName() {
return jobName;
}
public void setJobName(String jobName) {
this.jobName = jobName;
}
public String getJobAddr() {
return jobAddr;
}
public void setJobAddr(String jobAddr) {
this.jobAddr = jobAddr;
}
public String getJobInfo() {
return jobInfo;
}
public void setJobInfo(String jobInfo) {
this.jobInfo = jobInfo;
}
public String getSalary() {
return salary;
}
public void setSalary(String salary) {
this.salary = salary;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public String getTime() {
return time;
}
public void setTime(String time) {
this.time = time;
}
@Override
public String toString() {
return "JobInfo{" +
"id=" + id +
", companyName='" + companyName + '\'' +
", companyAddr='" + companyAddr + '\'' +
", companyInfo='" + companyInfo + '\'' +
", jobName='" + jobName + '\'' +
", jobAddr='" + jobAddr + '\'' +
", jobInfo='" + jobInfo + '\'' +
", salary='" + salary + '\'' +
", url='" + url + '\'' +
", time='" + time + '\'' +
'}';
}
}
dao层：JobInfoDao.java
package com.zzdreamz.job.dao;
import com.zzdreamz.job.pojo.JobInfo;
import org.springframework.data.jpa.repository.JpaRepository;
public interface JobInfoDao extends JpaRepository<JobInfo,Long> {
}
service层
JobInfoService.java
package com.zzdreamz.job.service;
import com.zzdreamz.job.pojo.JobInfo;
import java.util.List;
public interface JobInfoService {
public void save(JobInfo jobInfo);
public List<JobInfo> findJobInfo(JobInfo jobInfo);
}
JobInfoServiceImpl.java
package com.zzdreamz.job.service;
import com.zzdreamz.job.dao.JobInfoDao;
import com.zzdreamz.job.pojo.JobInfo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Example;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
@Service
public class JobInfoServiceImpl implements JobInfoService {
@Autowired
private JobInfoDao jobInfoDao;
@Override
@Transactional
public void save(JobInfo jobInfo) {
//根据url和发布时间查询数据
JobInfo param = new JobInfo();
param.setUrl(jobInfo.getUrl());
param.setTime(jobInfo.getTime());
//执行查询
List<JobInfo> list = this.findJobInfo(param);
//判断结果是否为空
if (list.size() == 0) {
//如果查询结果为空，表示数据不存在或需要更新
this.jobInfoDao.saveAndFlush(jobInfo);
}
}
@Override
public List<JobInfo> findJobInfo(JobInfo jobInfo) {
//设置查询条件
Example<JobInfo> example = Example.of(jobInfo);
//执行查询
List<JobInfo> list = this.jobInfoDao.findAll(example);
return list;
}
}
PipeLine实现数据放入数据库
SpringDataPipeline.java
package com.zzdreamz.job.task;
import com.zzdreamz.job.pojo.JobInfo;
import com.zzdreamz.job.service.JobInfoService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;
@Component
public class SpringDataPipeline implements Pipeline {
@Autowired
private JobInfoService jobInfoService;
@Override
public void process(ResultItems resultItems, Task task) {
//获取封装好的招聘详情对象
JobInfo jobInfo = resultItems.get("jobInfo");
//判断数据是否为空
if (jobInfo != null) {
//如果不为空则保存到数据库
jobInfoService.save(jobInfo);
}
}
}
任务类
JobProcessor.java
package com.zzdreamz.job.task;
import com.zzdreamz.job.pojo.JobInfo;
import org.jsoup.Jsoup;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;
import java.util.List;
@Component
public class JobProcessor implements PageProcessor {
private String url = "https://search.51job.com/list/000000,000000,0000,01%252C32,9,99,Java,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=";
@Override
public void process(Page page) {
//解析页面，获取详情页的url地址
List<Selectable> list = page.getHtml().css("div#resultList div.el").nodes();
//判断获取的集合是否为空
if (list.size() == 0) {
//如果为空表示招聘详情页，解析页面获取商品信息并保存数据
this.saveJobInfo(page);
} else {
//不为空表示列表页,解析页面获取详情页url放到任务队列中
for (Selectable selectable : list) {
//获取url地址
String jobInfoUrl = selectable.links().toString();
//把获取到的url地址放到任务队列中
page.addTargetRequest(jobInfoUrl);
}
//获取下一页的url
String bkUrl = page.getHtml().css("div.p_in li.bk").nodes().get(1).links().toString();
page.addTargetRequest(bkUrl);
}
String html = page.getHtml().toString();
}
//解析页面获取商品信息并保存数据
private void saveJobInfo(Page page) {
//创建招聘详情对象
JobInfo jobInfo = new JobInfo();
//解析页面
Html html = page.getHtml();
//获取数据，封装到对象中
jobInfo.setCompanyName(html.css("div.cn p.cname a","text").toString());
jobInfo.setCompanyAddr(Jsoup.parse(html.css("div.bmsg").nodes().get(1).toString()).text());
jobInfo.setCompanyInfo(Jsoup.parse(html.css("div.tmsg").toString()).text());
jobInfo.setJobName(html.css("div.cn h1","text").toString());
String info = html.css("p.msg","text").toString();
jobInfo.setJobAddr(info.substring(0,info.indexOf(" ")));
jobInfo.setJobInfo(Jsoup.parse(html.css("div.job_msg").toString()).text());
jobInfo.setUrl(page.getUrl().toString());
jobInfo.setSalary(html.css("div.cn strong", "text").toString());
jobInfo.setTime(info.substring(info.lastIndexOf(" ")));
//把结果保存起来
page.putField("jobInfo",jobInfo);
}
private Site site = Site.me()
.setCharset("gbk")
.setTimeOut(10*1000)
.setRetrySleepTime(3000)
.setRetryTimes(3);
@Override
public Site getSite() {
return site;
}
@Autowired
private SpringDataPipeline springDataPipeline;
@Scheduled(initialDelay = 1000, fixedDelay = 3600 * 1000)
public void process() {
Spider.create(new JobProcessor())
.addUrl(url)
.setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
.thread(10)
.addPipeline(this.springDataPipeline)
.run();
}
}
引导类：Application.java
package com.zzdreamz.job;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
@SpringBootApplication
@EnableScheduling
public class Application {
public static void main(String[] args) {
SpringApplication.run(Application.class,args);
}
}