一、需求
因为业务需求,需要实现新闻资讯功能。初步方案通过第三方提供的服务接口来实现此功能。由于谈判失败,因此决定自开发一套爬虫接口。因此通过查询相关文档,决定采用webmagic开源框架实现自己的爬虫功能。
二、实施过程
1、引入依赖
在pom文件中添加依赖:
us.codecraft
webmagic-core
0.7.3
us.codecraft
webmagic-extension
0.7.3
2、创建相关接口
创建实现类,代码如下(仅供参考):
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import team.biteeny.admin.db.write.cache.ConfigMapper;
import team.biteeny.admin.db.write.mapper.CrawlMapper;
import team.biteeny.admin.db.write.model.CrawlModel;
import team.biteeny.push.getui.PushApp;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Json;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Component
public class HuobiInfoProcessor implements PageProcessor {
@Autowired
private CrawlMapper crawlMapper;
@Autowired
private ConfigMapper configMapper;
private Site site;
private static Map map = new ConcurrentHashMap();
@Override
public void process(Page page) {
if (page.getUrl().toString().contains("flash")){
insertFlash(page);
}
if (page.getUrl().toString().contains("article")){
List urlList = new ArrayList<>();
Json json = page.getJson();
JSONObject jsonObject = JSONObject.parseObject(json.toString());
JSONArray jsonArray = jsonObject.getJSONObject("data").getJSONArray("data");
for (Object o : jsonArray) {
JSONObject object = JSONObject.parseObject(JSONObject.toJSONString(o));
String key = "baseDetail_" + object.getString("id");
urlList.add("https://www.huobiinfo.com/news/"+key);
map.put(key + "_listPicturePath",object.getString("listPicturePath"));
map.put(key + "_title",object.getString("title"));
}
page.addTargetRequests(urlList);
}
if (page.getUrl().toString().contains("news/baseDetail_")){
insertNews(page);
}
}
@Override
public Site getSite() {
if (site==null){
site= Site.me().setDomain("www.huobiinfo.com")
.setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36")
.setCharset("UTF-8")
.setSleepTime(500);
}
return site;
}
public static void main(String[] args) {
// Spider.create(new HuobiInfoProcessor()).addUrl("https://www.huobiinfo.com/flash/").runAsync();
// Request request = new Request("https://huobi-news-app-gateway-outer.huobi.cn:8005/article/listPagedArticleListByParam");
// request.setMethod(HttpConstant.Method.POST);
// request.setRequestBody(HttpRequestBody.json("{\"pageSize\":10,\"pageNum\":1,\"categoryPcId\":15}","utf-8"));
// Spider.create(new HuobiInfoProcessor()).addRequest(request).runAsync();
// String title = "BTC链上基础指标略有回暖,链上场内场外交易均较活跃";
// String c = "根据Searchain.io数据分析:昨日BTC从4100下降到3900点。从链上指标来看,昨日反映BTC内部价值的基础指标整体有所上升,新增地址上升14.89%,活跃地址上升12.20%。从链上交易指标来看,交易用户的活跃度也在上升,交易所流入增加49.16%,流出增加40.78%;链上大额转账的活跃程度集中在100-600 BTC区间,600+ BTC的转账有所下降,大额流入交易所占比有所上升,场内场外均比较活跃。综合链上基础和交易指标来看,近期BTC内部价值略有回暖,链上场内场外交易均活跃。独立分析师Edward对近期BTC市场呈较为悲观状态。\n" +
// "只有币名和百分比,没有价格波动词,所以不符合推送条件";
// boolean b = checkPush(title+c);
// System.out.println(b);
}
private void insertFlash(Page page){
Elements elements = page.getHtml().getDocument().getElementsByClass("item-flash");
for (Element element : elements) {
Html html = new Html(element.toString());
String s = html.xpath("//div[@class='item-flash']//h3[@class='med']//nuxt-link/@to").toString();
String key = s.substring(1, s.lastIndexOf("/")).replace("/", "_");
if (crawlMapper.checkExist(key) <= 0){
String title = html.xpath("//div[@class='item-flash']//h3[@class='med']//nuxt-link/text()").toString();
String content = html.xpath("//div[@class='item-flash']//div[@class='content']/text()").toString();
CrawlModel model = new CrawlModel();
boolean b = checkPush(title + content);
model.setId(key);
model.setBody(content);
model.setTitle(title);
model.setSource("HuobiInfo");
model.setType("flash");
if (b){
model.setIs_push(true);
push(title,content);
}else {
model.setIs_push(false);
}
model.setCreate_time(new Date());
crawlMapper.crawlInsert(model);
}
}
}
private void insertNews(Page page){
String path = page.getUrl().toString();
String key = path.substring(path.lastIndexOf("/") + 1);
if (crawlMapper.checkExist(key) <= 0) {
String source = "来源:" + page.getHtml().xpath("//div[@class='detail-platform-msg']//p[@class='detail-platform']/text()").toString()+"
";
String notice = "" +
page.getHtml().xpath("//div[@class='detail-source']/text()") +
"";
String article = page.getHtml().xpath("//div[@class='detail-content article-content hb-article']").toString();
String content = source + notice + article;
if (!checkDomain(article)){
CrawlModel model = new CrawlModel();
model.setId(key);
model.setTitle((String) map.get(key + "_title"));
model.setBody(content);
model.setList_picture((String) map.get(key + "_listPicturePath"));
model.setSource("HuobiInfo");
model.setType("news");
model.setCreate_time(new Date());
crawlMapper.crawlInsert(model);
}
}
}
private static boolean checkPush(String str){
if (str == null){
return false;
}
String regex = "btc|eth|bch|ltc|etc|eos|xrp|dash|trx";
String regex1 = "涨|跌|涨幅|跌幅|上涨|下跌";
String regexF = "大额转账|净流入|净流出";
String regexH = "okex|火币|币安|比特大陆";
String regex2 = "\\d+(\\.?\\d*?)(?=%)";
Pattern p = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
Pattern p1 = Pattern.compile(regex1);
Pattern pf = Pattern.compile(regexF);
Pattern ph = Pattern.compile(regexH);
Pattern p2 = Pattern.compile(regex2);
Matcher matcher = p.matcher(str);
Matcher matcher1 = p1.matcher(str);
Matcher matcherF = pf.matcher(str);
Matcher matcherH = ph.matcher(str);
Matcher matcher2 = p2.matcher(str);
if (matcher.find() && matcherF.find()){
return true;
}
if (matcherH.find()){
return true;
}
if (matcher.find() && matcher1.find()){
while (matcher2.find()){
Double d = Double.valueOf(matcher2.group());
if (d > 5){
return true;
}
}
return false;
}
return false;
}
private void push(String title,String text){
// 推送相关
// int hour = Calendar.getInstance().get(Calendar.HOUR);
// if (hour >= 8 && hour <= 22){
// }
}
private boolean checkDomain(String content){
if (content == null){
return false;
}
String pattern = "mmbiz\\.qpic\\.cn";
Pattern p = Pattern.compile(pattern);
Matcher m = p.matcher(content);
if(m.find()){
return true;
}
return false;
}
以上是一个简单的实例,包括一些过滤逻辑和数据持久化逻辑等。
三、总结
以上是java通过webmagic实现网页爬虫的简单例子,对webmagic的应用也不是很到位,只是已经满足了当前的业务需求。也未作深入研究。对webmagic也没有详细描述,有兴趣的童鞋们可以自行查阅相关文档学习,欢迎大家加入进来一起讨论学习。
能力有限,难免有不当之处,欢迎大家批评指正。把技术死磕到底!