java爬虫抓取百度的搜索热点:
直接上代码:
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;
import java.io.IOException;
public class crawlBaiDuTop {
@Test
public void main() throws IOException {
WebClient webClient = new WebClient(BrowserVersion.CHROME);
//本次抓取内容的网页,无需css和JavaScript,设为false
webClient.getOptions().setJavaScriptEnabled(false);
webClient.getOptions().setCssEnabled(false);
//抓取的网址 http://top.baidu.com/buzz?b=1
HtmlPage page = webClient.getPage("http://top.baidu.com/buzz?b=1");
String html = page.asXml();
Document doc = Jsoup.parse(html);
Elements as;
//解析获取拥有list-title的类属性的标签
as = doc.select(".list-title");
//处理获取结果
for(Element e:as){
System.out.println(e.text()+"-->"+e.attr("href"));
}
}
}
结果:
京东 网约车-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%BE%A9%B6%AB%20%CD%F8%D4%BC%B3%B5
表演钢管舞遭免职-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%B1%ED%D1%DD%B8%D6%B9%DC%CE%E8%D4%E2%C3%E2%D6%B0
咪蒙承认离婚-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%DF%E4%C3%C9%B3%D0%C8%CF%C0%EB%BB%E9
刘强东已经回国-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%C1%F5%C7%BF%B6%AB%D2%D1%BE%AD%BB%D8%B9%FA
秦皇岛超市杀人-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%C7%D8%BB%CA%B5%BA%B3%AC%CA%D0%C9%B1%C8%CB
中秋国庆放假通知-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%D6%D0%C7%EF%B9%FA%C7%EC%B7%C5%BC%D9%CD%A8%D6%AA
撒旦教造谣事件-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%C8%F6%B5%A9%BD%CC%D4%EC%D2%A5%CA%C2%BC%FE
房租暴涨-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%B7%BF%D7%E2%B1%A9%D5%C7
锦觅死了-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%BD%F5%C3%D9%CB%C0%C1%CB
江苏宜兴非洲猪瘟-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%BD%AD%CB%D5%D2%CB%D0%CB%B7%C7%D6%DE%D6%ED%CE%C1
裸辞的7条理由-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%C2%E3%B4%C7%B5%C47%CC%F5%C0%ED%D3%C9
爱奇艺关闭播放量-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%B0%AE%C6%E6%D2%D5%B9%D8%B1%D5%B2%A5%B7%C5%C1%BF
财政部 彩票-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%B2%C6%D5%FE%B2%BF%20%B2%CA%C6%B1
台风登陆日本-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%CC%A8%B7%E7%B5%C7%C2%BD%C8%D5%B1%BE
开学装备配齐3万-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%BF%AA%D1%A7%D7%B0%B1%B8%C5%E4%C6%EB3%CD%F2
伦敦市长比基尼气球-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%C2%D7%B6%D8%CA%D0%B3%A4%B1%C8%BB%F9%C4%E1%C6%F8%C7%F2
乒乓球选手去世-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%C6%B9%C5%D2%C7%F2%D1%A1%CA%D6%C8%A5%CA%C0
萧淑慎直播落泪-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%CF%F4%CA%E7%C9%F7%D6%B1%B2%A5%C2%E4%C0%E1
周立波声援刘强东-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%D6%DC%C1%A2%B2%A8%C9%F9%D4%AE%C1%F5%C7%BF%B6%AB
虹鳟出标准后滞销-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%BA%E7%F7%AE%B3%F6%B1%EA%D7%BC%BA%F3%D6%CD%CF%FA
四川高校 共享学长-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%CB%C4%B4%A8%B8%DF%D0%A3%20%B9%B2%CF%ED%D1%A7%B3%A4
梅西犯罪式进球-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%C3%B7%CE%F7%B7%B8%D7%EF%CA%BD%BD%F8%C7%F2
滴滴代驾司机身亡-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%B5%CE%B5%CE%B4%FA%BC%DD%CB%BE%BB%FA%C9%ED%CD%F6
刘强东性侵大学生-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%C1%F5%C7%BF%B6%AB%D0%D4%C7%D6%B4%F3%D1%A7%C9%FA
中国队亚运遗憾-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%D6%D0%B9%FA%B6%D3%D1%C7%D4%CB%D2%C5%BA%B6
北电大一新生合影-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%B1%B1%B5%E7%B4%F3%D2%BB%D0%C2%C9%FA%BA%CF%D3%B0
美团外卖回应约谈-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%C3%C0%CD%C5%CD%E2%C2%F4%BB%D8%D3%A6%D4%BC%CC%B8
韩春雨事件-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%BA%AB%B4%BA%D3%EA%CA%C2%BC%FE
景甜 张继科-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%BE%B0%CC%F0%20%D5%C5%BC%CC%BF%C6
如懿出冷宫-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%C8%E7%DC%B2%B3%F6%C0%E4%B9%AC
南非弹药库爆炸-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%C4%CF%B7%C7%B5%AF%D2%A9%BF%E2%B1%AC%D5%A8
军训-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%BE%FC%D1%B5
无锡滴滴司机被拘-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%CE%DE%CE%FD%B5%CE%B5%CE%CB%BE%BB%FA%B1%BB%BE%D0
英尺 男子 高空 上演-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%D3%A2%B3%DF%20%C4%D0%D7%D3%20%B8%DF%BF%D5%20%C9%CF%D1%DD
霸座男被列黑名单-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%B0%D4%D7%F9%C4%D0%B1%BB%C1%D0%BA%DA%C3%FB%B5%A5
手机扫码取厕纸-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%CA%D6%BB%FA%C9%A8%C2%EB%C8%A1%B2%DE%D6%BD
广东汕头积水渐退-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%B9%E3%B6%AB%C9%C7%CD%B7%BB%FD%CB%AE%BD%A5%CD%CB
辣条抽检不合格-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%C0%B1%CC%F5%B3%E9%BC%EC%B2%BB%BA%CF%B8%F1
独居老人家中身亡-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%B6%C0%BE%D3%C0%CF%C8%CB%BC%D2%D6%D0%C9%ED%CD%F6
为什么健身房内没人打架-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%CE%AA%CA%B2%C3%B4%BD%A1%C9%ED%B7%BF%C4%DA%C3%BB%C8%CB%B4%F2%BC%DC
河流 民众 抓鱼-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%BA%D3%C1%F7%20%C3%F1%D6%DA%20%D7%A5%D3%E3
梅西无缘最佳球员前三-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%C3%B7%CE%F7%CE%DE%D4%B5%D7%EE%BC%D1%C7%F2%D4%B1%C7%B0%C8%FD
美团员工举报同事-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%C3%C0%CD%C5%D4%B1%B9%A4%BE%D9%B1%A8%CD%AC%CA%C2
哈尔滨火灾原因-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%B9%FE%B6%FB%B1%F5%BB%F0%D4%D6%D4%AD%D2%F2
梅西 让点-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%C3%B7%CE%F7%20%C8%C3%B5%E3
特朗普侮辱加拿大-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%CC%D8%C0%CA%C6%D5%CE%EA%C8%E8%BC%D3%C4%C3%B4%F3
谷歌 诈骗广告-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%B9%C8%B8%E8%20%D5%A9%C6%AD%B9%E3%B8%E6
幼儿园现钢管舞-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%D3%D7%B6%F9%D4%B0%CF%D6%B8%D6%B9%DC%CE%E8
记者在缅7年监禁-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%BC%C7%D5%DF%D4%DA%C3%E57%C4%EA%BC%E0%BD%FB
蒲熠星 答题风格-->http://www.baidu.com/baidu?cl=3&tn=SE_baiduhomet8_jmjb7mjw&rsv_dl=fyb_top&fr=top1000&wd=%C6%D1%EC%DA%D0%C7%20%B4%F0%CC%E2%B7%E7%B8%F1
Process finished with exit code 0