Java爬虫：Jsoup利用dom方法遍历Document对象
先给出网页地址：
https://wall.alphacoders.com/featured.php?lang=Chinese
主要步骤：
- 利用Jsoup的connect方法获取Document对象
String html = "https://wall.alphacoders.com/featured.php?lang=Chinese";
Document doc = Jsoup.connect(html).get();
内容过长，就不再显示。
我们以这部分为例：
<ul class="nav nav-pills">
<li><a href="https://alphacoders.com/site/about-us" rel="nofollow">About Usa>li>
<li><a href="https://alphacoders.com/site/faq" rel="nofollow">FAQa>li>
<li><a href="https://alphacoders.com/site/privacy" rel="nofollow">Privacy Policya>li>
<li><a href="https://alphacoders.com/site/tos" rel="nofollow">Terms Of Servicea>li>
<li><a href="https://alphacoders.com/site/acceptable_use" rel="nofollow">Acceptable Usea>li>
<li><a href="https://alphacoders.com/site/etiquette" rel="nofollow">Etiquettea>li>
<li><a href="https://alphacoders.com/site/advertising" rel="nofollow">Advertise With Usa>li>
<li><a id="change_consent">Change Consenta>li>
ul>
- 我们先找到所有的ul：
Elements elements = doc.getElementsByTag("ul");
输出如下：
<ul class="nav navbar-nav center">
<li> <a title="Submit Wallpapers" href="https://alphacoders.com/site/submit-wallpaper"><i class="el el-circle-arrow-up">i> 提交a> li>
<li> <a href="https://alphacoders.com/contest"><i class="el el-gift">i> 精美奖品a> li>
ul>
<ul class="nav navbar-nav navbar-right center">
<li> <a href="language.php?lang=Chinese"> <img src="https://static.alphacoders.com/wa/Chinese-flag.png" alt="Chinese-flag"> 中文 a> li>
<li> <a rel="nofollow" href="https://alphacoders.com/users/login"><i class="el el-user">i> 登录a> li>
<li> <a href="https://alphacoders.com/users/register"><i class="el el-edit">i> 注册a> li>
ul>
<ul class="pagination">
<li class="active"><a id="prev_page" href="#">< 上一页a>li>
<li class="active"><a>1a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=2">2a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=3">3a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=4">4a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=5">5a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=6">6a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=7">7a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=8">8a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=9">9a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=10">10a>li>
<li><a>...a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=319">319a>li>
<li><a id="next_page" href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=2">下一页 >a>li>
ul>
<ul class="pagination">
<li class="active"><a href="#">< 上一页a>li>
<li class="active"><a>1a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=2">2a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=3">3a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=4">4a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=5">5a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=6">6a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=7">7a>li>
<li><a>...a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=319">319a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=2">下一页 >a>li>
ul>
<ul class="pagination">
<li class="active"><a href="#"><< a>li>
<li class="active"><a href="#">< 上一页a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=2">下一页 >a>li>
<li><a title="末页 (319)" href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=319"> >>a>li>
ul>
<ul class="pagination">
<li class="active"><a href="#">1a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=2">2a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=3">3a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=4">4a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=5">5a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=6">6a>li>
<li><a href="https://wall.alphacoders.com/featured.php?lang=Chinese&page=7">7a>li>
ul>
<ul class="nav nav-pills">
<li><a href="https://alphacoders.com/site/about-us" rel="nofollow">About Usa>li>
<li><a href="https://alphacoders.com/site/faq" rel="nofollow">FAQa>li>
<li><a href="https://alphacoders.com/site/privacy" rel="nofollow">Privacy Policya>li>
<li><a href="https://alphacoders.com/site/tos" rel="nofollow">Terms Of Servicea>li>
<li><a href="https://alphacoders.com/site/acceptable_use" rel="nofollow">Acceptable Usea>li>
<li><a href="https://alphacoders.com/site/etiquette" rel="nofollow">Etiquettea>li>
<li><a href="https://alphacoders.com/site/advertising" rel="nofollow">Advertise With Usa>li>
<li><a id="change_consent">Change Consenta>li>
ul>
- 可以发现class为"nav nav-pills"的只有一个，我们找到它：
Elements elements = doc.getElementsByTag("ul");
//System.out.println(elements);
Element tempElement = null;
for(Element element : elements) {
if (element.className().equals("nav nav-pills")) {
tempElement = element;
//System.out.println(element.className());
break;
}
}
- 循环遍历这个ul，输出其中每一个li里每一个a的href和rel属性：
Elements li = tempElement.getElementsByTag("li");
for(Element element : li) {
Elements element2 = element.getElementsByTag("a");
for(Element element3 : element2) {
String hrefString = element3.attr("href");
String relString = element3.attr("rel");
if(hrefString != "" && relString != "") {
System.out.println("href=" + hrefString + " " + "rel=" + relString);
}
}
}
最终结果：
href=https://alphacoders.com/site/about-us rel=nofollow
href=https://alphacoders.com/site/faq rel=nofollow
href=https://alphacoders.com/site/privacy rel=nofollow
href=https://alphacoders.com/site/tos rel=nofollow
href=https://alphacoders.com/site/acceptable_use rel=nofollow
href=https://alphacoders.com/site/etiquette rel=nofollow
href=https://alphacoders.com/site/advertising rel=nofollow
完整代码：
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
import org.jsoup.Jsoup;
/**
* @ClassName: Jsoup_Test
* @description:
* @author: KI
* @Date: 2020年8月17日 下午8:15:14
*/
public class Jsoup_Test {
public static void main(String[] args) throws IOException {
// TODO 自动生成的方法存根
String html = "https://wall.alphacoders.com/featured.php?lang=Chinese";
Document doc = Jsoup.connect(html).get();
System.out.println(doc);
Elements elements = doc.getElementsByTag("ul");
//System.out.println(elements);
Element tempElement = null;
for(Element element : elements) {
if (element.className().equals("nav nav-pills")) {
tempElement = element;
//System.out.println(element.className());
break;
}
}
System.out.println(tempElement);
Elements li = tempElement.getElementsByTag("li");
for(Element element : li) {
Elements element2 = element.getElementsByTag("a");
for(Element element3 : element2) {
String hrefString = element3.attr("href");
String relString = element3.attr("rel");
if(hrefString != "" && relString != "") {
System.out.println("href=" + hrefString + " " + "rel=" + relString);
}
}
}
}
}