java简单爬虫实现打印小说章节至控制台
最近公司比较闲，就想着研究研究爬虫，因为平时爱看点小说，那就对小说下手吧。话不多少，上源码
目录结构：
主处理方法：
package controller;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.util.*;
import org.apache.cxf.binding.corba.wsdl.Object;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import util.Browser;
import util.StringUtils;
/**
* Created by lenovo on 2018/11/29.
*/
public class test {
/**
* @param args
* @throws Exception
*/
public static void main(String[] args) throws Exception{
Map map = read();
String baseUrl = map.get("baseUrl");
String child = map.get("child");
if (baseUrl==null||baseUrl.equals("")){
run(search(),null);
}else{
run(baseUrl,child);
}
}
public static void run(String baseUrl,String child) throws Exception {
String chrildUrl = child;
try {
Document document = null;
if (child!=null&&!child.equals("")){
document = connect(baseUrl+chrildUrl);
content(document);
}else{
chrildUrl = menu(baseUrl);
document = connect(baseUrl+chrildUrl);
content(document);
}
while (true){
System.out.println(StringUtils.chooseMenu);
int result = choose();
switch (result){
case 1:
chrildUrl = lastUrl(document);
document = connect(baseUrl+chrildUrl);
content(document);
save("",chrildUrl);
break;
case 2:
chrildUrl = menu(baseUrl);
document = connect(baseUrl+chrildUrl);
content(document);
save("",chrildUrl);
break;
case 3:
chrildUrl = nextUrl(document);
document = connect(baseUrl+chrildUrl);
content(document);
save("",chrildUrl);
break;
case 4:
document = connect(baseUrl+chrildUrl);
content(document);
break;
}
}
}catch (Exception e){
try {
Document document = connect(baseUrl+chrildUrl);
content(document);
while (true){
System.out.println(StringUtils.chooseMenu);
int result = choose();
switch (result){
case 1:
chrildUrl = lastUrl(document);
document = connect(baseUrl+chrildUrl);
content(document);
save("",chrildUrl);
break;
case 2:
chrildUrl = menu(baseUrl);
document = connect(baseUrl+chrildUrl);
content(document);
save("",chrildUrl);
break;
case 3:
chrildUrl = nextUrl(document);
document = connect(baseUrl+chrildUrl);
content(document);
save("",chrildUrl);
break;
case 4:
document = connect(baseUrl+chrildUrl);
content(document);
break;
}
}
}catch (Exception el){
save("",chrildUrl);
el.printStackTrace();
System.out.print(el);
}
}
}
public static Document connect(String url) throws Exception {
Document document=Jsoup.connect(url)
.userAgent(Browser.Chrome_17$0_MAC)
.header("Connection", "close")//如果是这种方式，这里务必带上
.timeout(8000)//超时时间
.get();
return document;
}
public static Document getIndex() throws Exception {
return connect(StringUtils.indexUrl);
}
public static String menu(String url) throws Exception{
Document document = connect(url);
Elements mulu = document.select(".mulu").select("a[href]");
List> menu = new ArrayList<>();
for (int i = 0;i map = new HashMap<>();
map.put("url",mulu.get(i).attr("href"));
map.put("menu",mulu.get(i).text());
menu.add(map);
}
int state = choose();
return menu.get(state).get("url");
}
public static String search() throws Exception{
System.out.println("请输入内容：");
String key = choose(null);
Document document=Jsoup.connect(StringUtils.searchUrl).data("q",key)
.userAgent(Browser.Chrome_17$0_MAC)
.header("Connection", "close")//如果是这种方式，这里务必带上
.timeout(8000)//超时时间
.get();
Elements elements = document.select("h2").select("a[href]");
for (int i = 0;i","");
System.out.println(content);
}
public static String nextUrl(Document document){
Elements menuDiv = document.select(".pereview");
String url = menuDiv.get(0).select("a[href]").get(2).select("a[href]").attr("href");
return url;
}
public static String lastUrl(Document document){
Elements menuDiv = document.select(".pereview");
String url = menuDiv.get(0).select("a[href]").get(0).select("a[href]").attr("href");
return url;
}
public static int choose(){
Scanner sc = new Scanner(System.in);
return sc.nextInt();
}
public static String choose(Object ...param){
Scanner sc = new Scanner(System.in);
return sc.nextLine();
}
public static void save(String baseUrl,String child){
Properties prop = new Properties();
try{
FileOutputStream oFile = new FileOutputStream("resources/url.properties", true);//true表示追加打开
if (baseUrl!=null&&baseUrl!="") {
prop.setProperty("baseUrl", baseUrl);
}
if (child!=null&&child!=""){
prop.setProperty("child", child);
}
prop.store(oFile, "The New properties file");
oFile.close();
}catch (Exception e){
System.out.println(e);
}
}
public static Map read(){
Properties prop = new Properties();
try{
Map map = new HashMap();
//读取属性文件a.properties
InputStream in = new BufferedInputStream (new FileInputStream("resources/url.properties"));
prop.load(in); ///加载属性列表
String baseUrl = prop.getProperty("baseUrl");
String child = prop.getProperty("child");
map.put("baseUrl",baseUrl);
map.put("child",child);
in.close();
return map;
}catch (Exception e){
System.out.println(e);
}
return null;
}
} 工具类：
package util;
/**
* Created by lenovo on 2018/12/5.
*/
public class StringUtils {
public static final String indexUrl = "https://www.88dush.com";
public static final String searchUrl = "https://so.88dush.com/search/so.php";
public static final String chooseMenu = "1:last;2:menu;3:next;4:refresh";
}
package util;
/**
* 浏览器的UserAgent
* Created by dongxiaoqi on 2018/11/30.
*/
public class Browser {
public static final String safari_5$1_MAC = "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50";
public static final String safari_5$1_Windows = "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50";
public static final String Firefox_38esr = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0";
public static final String IE11 = "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko";
public static final String IE9 = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)";
public static final String IE8 = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)";
public static final String IE7 = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)";
public static final String IE6 = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
public static final String Firefox_4$0$1_MAC = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1";
public static final String Firefox_4$0$1_Windows = "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1";
/**
* 欧朋
*/
public static final String Opera_11$11_MAC = "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11";
public static final String Opera_11$11_Windows = "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11";
public static final String Chrome_17$0_MAC = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11";
/**
* 遨游浏览器
*/
public static final String Maxthon = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)";
public static final String TencentTT = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)";
/**
* 世界之窗
*/
public static final String The_World_2X = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)";
public static final String The_World_3X = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)";
public static final String sougou = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)";
public static final String SE360 = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)";
public static final String Avant = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)";
public static final String Green_Browser = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)";
}
url.properties 这个是记录当前阅读位置的文件，方便下次运行时接续上次位置
#The New properties file
#Wed Dec 05 11:33:17 CST 2018
baseUrl=https\://www.88dush.com/xiaoshuo/35/35954/
#The New properties file
#Wed Dec 05 11:57:44 CST 2018
#The New properties file
#Thu Dec 06 08:31:59 CST 2018
#The New properties file
#Fri Dec 07 13:37:23 CST 2018
child=11384708.html
#The New properties file
#Fri Dec 07 13:51:15 CST 2018
#The New properties file
#Fri Dec 07 13:52:20 CST 2018
写在最后：本次爬取目标站为88dush，挺棒的免费小说网，网址：https://www.88dush.com，如果被认为侵权请联系我删除，没看到就这么挂着吧 哈哈