爬虫
方法一
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.Iterator;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class Test {
public static void main(String[] args) throws IOException {
Document document = Jsoup.connect("https://www.doutula.com/article/list/?page=8").get();
//获取图片链接 逗号是同一级元素
Elements els = document.select(".lazy,.image_dtb,.img-responsive");//css选择器 id class tagName
Iterator iter = els.iterator();
while (iter.hasNext()) {
Element object = (Element) iter.next();
String imgUrl = object.attr("data-original");
//获取图片
if (!imgUrl.contains("http")) {
continue;
}
URL url = new URL(imgUrl);
URLConnection con = url.openConnection();
con.connect();
InputStream stream = con.getInputStream();
byte[] by = new byte[1024];
int length = -1;
//out的流
String[] imageUrlArr = imgUrl.split("/");
String fileName = "img/" + imageUrlArr[imageUrlArr.length - 1];
FileOutputStream fileOutputStream = new FileOutputStream(fileName);
while ((length = stream.read(by)) != -1) {
//把流写入到本地文件夹
fileOutputStream.write(by,0,length);
}
fileOutputStream.close();
stream.close();
}
}
}
方法二
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.Iterator;
import java.util.LinkedList;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class Test02 {
public static void main(String[] args) {
// 创建图片下载对象
final ImgDownload imgDownload = new ImgDownload();
// 生产图片链接的
new Thread(new Runnable() {
@Override
public void run() {
for (int i = 1; i < 500; i++) {
try {
imgDownload.parseImageUrl(i);
} catch (Exception e) {
// TODO: handle exception
}
}
}
}).start();
new Thread(new Runnable() {
@Override
public void run() {
for (int i = 500; i < 1000; i++) {
try {
imgDownload.parseImageUrl(i);
} catch (Exception e) {
// TODO: handle exception
}
}
}
}).start();
new Thread(new Runnable() {
@Override
public void run() {
for (int i = 1000; i < 1500; i++) {
try {
imgDownload.parseImageUrl(i);
} catch (Exception e) {
// TODO: handle exception
}
}
}
}).start();
// 下载
new Thread(new Runnable() {
@Override
public void run() {
try {
while (true) {
imgDownload.downloadImage();
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}).start();
new Thread(new Runnable() {
@Override
public void run() {
try {
while (true) {
imgDownload.downloadImage();
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}).start();
new Thread(new Runnable() {
@Override
public void run() {
try {
while (true) {
imgDownload.downloadImage();
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}).start();
new Thread(new Runnable() {
@Override
public void run() {
try {
while (true) {
imgDownload.downloadImage();
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}).start();
new Thread(new Runnable() {
@Override
public void run() {
try {
while (true) {
imgDownload.downloadImage();
}
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}).start();
}
}
class ImgDownload {
// imgUrl的数组
LinkedList list = new LinkedList<>();
// 解析链接
void parseImageUrl(int page) throws Exception {
synchronized (this) {
if (list.size() > 100) {
wait();
}
String htmlUrl = "https://www.doutula.com/article/list/?page=" + page;
Document document = Jsoup.connect(htmlUrl).get();
// 获取图片链接 逗号是同一级元素
Elements els = document.select(".lazy,.image_dtb,.img-responsive");// css选择器
// id
// class
// tagName
Iterator iter = els.iterator();
while (iter.hasNext()) {
Element object = (Element) iter.next();
String imgUrl = object.attr("data-original");
if (!imgUrl.contains("http")) {//如果不包含,则跳过
continue;
}
this.list.add(imgUrl);
}
// 通知可以下载图片
notifyAll();
}
}
// 下载图片
void downloadImage() throws Exception {
synchronized (this) {
if (this.list.size() <= 1) {
wait();
}
// String imgUrl = null;
// if (this.list.size() > 0) {
// imgUrl = this.list.removeFirst();
// } else {
// return;
// }
String imgUrl = this.list.removeFirst();
URL url = new URL(imgUrl);
URLConnection con = url.openConnection();
con.connect();
InputStream stream = con.getInputStream();
byte[] by = new byte[1024];
int length = -1;
// out的流
String[] imageUrlArr = imgUrl.split("/");
String fileName = "img/" + imageUrlArr[imageUrlArr.length - 1];
FileOutputStream fileOutputStream = new FileOutputStream(fileName);
while ((length = stream.read(by)) != -1) {
// 把流写入到本地文件夹
fileOutputStream.write(by, 0, length);
}
fileOutputStream.close();
stream.close();
if (this.list.size() < 100) {
notify();
//notifyAll();
}
}
}
}