java娱乐-利用JXBrower进行网页数据抓取
java进行web网页抓取最常见的方式,老司机们应该都知道,那就是使用JSOUP.这个工具之前也有玩过,但是发现一个问题,比如你需要去抓取的数据,必须要登录才能抓取,如果涉及到验证码,就更难操作了.
很早以前,就用jsoup尝试实现自动下载彼岸图网的高清图,但是他们的登录方式只有第三方登录,这样只用jsoup就会很难实现.
后面发现了JXbrower工具,他可以很轻松的实现这样的需求.
下面通过抓取彼岸图网的案例开始详细介绍JX.
首先下载
https://jxbrowser.support.teamdev.com/support/home
具体如何下载这里就不详细说明了,注意一点我这里用的是 6.18版本,一定要使用6.18版本
将下载下来的压缩包解压,
获取如下两个jar包
jxbrowser-6.18.jar
jxbrowser-win32-6.18.jar
项目结构如下:
新建文件 teamdev.licenses
Product: JxBrowser
Version: 6.x
Licensed to:
License type: Enterprise
License info: JxBrowser License
Expiration date: 01-01-9999
Support expiration date: NO SUPPORT
Generation date: 01-01-1970
Platforms: win32/x86;win32/x64;mac/x86;mac/x64;linux/x86;linux/x64
Company name: TeamDev Ltd.
SigB: 1
SigA: 1
main代码
这里提一下,博主装过WindowBuilder插件了,具体使用参考如下链接
https://blog.csdn.net/xiaoxiao123jun/article/details/77330734
package code;
import java.awt.BorderLayout;
import java.awt.EventQueue;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.math.BigInteger;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import javax.swing.JFrame;
import com.teamdev.jxbrowser.chromium.Browser;
import com.teamdev.jxbrowser.chromium.DownloadHandler;
import com.teamdev.jxbrowser.chromium.DownloadItem;
import com.teamdev.jxbrowser.chromium.az;
import com.teamdev.jxbrowser.chromium.dom.By;
import com.teamdev.jxbrowser.chromium.dom.DOMElement;
import com.teamdev.jxbrowser.chromium.events.DownloadEvent;
import com.teamdev.jxbrowser.chromium.events.DownloadListener;
import com.teamdev.jxbrowser.chromium.events.LoadAdapter;
import com.teamdev.jxbrowser.chromium.events.StatusEvent;
import com.teamdev.jxbrowser.chromium.events.StatusListener;
import com.teamdev.jxbrowser.chromium.swing.BrowserView;
import javax.swing.JButton;
import java.awt.Button;
import javax.swing.JMenuBar;
import javax.swing.JMenu;
import javax.swing.JPanel;
import java.awt.event.ActionListener;
import java.awt.event.ActionEvent;
import java.awt.event.MouseAdapter;
import java.awt.event.MouseEvent;
import java.io.File;
import java.util.concurrent.*;
public class Windos {
private JFrame frame;
/**
* Launch the application.
*/
static {
try {
Field e = az.class.getDeclaredField("e");
e.setAccessible(true);
Field f = az.class.getDeclaredField("f");
f.setAccessible(true);
Field modifersField = Field.class.getDeclaredField("modifiers");
modifersField.setAccessible(true);
modifersField.setInt(e, e.getModifiers() & ~Modifier.FINAL);
modifersField.setInt(f, f.getModifiers() & ~Modifier.FINAL);
e.set(null, new BigInteger("1"));
f.set(null, new BigInteger("1"));
modifersField.setAccessible(false);
} catch (Exception e1) {
e1.printStackTrace();
}
}
public static void main(String[] args) {
EventQueue.invokeLater(new Runnable() {
public void run() {
try {
Windos window = new Windos();
window.frame.setVisible(true);
} catch (Exception e) {
e.printStackTrace();
}
}
});
}
/**
* Create the application.
*/
public Windos() {
initialize();
}
/**
* Initialize the contents of the frame.
*/
volatile boolean flag=true;
int i = 194;
private void initialize() {
frame = new JFrame();
frame.getContentPane().setEnabled(false);
final String url = "http://pic.netbian.com/tupian/1.html";
final String title = "彼岸抓图工具";
//不显示标题栏,最大化,最小化,退出按钮
//frame.setUndecorated(true);
frame.setSize(1500, 1500);
frame.getContentPane().setLayout(null);
frame.setExtendedState(JFrame.MAXIMIZED_BOTH);
frame.setLocationByPlatform(true);
frame.setVisible(true);
Browser browser = new Browser();
BrowserView view = new BrowserView(browser);
view.setBounds(152, 39, 1322, 989);
frame.getContentPane().add(view);
Lock lock= new ReentrantLock();;
JButton button = new JButton("开始抓图");
browser.setDownloadHandler(new DownloadHandler() {
//下载事件监听
public boolean allowDownload(DownloadItem download) {
File file = new File("D:/file/"+i+download.getDestinationFile().getName());
download.setDestinationFile(file);
download.addDownloadListener(new DownloadListener() {
public void onDownloadUpdated(DownloadEvent event) {
DownloadItem download = event.getDownloadItem();
File file = new File("D:/file/"+download.getDestinationFile().getName());
download.setDestinationFile(file);
if (download.isCompleted()) {
System.out.println("Download is completed!");
}
}
});
System.out.println("Dest file: " + download.getDestinationFile().getAbsolutePath());
return true;
}
});
button.addMouseListener(new MouseAdapter() {
@Override
public void mouseClicked(MouseEvent e) {
for (; i <50000; i++) {
browser.loadURL("http://pic.netbian.com/tupian/"+i+".html");
//执行JS的代码
//browser.executeJavaScript(jString);
//添加加载监听
browser.addLoadListener(new LoadAdapter() {
@Override
public void onFinishLoadingFrame(com.teamdev.jxbrowser.chromium.events.FinishLoadingEvent event) {
while (true) {
try {
DOMElement findElement = event.getBrowser().getDocument().findElement(By.id("img"));
findElement.click();
break;
} catch (Exception e2) {
e2.printStackTrace();
try {
Thread.sleep(60*1000*5);
System.err.println("下载出错");
browser.loadURL("http://pic.netbian.com/tupian/"+i+".html");
DOMElement findElement = event.getBrowser().getDocument().findElement(By.id("img"));
findElement.click();
break;
} catch (Exception e) {
e.printStackTrace();
}
}
}
};
@Override
public void onDocumentLoadedInFrame(com.teamdev.jxbrowser.chromium.events.FrameLoadEvent event) {
};
});
try {
Thread.sleep(15000);
} catch (InterruptedException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
}
}
});
button.setBounds(49, 311, 93, 23);
frame.getContentPane().add(button);
browser.loadURL(url);
}
}
效果如下
这只是一个简单的案例,更强大的功能大家可以去看他的API.