获取chrome的network内容并选择下载其中的资源
现在存在问题：要将网站反馈回来的get信号的内容获取到并下载资源
eg：
chrome+键F12可见的内容
解决方式与思路：
方法：
- 使用webDriver或者selenium模拟浏览器传输信号，再截取返回回来的信号（这里使用的就是这个方法）。
- 使用selenium与browsermob-proxy；browsermob-proxy可以获取http内容导出为HAR文件。
思路：
3. 用selenium的api获取其中里面的message
4. 其中存在将请求变成log的文件的api( LogEntries logs = driver.manage().logs().get(LogType.PERFORMANCE);
)，该文件是个json文件,可以使用json来解析获取想要的信息以及其的链接
5. 使用URL获取文件流，下载文件
代码如下：
package test;
import java.io.*;
import java.net.URL;
import java.util.Iterator;
import java.util.logging.Level;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.json.JSONException;
import org.json.JSONObject;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeDriverService;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.logging.LogEntries;
import org.openqa.selenium.logging.LogEntry;
import org.openqa.selenium.logging.LogType;
import org.openqa.selenium.logging.LoggingPreferences;
import org.openqa.selenium.remote.CapabilityType;
import org.openqa.selenium.remote.DesiredCapabilities;
/**
* chromedriver
* dev-tool
*/
public class ChromedriverNetwork {
public static final String port = "1527";
public static final String filePath = "C:\\Users\\28124\\Desktop\\测试自动化\\自动化项目\\driver\\chromedriver.exe";
public static void main(String[] args) {
String sessionId ;
String url = "http://www.ting56.com/video/18743-0-3.html";
//System.setProperty("webdriver.chrome.driver",filePath);
ChromeDriver driver = null;
try {
ChromeOptions options = new ChromeOptions();
DesiredCapabilities cap = new DesiredCapabilities();
ChromeDriverService service = new ChromeDriverService.Builder().usingPort(Integer.valueOf(port))
.usingDriverExecutable(new File(filePath))
.build();
options.addArguments("disable-infobars");
cap.setCapability(ChromeOptions.CAPABILITY, options);
//配置日志
LoggingPreferences logPrefs = new LoggingPreferences();
logPrefs.enable(LogType.PERFORMANCE, Level.ALL);
/**flash 扩展*/
cap.setCapability("profile.managed_default_content_settings.images",1);
cap.setCapability("profile.content_settings.plugin_whitelist.adobe-flash-player",1);
cap.setCapability("profile.content_settings.exceptions.plugins.*,*.per_resource.adobe-flash-player",1);
cap.setCapability(CapabilityType.LOGGING_PREFS, logPrefs);
driver = new ChromeDriver(service, cap);
//or driver.get(url)
driver.navigate().to(url);
// System.out.println("session id :" + driver.getSessionId());
sessionId = driver.getSessionId().toString();
LogEntries logs = driver.manage().logs().get(LogType.PERFORMANCE);
for (Iterator<LogEntry> it = logs.iterator(); it.hasNext();) {
LogEntry entry = it.next();
try {
//System.out.println(entry.toString());
JSONObject json = new JSONObject(entry.getMessage());
// System.out.println(json.toString());
JSONObject message = json.getJSONObject("message");
String method = message.getString("method");
//如果是响应
if (method != null && "Network.responseReceived".equals(method)) {
JSONObject params = message.getJSONObject("params");
JSONObject response = params.getJSONObject("response");
String messageUrl = response.getString("url");
// System.out.println("-----------------------------");
if (params.toString().contains("https://od.qingting.fm/m4a/")) {
InputStream in=new URL(messageUrl).openConnection().getInputStream(); //创建连接、输入copy流
FileOutputStream f = new FileOutputStream("C:\\Users\\28124\\Desktop\\测试自动化\\自动化项目\\file2\\a.m4a");//创建文百件度输出流
byte [] bb=new byte[1024]; //接收缓存
int len;
while( (len=in.read(bb))>0){ //接收
f.write(bb, 0, len); //写入问文件
}
f.close();
in.close();
}
// System.err.println("url:" + url + " params: " + params.toString() + " response: " + response.toString());
//打印调用chromedriver 调chrome httpapi 结果
// System.out.println(getResponseBody(params.getString("requestId"),port,sessionId));
// System.out.println("-----------------------------");
}
} catch (Exception e) {
e.printStackTrace();
}
}
} catch (Exception e) {
e.printStackTrace();
}finally{
if (driver != null)
{
driver.quit();
}
}
}
// 根据请求ID获取返回内容
public static String getResponseBody(String requestId,String port,String sessionId) {
try {
// CHROME_DRIVER_PORT chromeDriver提供的端口
String url = String.format("http://localhost:%s/session/%s/goog/cdp/execute",
port, sessionId);
HttpPost httpPost = new HttpPost(url);
JSONObject object = new JSONObject();
JSONObject params = new JSONObject();
params.put("requestId", requestId);
//api https://chromedevtools.github.io/devtools-protocol/tot/Network
object.put("cmd", "Network.getResponseBody");
object.put("params", params);
httpPost.setEntity(new StringEntity(object.toString()));
RequestConfig requestConfig = RequestConfig
.custom()
.setSocketTimeout(60 * 1000)
.setConnectTimeout(60 * 1000).build();
CloseableHttpClient httpClient = HttpClientBuilder.create()
.setDefaultRequestConfig(requestConfig).build();
HttpResponse response = httpClient.execute(httpPost);
return EntityUtils.toString(response.getEntity());
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
}
总结：
- 如果使用思路2，可以参考链接
https://blog.csdn.net/m0_37618247/article/details/85066272
参考链接：
https://blog.csdn.net/qq_36783371/article/details/100122599