JAVA爬数据


/**
* httClient 请求 GET
* 获取数据测试
*/
    @Test
public void testDepartmentList(){
//数据链接
String url = "https://search-x.jd.com/Search?callback=jQuery1352084&area=22&enc=utf-8&keyword=%E7%AC%94%E8%AE%B0%E6%9C%AC&adType=7&urlcid3=672&page=1&ad_ids=291%3A33&xtest=new_search&_=1585561488653";
//网页链接
String referer = "https://search.jd.com/search?keyword=%E7%AC%94%E8%AE%B0%E6%9C%AC&enc=utf-8&qrst=1&rt=1&stop=1&spm=2.1.1&vt=2&cid2=671&cid3=672&ev=exbrand_%E8%81%94%E6%83%B3%EF%BC%88Lenovo%EF%BC%89%5E&uc=0";
System.out.println("请求url:" + url);
//http请求
HttpRequest request = HttpUtil.createGet(url);

request.header("Referer", referer);
String str = request.execute().body();
//获取str的长度
int length = str.length();
//indexOf返回某个指定的字符串值在字符串中首次出现的位置
int i = str.indexOf("(");
System.out.println(i);
//截取字符串
str = str.substring(i + 1, length - 1);
System.out.println("请求响应:" + str);
//转换为Obj类型
JSONObject jsonObject = JSON.parseObject(str);
System.out.println(jsonObject);
//获取数组
JSONArray jsonArray = jsonObject.getJSONArray("291");
//计算数组的长度
int size = jsonArray.size();
//for循环遍历
for (int j = 0; j < size; j++) {
JSONObject jsonObject1 = jsonArray.getJSONObject(j);
String productName = jsonObject1.getString("ad_title");
Integer productPrice = jsonObject1.getInteger("pc_price");
// Integer productPrice = Integer.parseInt(productPrice1);
String imageUrl = jsonObject1.getString("image_url");
String productBrief = jsonObject1.getString("color");

Product product = new Product();
product.setProductName(productName);
product.setProductPrice(productPrice);
product.setImageUrl(imageUrl);
product.setProductBrief(productBrief);
DateTime dateTime = new DateTime();
System.out.println(dateTime);
//product.setCreateTime("dateTime");
productDao.saveProduct(product);
}
}

你可能感兴趣的:(JAVA爬数据)