java爬周边信息解决方案
日前用java爬虫去爬取某些地理信息周边信息的时候，发现大部分网站的周边信息都是预加载页面，用js去调API来实现，不同网站的具体实现方式还不同。所以我这边的解决方案为，先根据所搜目标的名称以及所在区域去调用百度API，获取经纬度，再根据经纬度去搜索周边信息，具体查看百度地图提供的公有API http://lbsyun.baidu.com/index.php?title=webapi
package com.jrzh.thread;
import java.math.BigDecimal;
import java.net.URLEncoder;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.alibaba.fastjson.JSONObject;
import com.jrzh.bean.jwd.JwdBean;
import com.jrzh.bean.jwd.JwdInfo;
import com.jrzh.common.exception.ProjectException;
import com.jrzh.framework.bean.SessionUser;
import com.jrzh.mvc.model.reptile.ReptileNewHouseModel;
import com.jrzh.mvc.service.reptile.manager.ReptileServiceManage;
import com.jrzh.tools.MyPayUtils;
public class LatitudeLongitudeThread extends Thread {
static Log log = LogFactory.getLog(LatitudeLongitudeThread.class);
private ReptileServiceManage reptileServiceManage;
public LatitudeLongitudeThread(ReptileServiceManage reptileServiceManage) {
this.reptileServiceManage = reptileServiceManage;
}
@Override
public void run() {
// 更新坐标方法
Object object = new Object();
synchronized (object) {
List reptileNewHouseModelList;
try {
// 查找所有带名称的实例
reptileNewHouseModelList = reptileServiceManage.reptileNewHouseService.findAll();
for (ReptileNewHouseModel reptileNewHouseModel : reptileNewHouseModelList) {
if (reptileNewHouseModel.getLatitude() != null && reptileNewHouseModel.getLongitude() != null) {
// 如果已经有经纬度，直接跳过
continue;
}
try {
// 具体获取坐标方法
String lat = null;
String lng = null;
List list = getJwd(reptileNewHouseModel.getHouseCity(),
reptileNewHouseModel.getHouseName());
if (list == null) {
if(reptileNewHouseModel.getHouseOtherName()==null) continue;
list = getJwd(reptileNewHouseModel.getHouseCity(),reptileNewHouseModel.getHouseOtherName());
if(list==null) continue;
}
String[] strs = getJwd(list, reptileNewHouseModel.getHousePart(),
reptileNewHouseModel.getHouseName());
lat = strs[0];
lng = strs[1];
log.info(lat + "----" + lng);
if (lat != null && lng != null) {
reptileNewHouseModel.setLatitude(new BigDecimal(lat));
reptileNewHouseModel.setLongitude(new BigDecimal(lng));
reptileServiceManage.reptileNewHouseService.edit(reptileNewHouseModel,
SessionUser.getSystemUser());
}
Thread.sleep(300);
} catch (Exception e) {
e.printStackTrace();
continue;
}
}
log.info("一手房经纬度结束");
} catch (ProjectException e1) {
e1.printStackTrace();
}
}
}
public static List getJwd(String city, String position) {
String[] appKeyArr = { "key的数组" };
Integer index = 0;
String requestContent = null;
try {
// 具体获取坐标方法
position = URLEncoder.encode(position, "utf-8");
String appkey = appKeyArr[index];
// 调用url
String url = "http://api.map.baidu.com/place/v2/suggestion?query=" + position + "®ion=" + city
+ "&city_limit=true&output=json&ak=" + appkey;
requestContent = MyPayUtils.sendHttpByGet(url, 2000000, 20000000);
JwdBean jwdBean = JSONObject.parseObject(requestContent, JwdBean.class); // 获取返回信息
if (StringUtils.equals(jwdBean.getMessage(), "302")) {
appkey = appKeyArr[index++];
url = "http://api.map.baidu.com/place/v2/suggestion?query=" + position + "®ion=" + city
+ "&city_limit=true&output=json&ak=" + appkey;
requestContent = MyPayUtils.sendHttpByGet(url, 2000000, 20000000);
jwdBean = JSONObject.parseObject(requestContent, JwdBean.class);
}
if (StringUtils.equals(jwdBean.getMessage(), "ok")) {
List info = jwdBean.getResult();
return info;
}
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
public static String[] getJwd(List list, String part, String houserName) {
String lat = null;
String lng = null;
for (JwdInfo info : list) {
if (info.getDistrict().contains(part)) {
if (StringUtils.equals(info.getName(), houserName)) {
lat = info.getLocation().getLat();
lng = info.getLocation().getLng();
return new String[] { lat, lng };
}
}
}
for (JwdInfo info : list) {
if (info.getDistrict().contains(part)) {
if (info.getName().contains(houserName)) {
lat = info.getLocation().getLat();
lng = info.getLocation().getLng();
return new String[] { lat, lng };
}
}
}
for (JwdInfo info : list) {
lat = info.getLocation().getLat();
lng = info.getLocation().getLng();
return new String[] { lat, lng };
}
return null;
}
}
获取完经纬度之后，再用一个定时器去根据经纬度爬取周边
package com.jrzh.thread;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.alibaba.fastjson.JSONObject;
import com.jrzh.bean.testBean;
import com.jrzh.common.exception.ProjectException;
import com.jrzh.contants.Contants;
import com.jrzh.framework.bean.SessionUser;
import com.jrzh.mvc.model.reptile.ReptileNewHouseModel;
import com.jrzh.mvc.search.reptile.ReptileNewHouseSearch;
import com.jrzh.mvc.service.reptile.manager.ReptileServiceManage;
import com.jrzh.tools.MyPayUtils;
public class FindNearByThread extends Thread {
static Log log = LogFactory.getLog(FindNearByThread.class);
private ReptileServiceManage reptileServiceManage;
public FindNearByThread(ReptileServiceManage reptileServiceManage) {
this.reptileServiceManage = reptileServiceManage;
}
@Override
public void run() {
// 获取周边方法
Object object = new Object();
synchronized (object) {
ReptileNewHouseSearch rnhSearch = new ReptileNewHouseSearch();
List list = new ArrayList();
try {
// 获取所有房
list = reptileServiceManage.reptileNewHouseService.list(rnhSearch);
} catch (ProjectException e) {
log.error("gg");
}
for (ReptileNewHouseModel model : list) {
try {
this.getNearBy(model);
} catch (Exception e) {
continue;
}
}
}
}
public void getNearBy(ReptileNewHouseModel model) {
if (model.getLatitude() == null || model.getLongitude() == null) {
return; //没有经纬度就跳过
}
String[] appKeyArr = { "key数组" };
Integer index = 0;
String requestContent = null;
StringBuffer postKD = new StringBuffer();
StringBuffer postYZ = new StringBuffer();
StringBuffer subway = new StringBuffer();
StringBuffer bus = new StringBuffer();
StringBuffer hospital = new StringBuffer();
StringBuffer school = new StringBuffer();
StringBuffer kindergarten = new StringBuffer();
StringBuffer bank = new StringBuffer();
StringBuffer sc = new StringBuffer();
StringBuffer cs = new StringBuffer();
String latitude = String.valueOf(model.getLatitude());
String longitude = String.valueOf(model.getLongitude());
String radius = "2000"; // 距离
String outPutType = "json"; // 格式
String appkey = appKeyArr[index];
// 遍历所有搜索类别
for (String string : Contants.NEAR_BY_LIST) {
String query = string; // 搜索條件
String url = "http://api.map.baidu.com/place/v2/search?query=" + query + "&location=" + latitude + ","
+ longitude + "&radius=" + radius + "&output=" + outPutType + "&ak=" + appkey;
try {
requestContent = MyPayUtils.sendHttpByGet(url, 2000000, 20000000);
} catch (ProjectException e) {
e.printStackTrace();
continue;
}
// 无搜索结果
if (StringUtils.isBlank(requestContent)) {
System.out.println("=============");
continue;
}
JSONObject j = JSONObject.parseObject(requestContent);
List ss = JSONObject.parseArray(j.get("results") + "", testBean.class);
// 生成字段
for (testBean tt : ss) {
try {
if (StringUtils.equals(string, "银行")) {
bank.append(tt.getName() + "、");
} else if (StringUtils.equals(string, "医院")) {
hospital.append(tt.getName() + "、");
} else if (StringUtils.equals(string, "商场")) {
if (StringUtils.isBlank(sc.toString())) {
sc.append("商场：");
}
sc.append(tt.getName() + "、");
} else if (StringUtils.equals(string, "超市")) {
if (StringUtils.isBlank(cs.toString())) {
cs.append("超市：");
}
cs.append(tt.getName() + "、");
} else if (StringUtils.equals(string, "邮政")) {
if (StringUtils.isBlank(postYZ.toString())) {
postYZ.append("邮政：");
}
postYZ.append(tt.getName() + "、");
} else if (StringUtils.equals(string, "快递")) {
if (StringUtils.isBlank(postKD.toString())) {
postKD.append("快递：");
}
postKD.append(tt.getName() + "、");
} else if (StringUtils.equals(string, "学校")) {
school.append(tt.getName() + "、");
} else if (StringUtils.equals(string, "幼儿园")) {
kindergarten.append(tt.getName() + "、");
} else if (StringUtils.equals(string, "公交")) {
if (StringUtils.isBlank(bus.toString())) {
bus.append("公交：");
}
bus.append(tt.getName() + "、");
} else if (StringUtils.equals(string, "地铁")) {
if (StringUtils.isBlank(subway.toString())) {
subway.append("地铁：");
}
subway.append(tt.getName() + "、");
}
} catch (Exception e) {
e.printStackTrace();
continue;
}
}
}
// 存库
try {
if (StringUtils.isNotBlank(kindergarten.toString())) {
model.setKindergarten(kindergarten.substring(0, kindergarten.length() - 1));
}
if (StringUtils.isNotBlank(bank.toString())) {
model.setBank(bank.substring(0, bank.length() - 1));
}
if (StringUtils.isNotBlank(hospital.toString())) {
model.setHospital(hospital.substring(0, hospital.length() - 1));
}
if (StringUtils.isNotBlank(school.toString())) {
model.setSchool(school.substring(0, school.length() - 1));
}
if (StringUtils.isNotBlank(cs.toString()) && StringUtils.isNotBlank(sc.toString())) {
model.setPowerCenter(cs.substring(0, cs.length() - 1) + " " + sc.substring(0, sc.length() - 1));
} else if (StringUtils.isNotBlank(cs.toString()) && StringUtils.isBlank(sc.toString())) {
model.setPowerCenter(cs.substring(0, cs.length() - 1));
} else if (StringUtils.isBlank(cs.toString()) && StringUtils.isNotBlank(sc.toString())) {
model.setPowerCenter(sc.substring(0, sc.length() - 1));
}
if (StringUtils.isNotBlank(bus.toString()) && StringUtils.isNotBlank(subway.toString())) {
model.setTraffic(bus.substring(0, bus.length() - 1) + " " + subway.substring(0, subway.length() - 1));
} else if (StringUtils.isNotBlank(bus.toString()) && StringUtils.isBlank(subway.toString())) {
model.setTraffic(bus.substring(0, bus.length() - 1));
} else if (StringUtils.isBlank(bus.toString()) && StringUtils.isNotBlank(subway.toString())) {
model.setTraffic(subway.substring(0, subway.length() - 1));
}
if (StringUtils.isNotBlank(postYZ.toString()) && StringUtils.isNotBlank(postKD.toString())) {
model.setPost(
postYZ.substring(0, postYZ.length() - 1) + " " + postKD.substring(0, postKD.length() - 1));
} else if (StringUtils.isNotBlank(postYZ.toString()) && StringUtils.isBlank(postKD.toString())) {
model.setPost(postYZ.substring(0, postYZ.length() - 1));
} else if (StringUtils.isBlank(postYZ.toString()) && StringUtils.isNotBlank(postKD.toString())) {
model.setPost(postKD.substring(0, postKD.length() - 1));
}
reptileServiceManage.reptileNewHouseService.edit(model, SessionUser.getSystemUser());
log.info(model.getHouseName() + "更新周边信息！");
} catch (Exception e) {
e.printStackTrace();
return;
}
}
}