使用Jsoup爬取美团酒店信息

Jsoup

jsoup 是一款Java 的HTML解析器，可直接解析某个URL地址、HTML文本内容。它提供了一套非常省力的API，可通过DOM，CSS以及类似于jQuery的操作方法来取出和操作数据。使用细节可参考官方api

爬取的过程

首先我们先看一下我们爬取的信息，打开网页按下F12，找到我们需要的信息的标签

jsoup使用起来非常简单

	public static void testName11() throws Exception {
		Document document = Jsoup.connect("https://hotel.meituan.com/hefei/").get();//需要爬取的网页地址
			Elements select = document.select("h3.poi-title-wrapper");
			for (Element element : select ) {   //遍历打印class=poi-title-wrapper的h3内所含字符
				System.out.println(element.text());
			}
	}

现在我们就很简单的吧酒店的名称给扒下来啦

但是呢我发现一个问题，在爬取第二页信息时候失败了！！！！！！

好吧，原来是美团采用的ajax做的分页处理，我们需要去获取后台发来json串。

找到从后获取json数据的连接
https://ihotel.meituan.com/hbsearch/HotelSearch?utm_medium=pc&version_name=999.9&cateId=20&attr_28=129&uuid=B8FD28413E610560110C17A9A5BD0A3C2585A41BBCAE31C79F298C0C57643E0D%401565538810885&cityId=56&offset=20&limit=20&startDay=20190811&endDay=20190811&q=&sort=defaults&X-FOR-WITH=OiVIQkyPhSIUyTMoFpmIYbkJZrDEzCji0daGarSOo3MV9WAgYcfaCahshMCcg4rc5zjk5CvtS%2FOU1ANlS5scz96yNNZQdT74MytGZLKkhv8K0S%2Bb7pO%2ByeAxRMdQhbShelYsaRKhjOl3gWAX0ttPRQ%3D%3D
URL中有两个数offset=20&limit=20，offset（从第几条开始取），limit（每次取多少条）
这样我们就可以自己拼接URL进行分页获取酒店信息啦

private static List list = new ArrayList();
public static void testName11() throws Exception {
		int set = 0;
		for (int i = 0; i < 90; i++) {
			String url = "https://ihotel.meituan.com/hbsearch/HotelSearch?utm_medium=pc&version_name=999.9&cateId=20&attr_28=129&uuid=B8FD28413E610560110C17A9A5BD0A3C2585A41BBCAE31C79F298C0C57643E0D%401565455315987&cityId=56&offset="
					+ set
					+ "&limit=20&startDay=20190811&endDay=20190811&q=&sort=defaults&X-FOR-WITH=Do9KNx%2FedD5nwxAo%2BdGSoOmVeVudNYEFoGBIEoZox4M5x%2F9tJrgpb1uTwm7u0VvpODkpLGfnvLhYh2zkv%2Fu%2BgONqJ9mSSDBwteqsYGJdu6cEA8xgYEse8A6%2FH86LGZIpGhbdkzOnE%2BA%2Fp2cakNGesg%3D%3D";
			Document document = Jsoup.connect(url).ignoreContentType(true)
					.get();
			Elements select = document.select("body");
			String text = select.text();//或取到json格式的字符串
			JsonRootBean jsonRootBean = (JsonRootBean) JSONObject.parseObject(
					text, JsonRootBean.class);//采用实体类解析json
			for (int k = 0; k < 20; k++) {
				String name = jsonRootBean.getData().getSearchresult().get(k)
						.getRealPoiId();
				list.add(name);
			}
		}
		set = set + 20;
	}

一共三个实体类
JsonRootBean

public class JsonRootBean {
    private String stid;
    private Data data;
    public void setStid(String stid) {
         this.stid = stid;
     }
     public String getStid() {
         return stid;
     }

    public void setData(Data data) {
         this.data = data;
     }
     public Data getData() {
         return data;
     }
}

Data

public class Data {
	private List searchresult;

	public void setSearchresult(List searchresult) {
		this.searchresult = searchresult;
	}

	public List getSearchresult() {
		return searchresult;
	}
}

Searchresult

public class Searchresult {
	private String name; //酒店名称
	private String ralPoiId;//酒店id

	public void setName(String name) {
		this.name = name;
	}

	public String getName() {
		return name;
	}

	public void setRealPoiId(String ralPoiId) {
		this.ralPoiId = ralPoiId;
	}

	public String getRealPoiId() {
		return ralPoiId;
	}

}

---

为啥要取酒店ID呢？

---

酒店详情页的地址是由"https://hotel.meituan.com/" + 酒店ID+ "/"拼接而成，这样我们通过前面获取的酒店ID就可遍历访问每个酒店的详情页面了，从而获取酒店更加详细的信息

public static void test001() throws Exception {
		for (int i = 0; i < list.size(); i++) {
			String url = "https://hotel.meituan.com/" + list.get(i) + "/";
			// String url = "https://hotel.meituan.com/160194576/";
			Document document = Jsoup.connect(url).ignoreContentType(true)
					.get();
			Elements name = document.select(".relative > span");
			Elements dizhi = document.select(".fs12 > span");
			Elements select = document.select(".mb10");
			System.out.println("名称：" + name.text());
			System.out.println("地址：" + dizhi.text());

			for (Element element : select) {
					System.out.println(element.text() + "\n");
					break;
			}
		}
	}