多线程爬虫 用jsoup爬杭州房价
初学者学爬虫可以玩玩我做的这个小demo，原理就是用了jsoup这个小玩意，数据源是房天下的数据，杭州的房价，总共爬了100页。
三个类，超简单的，一个是爬虫demo，另一个是简单的开发商类，里面存放了这个开放商有多少套房源，均价多少，总价多少，方便后面对所有开放商的均价做了个排行。
import lombok.Data;
import lombok.SneakyThrows;
import lombok.Synchronized;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/** s
* Created by Precious_Life on 2018/9/2.
*/
@Data
public class CrawlByJsoupTest {
public static int count = 0;
public static final Map treeMap=new TreeMap();
public static final Queue urlList=new LinkedList<>();
public static final CountDownLatch countDownLatch=new CountDownLatch(100);
@Synchronized
public void addDeveloperInfo(String name,int totalPrice,int avengePrice){
if(treeMap.containsKey(name)){
DeveloperInfo tmp=treeMap.get(name);
int tmpTotalPrice=tmp.getTotalPrice()+totalPrice;
int tmpCount=tmp.getHouseNum()+1;
int tmpAvengePrice=(tmp.getAveragePrice()+avengePrice)/2;
treeMap.put(name,new DeveloperInfo(name,tmpCount,tmpTotalPrice,tmpAvengePrice));
}else{
treeMap.put(name,new DeveloperInfo(name,1,totalPrice,avengePrice));
}
}
public static Map getDeveloperInfoMap(){
return treeMap;
}
@Synchronized
public static void increaseCount(){
count++;
}
@SneakyThrows
public static void main(String[] args){
ExecutorService executorService= Executors.newCachedThreadPool();
CrawlByJsoupTest crawlByJsoupTest=new CrawlByJsoupTest();
for(int j=1;j<=100;j++){
urlList.offer("http://esf.hz.fang.com/house-a0154/i3" + j);
}
for(int i=0;i<10;i++){
executorService.execute(new ElementsAnalysis(countDownLatch,crawlByJsoupTest));
}
countDownLatch.await();
System.out.println("总共" + count + "套房源!");
sortByAveragePrice(treeMap);
}
public static void sortByAveragePrice(Map treeMap){
List> list = new ArrayList>(treeMap.entrySet());
list.sort((Map.Entry o1, Map.Entry o2)->o1.getValue().getAveragePrice()-o2.getValue().getAveragePrice());
System.out.println("下面是滨江区的开放商房价排行");
int houseCount=0;
for (Map.Entry e: list) {
houseCount+=e.getValue().getHouseNum();
System.out.println(e.getKey()+" 均价:"+e.getValue().getAveragePrice()+"元/平"+ " 房源共"+e.getValue().getHouseNum()+"套");
}
System.out.println("滨江现有房源"+houseCount+"套");
}
@Synchronized
public static String pollUrl(){
String url="";
if(urlList.size()!=0){
url=urlList.poll();
}
return url;
}
public static int getCount(){
return count;
}
}
下面就是开放商的类了，我都是用拼音进行命名的，应该很容易看懂吧。
@Data
@AllArgsConstructor
@NoArgsConstructor
public class DeveloperInfo {
private String developerName;
private int houseNum;
private int totalPrice;
private int averagePrice;
}
import com.sun.corba.se.impl.orbutil.concurrent.Sync;
import lombok.*;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.util.*;
import java.util.concurrent.CountDownLatch;
/**
* Created by Precious_Life on 2018/9/2.
*/
@Data
@NoArgsConstructor
public class ElementsAnalysis implements Runnable{
public CountDownLatch countDownLatch;
CrawlByJsoupTest crawlByJsoupTest;
public ElementsAnalysis(CountDownLatch countDownLatch,CrawlByJsoupTest crawlByJsoupTest){
this.countDownLatch=countDownLatch;
this.crawlByJsoupTest=crawlByJsoupTest;
}
@Override
public void run() {
while(analysisPage(CrawlByJsoupTest.pollUrl())){
}
}
@SneakyThrows
public boolean analysisPage(String url){
if(url.equals("")){
return false;
}
System.out.println("now will analysis Num"+url+"页");
Document document = Jsoup.connect(url)
.userAgent("Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; MALC)")
.timeout(999999999)
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
.header("Accept-Encoding", "gzip, deflate")
.header("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3")
.header("Connection", "keep-alive")
.header("Host", "esf.hz.fang.com")
//是忽略请求类型Id
.ignoreContentType(true)
.get();
Element div_element = document.getElementsByClass("shop_list").get(0);
Elements elements = div_element.select("dl");
for (Element element : elements) {
CrawlByJsoupTest.increaseCount();
Elements elements1 = element.children();
Elements tmp = elements1.get(1).select("a");
if (tmp.size() == 0) continue;
String developer = elements1.get(1).select("a").get(2).attr("title");
String address = elements1.get(1).getElementsByClass("add_shop").select("span").text();
String type = elements1.get(1).getElementsByClass("tel_shop").text();
Element ss = elements1.get(2);
String totalPrice = elements1.get(2).getElementsByClass("red").text();
String averagePrice = elements1.get(2).select("span").get(1).text();
int finalAveragePrice=(int)Double.parseDouble(averagePrice.substring(0,averagePrice.indexOf("元")));
int finalTotalPrice=(int)Double.parseDouble(totalPrice.substring(0,totalPrice.indexOf("万")))*10000;
crawlByJsoupTest.addDeveloperInfo(developer,finalTotalPrice,finalAveragePrice);
System.out.println("Num"+CrawlByJsoupTest.count+" 开发商" + developer + " 地址:" + address + " 户型:" + type + " 单价:" + finalTotalPrice + "元/每平 总价:" + finalAveragePrice+"元");
}
countDownLatch.countDown();
return true;
}
}
对于上面的结果，就是及时输出了下房源的信息，然后又对房源数据进行了下分析，做了下均价排行。
截下图~~
这个图是各开放商的房源均价排行
把这个数据爬下来之后，有啥用么？？没啥用。。。。。哈哈哈，查了下房价最低的这个聆涛苑到滨江这边的距离
要两个小时呢！！！！！！！！！！！
后面看了一个那种开源的爬虫框架，实现逻辑和手写框架差不多，只不过封装性做的更好，扩展性更好，功能更强大，这个手写框架看好了的话，其他的框架代码也可以熟练上手的！