通过HtmlAgilityPack实现网页信息抓取

1. 下载Html Agility Pack，解压保存到本地 下载地址： http://htmlagilitypack.codeplex.com/
 

 1 void caijisoufun()
 2     {
 3         try
 4         {
 5             
 6             HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
 7             String str = "http://esf.wuxi.soufun.com/agent/agent/AloneHouseList.aspx?agentid=160148311&housetype=esf&price=&roomtype=&district=&page=1";
 8 　　　　　　　　
 9             String htmlstr = fhttp2(str);//获取html页面的源文件
10             doc.LoadHtml(htmlstr);
11             HtmlNode navNode = doc.GetElementbyId("right");//获取id为right的节点
12 //print(navNode.InnerHtml);
13             HtmlNodeCollection categoryNodeList = navNode.SelectNodes("//div[1]/table/tr[1]/td[1]/a[1]"); //分析html结构
14 
15             HtmlNode temp = null;
16 
17             foreach (HtmlNode categoryNode in categoryNodeList)
18             {
19                 temp = HtmlNode.CreateNode(categoryNode.OuterHtml);
20                
21                 String url = "http://esf.wuxi.soufun.com" + temp.Attributes["href"].Value;
22                 println(url);//其实就是个Response.Write
23                 String showstr = fhttp2(url);
24                 HtmlAgilityPack.HtmlDocument doc2 = new HtmlAgilityPack.HtmlDocument();
25                 doc2.LoadHtml(showstr);
26                 HtmlNode cnode = doc2.GetElementbyId("wrap");
27                 HtmlNode title = cnode.SelectSingleNode("//div[2]/div[1]/h1[1]/font[1]");
28                 println(title.InnerText);//
29                 //这里就可以做很多事情了，包括楼盘户型全部可以通过抓取获得信息，导入自己的数据库。
30                 flush();
31                 sleep(10);
32               
33                 //println(temp.Attributes["href"].Value);
34             }
35             
36         }
37         catch (Exception ex)
38         {
39             println(ex);
40         }
41     }
42 
43     String fhttp2(String url)
44     {
45         try
46         {
47             WebRequest rGet = WebRequest.Create(url);
48             WebResponse rSet = rGet.GetResponse();
49             Stream s = rSet.GetResponseStream();
50             StreamReader sr = new StreamReader(s, Encoding.GetEncoding("GB2312"));
51             StringBuilder sb = new StringBuilder();
52             String Str;
53 
54             while ((Str = sr.ReadLine()) != null)
55             {
56                 sb.Append(Str + "\n");
57             }
58 
59             sr.Close();
60             s.Close();
61             rSet.Close();
62 
63             return tostr(sb);
64         }
65         catch (Exception e)
66         {
67             return "";
68         }
69     }