获取大众点评数据
使用.NET实现一个从大众点评抓取一些基础数据（商家名称，地址，电话，经纬度）的小程序。
实现逻辑：
1、以异步方式从列表上获取前三项(商家名称、地址、电话)；
2、把获取下来的数据保存到数据库表里；
3、把存在数据表里的地址信息读取出来，通过调用QQ地图API把地址转化成经纬度；
4、按行更新GIS信息。
代码如下：
using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Data; using System.Data.SqlClient; using System.Diagnostics; using System.IO; using System.Linq; using System.Net; using System.Text; using System.Text.RegularExpressions; using System.Web; using System.Web.UI; using System.Web.UI.WebControls; using System.Xml; namespace DianPing_MeiFa { public partial class test : System.Web.UI.Page { protected void Page_Load(object sender, EventArgs e) { // string url = "http://www.dianping.com/search/category/2/50/p"; //this.lblNames.Text = "begin..."; //for (int i = 1; i < 51; i++)//循环分页 //{ // string url = "http://www.dianping.com/search/category/2/50/p"; // url += i; // this.SaveMeiFaData(url); //} this.setMapLocation(); } /// <summary> /// 获取数据 /// </summary> /// <returns></returns> private DataTable GetMeifa() { string sql = "SELECT * FROM t_meifa"; DataSet ds = SqlHelper.ExecuteDataset(SqlHelper.GetConnection(), CommandType.Text, sql); return ds.Tables[0]; } private void setMapLocation() { DataTable dt = this.GetMeifa(); IList<MeiFa> mfList = new List<MeiFa>(); //将DataTable转化成对象 foreach (DataRow dr in dt.Rows) { MeiFa mf = new MeiFa { Id = int.Parse(dr["id"].ToString()), Name = dr["name"].ToString(), Address = dr["address"].ToString() }; mfList.Add(mf); } //设置获经纬度 foreach (MeiFa mf in mfList) { QQMapGeocoder qmg = this.GetGeocoder(mf.Address); mf.lat = qmg.result.location.lat; mf.lng = qmg.result.location.lng; this.UpdateMeiFaLocation(mf); } } private void UpdateMeiFaLocation(MeiFa mf) { string sql = "UPDATE t_meifa_bak SET lng=@lng,lat=@lat WHERE id=@id"; SqlParameter[] sps ={ new SqlParameter("@lng",mf.lng), new SqlParameter("@lat",mf.lat), new SqlParameter("@id",mf.Id), }; SqlHelper.ExecuteNonQuery(SqlHelper.GetConnection(), CommandType.Text, sql, sps); } /// <summary> /// 根据地理位置获取经纬度 /// </summary> /// <param name="address">地址</param> /// <returns></returns> private QQMapGeocoder GetGeocoder(string address) { string apiMapUrl = "http://apis.map.qq.com/ws/geocoder/v1/?region=北京&address={0}&key=Y5QBZ-DEDR4-3W3U7-XL37W-VVMT6-3KB6K"; apiMapUrl = string.Format(apiMapUrl, address); HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(apiMapUrl); request.Method = "GET"; HttpWebResponse response = (HttpWebResponse)request.GetResponse(); Stream newstream = response.GetResponseStream(); StreamReader srRead = new StreamReader(newstream, Encoding.UTF8); string json = srRead.ReadToEnd(); QQMapGeocoder qmg = QQMapGeocoder.DeserializeGeocoder(json); return qmg; } /// <summary> /// 将列表保存到数据库 /// </summary> /// <param name="url"></param> private void SaveMeiFaData(string url) { HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url); request.Method = "GET"; request.KeepAlive = true; request.ContentType = "application/x-www-form-urlencoded"; request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; request.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.2; zh-CN; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8"; HttpWebResponse response = (HttpWebResponse)request.GetResponse(); Stream newstream = response.GetResponseStream(); StreamReader srRead = new StreamReader(newstream, Encoding.UTF8); string outString = srRead.ReadToEnd(); IList<MeiFa> mfList = this.getMeiFaList(outString); this.InsertDb(mfList); } private string GetContent(IList<string> list) { string str = string.Empty; foreach (string s in list) { str += s; } return str; } /// <summary> /// 将列表数据转化成对象 /// </summary> /// <param name="html"></param> /// <returns></returns> public IList<MeiFa> getMeiFaList(string html) { IList<MeiFa> mfList = new List<MeiFa>(); string reg = @"<ul[^>]*class=""detail""[^>]*>[\s\S]*?</ul>"; MatchCollection mc = Regex.Matches(html, reg); foreach (Match m in mc) { string strDom = m.Value; strDom = strDom.Replace(" ", " "); MeiFa mf = MeiFa.CreateMeifa(strDom); mfList.Add(mf); } return mfList; } public string ReplaceHtml(string HTMLStr) { return Regex.Replace(HTMLStr, "<[^>]*>", ""); } public void InsertDb(IList<MeiFa> mfs) { foreach (var mf in mfs) { if (string.IsNullOrEmpty(mf.Name)) continue; try { this.InsertDb(mf); } catch { continue; } } } /// <summary> /// 插入到数据表里 /// </summary> /// <param name="mf"></param> public void InsertDb(MeiFa mf) { string sql = "INSERT INTO t_meifa(name,address,tel) values(@name,@address,@tel)"; SqlParameter[] sps ={ new SqlParameter("@name",mf.Name), new SqlParameter("@address",mf.Address), new SqlParameter("@tel",mf.Tel), }; SqlHelper.ExecuteNonQuery(SqlHelper.GetConnection(), CommandType.Text, sql, sps); } public class MeiFa { public int Id { get; set; } public string Name { get; set; } public string Address { get; set; } public string Tel { get; set; } public float? lng { get; set; } public float? lat { get; set; } public MeiFa() { } public static MeiFa CreateMeifa(string domStr) { MeiFa m = new MeiFa(); try { Debug.WriteLine(domStr); XmlDocument dom = new XmlDocument(); dom.LoadXml(domStr); XmlNode nameNode = dom.SelectSingleNode("//ul/li[@class=\"shopname\"]/a[@data-hippo-type=\"shop\"]"); m.Name = nameNode.InnerText; XmlNode addressNode = dom.SelectSingleNode("//ul/*/li[@class=\"address\"]"); string at = ReplaceAddress(addressNode.InnerText); string[] ats = getArr(at); m.Address = ats[0]; m.Tel = ats[1]; } catch { m = new MeiFa(); } return m; } /// <summary> /// 生成数组 /// </summary> /// <param name="str"></param> /// <returns></returns> static private string[] getArr(string str) { string[] tempArr = str.Split(' '); List<string> ss = new List<string>(); for (int i = 0; i < tempArr.Length; i++) { if (!string.IsNullOrEmpty(tempArr[i])) { ss.Add(tempArr[i]); } } return ss.ToArray(); } static private string ReplaceAddress(string s) { s = s.Replace("地址:", ""); s = s.Replace("\n", ""); return s; } } } }
using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Linq; using System.Web; namespace DianPing_MeiFa { //[JsonProperty] public class QQMapGeocoder { public int status { get; set; } public string message { get; set; } public QQMapResult result { get; set; } public static QQMapGeocoder DeserializeGeocoder(string jsonStr) { JsonSerializerSettings jsz = new JsonSerializerSettings(); QQMapGeocoder qg = JsonConvert.DeserializeObject<QQMapGeocoder>(jsonStr, jsz); return qg; } } public class QQMapResult { public QQMapLocation location { get; set; } public QQMapAddressComponents address_components { get; set; } public string similarity { get; set; } } public class QQMapLocation { public float? lng { get; set; } public float? lat { get; set; } } public class QQMapAddressComponents { public string province { get; set; } public string city { get; set; } public string district { get; set; } public string street { get; set; } public string street_number { get; set; } } }
注：仅供学习使用!