抓取网页数据

using System; using System.Data; using System.Configuration; using System.Web; using System.Web.Security; using System.Web.UI; using System.Web.UI.WebControls; using System.Web.UI.WebControls.WebParts; using System.Web.UI.HtmlControls; using System.Net; using System.Text; using System.Text.RegularExpressions; using System.IO; using System.Collections; public partial class _Default : System.Web.UI.Page { protected void Page_Load(object sender, EventArgs e) { ShowInfo(); } #region 显示信息 /// <summary> /// 构造显示信息 /// </summary> private void ShowInfo() { string sEciticUrl = "http://www.ecitic.com/paijia.jsp"; string sICBCUrl = "http://www.icbc.com.cn/ICBCDynamicSite/Optimize/Quotation/QuotationListIframe.aspx"; string sBOCUrl = "http://www.boc.cn/sourcedb/whpj/"; string sCCBUrl = "https://ibsbjstar.ccb.com.cn/app/B2CMainPlat?TXCODE=R00002"; try { string sGetEciticHtml = GetPageData(sEciticUrl, null); string sGetICBCHtml = GetPageData(sICBCUrl, null); string sGetBOCHtml = GetPageData(sBOCUrl, null); string sGetCCBHtml = GetPageData(sCCBUrl); ArrayList arylstEcitic = GetEciticData(sGetEciticHtml); //中信银行 ArrayList arylstICBC = GetICBCData(sGetICBCHtml); //工商银行 ArrayList arylstBOC = GetBOCData(sGetBOCHtml); //中国银行 ArrayList arylstCCB = GetCCBData(sGetCCBHtml); //建设银行 ArrayList arylstSum = new ArrayList(); ArrayList arylstAverage = new ArrayList(); //中间价 ArrayList arylstBuyAverage = new ArrayList(); //汇买平均价 ArrayList arylstSellAverage = new ArrayList();//汇卖平均价 for (int m = 0; m < 8; m++) { double dSum = Convert.ToDouble(arylstEcitic[m]) + Convert.ToDouble(arylstICBC[m]) + Convert.ToDouble(arylstBOC[m]) + Convert.ToDouble(arylstCCB[m]); arylstSum.Add(dSum); //将总值添加到数组 } for (int n = 0; n < 8; n++) { double dBuyAverage = Convert.ToDouble(arylstSum[n]); double dSellAverage = Convert.ToDouble(arylstSum[n + 1]); double dAverage = (dBuyAverage / 4 + dSellAverage / 4) / 2; n++; arylstBuyAverage.Add(dBuyAverage / 4); arylstSellAverage.Add(dSellAverage / 4); arylstAverage.Add(dAverage); } string[] sMoneyTypeList = new string[] { "英镑", "美元", "日元", "欧元" }; string[] sBankList = new string[] { "中信银行", "中国工商银行", "中国银行", "中国建设银行", "财务公司" }; StringBuilder sb = new StringBuilder(); sb.Append("<table cellpadding=\"0px\" cellspacing=\"0px\"><tr><td rowspan=\"2\">银行/币种</td>"); for (int j = 0; j < sMoneyTypeList.Length; j++) { sb.Append("<td colspan=\"3\" align=\"center\">" + sMoneyTypeList[j] + "</td>"); } sb.Append("</tr><tr>"); for (int x = 0; x < sMoneyTypeList.Length; x++) { sb.Append("<td>汇买价</td>"); sb.Append("<td>汇卖价</td>"); sb.Append("<td>中间价</td>"); } sb.Append("</tr>"); sb.Append(CreateTD(sBankList[0], arylstEcitic)); sb.Append(CreateTD(sBankList[1], arylstICBC)); sb.Append(CreateTD(sBankList[2], arylstBOC)); sb.Append(CreateTD(sBankList[3], arylstCCB)); sb.Append("<tr><td>" + sBankList[4] + "</td>"); for (int k = 0; k < 4; k++) { sb.Append("<td>" + arylstBuyAverage[k].ToString() + "</td>"); sb.Append("<td>" + arylstSellAverage[k].ToString() + "</td>"); sb.Append("<td>" + arylstAverage[k].ToString() + "</td>"); } sb.Append("</tr>"); sb.Append("<tr><td colspan=\"13\" style="\" mce_style="\""text-align: right;\">人民币/100外币</td></tr></table>"); Response.Write(sb.ToString()); } catch (Exception ex) { Response.Write("错误提示:" + ex.Message); } } /// <summary> /// 生成TD /// </summary> /// <param name="sMoneyType"></param> /// <param name="arylst"></param> /// <returns></returns> private string CreateTD(string sMoneyType, ArrayList arylst) { StringBuilder sb = new StringBuilder(); sb.Append("<tr><td>" + sMoneyType + "</td>"); for (int i = 0; i < 8; i++) { sb.Append("<td>" + arylst[i].ToString() + "</td>"); sb.Append("<td>" + arylst[i + 1].ToString() + "</td>"); sb.Append("<td>" + (Convert.ToDouble(arylst[i]) + Convert.ToDouble(arylst[i + 1])) / 2 + "</td>"); i++; } sb.Append("</tr>"); return sb.ToString(); } #endregion #region 辅助方法 private string GetHrefCount(string strResponse) { string strStatus = string.Empty; // 解析页面，查找链接 // 此处尚需扩展，还有某些形式的链接不被识别 string strRef = @"(href|HREF|src|SRC|action|ACTION|Action)[ ]*=[ ]*[""'][^""'#>]+[""']"; MatchCollection matches = new Regex(strRef).Matches(strResponse); return strStatus += "找到: " + matches.Count + " 个链接\r\n"; } private string GetHtmlTitle(string strResponse) { string title = string.Empty; //获取标题 Match TitleMatch = Regex.Match(strResponse, "<title>([^<]*)</title>", RegexOptions.IgnoreCase | RegexOptions.Multiline); return title = TitleMatch.Groups[1].Value; } private string GetMetaInfo(string strResponse) { string strdesc = string.Empty; //获取描述信息 Match Desc = Regex.Match(strResponse, "<Meta name=\"DESCRIPTION\" content=\"([^<]*)\">", RegexOptions.IgnoreCase | RegexOptions.Multiline); return strdesc = Desc.Groups[1].Value; } private string GetEncodInfo(string url) { if (url == null || url.Trim() == "") return null; WebClient wc = new WebClient();//定义 wc.Credentials = CredentialCache.DefaultCredentials; Byte[] pageData = wc.DownloadData(url); return Encoding.GetEncoding(0).GetString(pageData); } //去除Html标签 private string StripHtml(string strHtml) { Regex objRegExp = new Regex("<(.|\n)+?>"); string strOutput = objRegExp.Replace(strHtml, ""); strOutput = strOutput.Replace("<", "<"); strOutput = strOutput.Replace(">", ">"); return strOutput; } #endregion #region 中信银行外汇牌价 /// <summary> /// 获取中信银行外汇牌价四种币种内容 /// </summary> /// <param name="strResponse"></param> /// <param name="sMoneyType"></param> /// <returns></returns> private string GetEciticHtml(string strResponse, string sMoneyType) { string sTableHtml = string.Empty; string sResult = string.Empty; string s = string.Empty; MatchCollection mTableHtmlCollection = Regex.Matches(strResponse, "<tr class=\"01\">(.|\n)+?</tr>", RegexOptions.IgnoreCase | RegexOptions.Multiline); foreach (Match mTableHtml in mTableHtmlCollection) { sTableHtml += mTableHtml.Value + "<br/>"; } MatchCollection mCollection = Regex.Matches(sTableHtml, "<td width=\"80\" height=\"20\" class=\"command\">[" + sMoneyType + "](.|\n)+?<br/>", RegexOptions.IgnoreCase | RegexOptions.Multiline); foreach (Match mHtml in mCollection) { sResult += mHtml.Value; } MatchCollection mCollections = Regex.Matches(sResult, "<td align=\"center\" class=\"command\">(.|\n)+?</td>", RegexOptions.IgnoreCase | RegexOptions.Multiline); foreach (Match mHtmls in mCollections) { s += mHtmls.Groups[0].Value + "<br/>"; } return s; } /// <summary> /// 获取中信银行外汇牌价，数组中0、1位置为英镑汇买入、卖出，2、3为美元汇买入、卖出，4、5为日元汇买入、卖出，6、7为欧元汇买入、卖出,对应【汇买、汇卖】 /// </summary> /// <param name="strResponse"></param> /// <returns></returns> private ArrayList GetEciticData(string strResponse) { string sGetHtml = string.Empty; ArrayList arylst = new ArrayList(); string[] sMoneyTypeList = new string[] { "英镑", "美元", "日元", "欧元" }; for (int m = 0; m < sMoneyTypeList.Length; m++) { sGetHtml = GetEciticHtml(strResponse, sMoneyTypeList[m]).Replace("<td align=\"center\" class=\"command\">", "").Replace("</td><br/>", "|").TrimEnd('|'); string[] sData = sGetHtml.Split('|'); for (int n = 0; n < sData.Length; n++) { if (n != 1 && n != 4) { continue; } else { arylst.Add(sData[n]); } } } return arylst; } #endregion #region 工商银行外汇牌价 /// <summary> /// 获取工商银行外汇牌价四种币种内容 /// </summary> /// <param name="strResponse"></param> /// <param name="sMoneyType"></param> /// <returns></returns> private string GetICBCHtml(string strResponse, string sMoneyType) { string sTableHtml = string.Empty; string sResult = string.Empty; MatchCollection mTableHtmlCollection = Regex.Matches(strResponse.Replace("\t","").Replace("16%","14%").Replace(";",""), "\\b" + sMoneyType + "(.|\n)+?</tr>", RegexOptions.IgnoreCase | RegexOptions.Multiline); foreach (Match mTableHtml in mTableHtmlCollection) { sTableHtml += mTableHtml.Value; } MatchCollection mCollection = Regex.Matches(sTableHtml, "\\b14%\">(.|\n)+?</td>", RegexOptions.IgnoreCase | RegexOptions.Multiline); foreach (Match mHtml in mCollection) { sResult += mHtml.Value; } return sResult; } /// <summary> /// 获取工商银行外汇牌价，数组中0、1为英镑汇买入、卖出，2、3为美元汇买入、卖出，4、5为日元汇买入、卖出，6、7为欧元汇买入、卖出,对应【汇买、汇卖】 /// </summary> /// <param name="strResponse"></param> /// <returns></returns> private ArrayList GetICBCData(string strResponse) { string sGetHtml = string.Empty; ArrayList arylst = new ArrayList(); string[] sMoneyTypeList = new string[] { "GBP", "USD", "JPY", "EUR" }; for (int m = 0; m < sMoneyTypeList.Length; m++) { sGetHtml = GetICBCHtml(strResponse, sMoneyTypeList[m]).Replace("14%\">", "").Replace("</td>", "|").TrimEnd('|'); string[] sData = sGetHtml.Split('|'); for (int n = 0; n < sData.Length; n++) { if (n != 1 && n != 3) { continue; } else { arylst.Add(sData[n]); } } } return arylst; } #endregion #region 中国银行外汇牌价 /// <summary> /// 获取中国银行外汇牌价四种币种内容 /// </summary> /// <param name="strResponse"></param> /// <param name="sMoneyType"></param> /// <returns></returns> private string GetBOCHtml(string strResponse, string sMoneyType) { string sTableHtml = string.Empty; string sResult = string.Empty; string s = string.Empty; MatchCollection mTableHtmlCollection = Regex.Matches(strResponse, "<tr align=\"center\">(.|\n)+?</tr>", RegexOptions.IgnoreCase | RegexOptions.Multiline); foreach (Match mTableHtml in mTableHtmlCollection) { sTableHtml += mTableHtml.Value.Replace(" ","").Replace("\n","").Replace("\t","").Trim(); } MatchCollection mCollection = Regex.Matches(sTableHtml, "<tdbgcolor=\"#FFFFFF\">[" + sMoneyType + "](.|\n)+?</td></tr>", RegexOptions.IgnoreCase | RegexOptions.Multiline); foreach (Match mHtml in mCollection) { sResult += mHtml.Value; } return sResult; } /// <summary> /// 获取中国银行外汇牌价，数组中0、1位置为英镑买入、卖出，2、3为美元买入、卖出，4、5为日元买入、卖出，6、7为欧元买入、卖出,对应【汇买、汇卖】 /// </summary> /// <param name="strResponse"></param> /// <returns></returns> private ArrayList GetBOCData(string strResponse) { string sGetHtml = string.Empty; ArrayList arylst = new ArrayList(); string[] sMoneyTypeList = new string[] { "英镑", "美元", "日元", "欧元" }; for (int m = 0; m < sMoneyTypeList.Length; m++) { sGetHtml = GetBOCHtml(strResponse, sMoneyTypeList[m]).Replace("<tdbgcolor=\"#FFFFFF\">", "").Replace("</td>", "|").Replace("</tr>","").TrimEnd('|'); string[] sData = sGetHtml.Split('|'); for (int n = 0; n < sData.Length; n++) { if (n != 1 && n != 3) { continue; } else { arylst.Add(sData[n]); } } } return arylst; } #endregion #region 中国建设银行外汇牌价 /// <summary> /// 获取中国建设银行外汇牌价四种币种内容 /// </summary> /// <param name="strResponse"></param> /// <param name="sMoneyType"></param> /// <returns></returns> private string GetCCBHtml(string strResponse, string sMoneyType) { string sTableHtml = string.Empty; string sResult = string.Empty; string s = string.Empty; MatchCollection mTableHtmlCollection = Regex.Matches(strResponse, "\\b"+sMoneyType+"(.|\n)+?</tr>", RegexOptions.IgnoreCase | RegexOptions.Multiline); foreach (Match mTableHtml in mTableHtmlCollection) { sTableHtml += mTableHtml.Value.Replace("\t", "").Trim(); } MatchCollection mCollection = Regex.Matches(sTableHtml, "\\bVerdana\">(.|\n)+?</span></div>", RegexOptions.IgnoreCase | RegexOptions.Multiline); foreach (Match mHtml in mCollection) { sResult += mHtml.Value; } return sResult; } /// <summary> /// 获取中国建设银行外汇牌价，数组中0、1位置为英镑买入、卖出，2、3为美元买入、卖出，4、5为日元买入、卖出，6、7为欧元买入、卖出。对应【汇买、汇卖】 /// </summary> /// <param name="strResponse"></param> /// <returns></returns> private ArrayList GetCCBData(string strResponse) { string sGetHtml = string.Empty; ArrayList arylst = new ArrayList(); string[] sMoneyTypeList = new string[] { "tranCode\\(\"826\"\\)", "tranCode\\(\"840\"\\)", "tranCode\\(\"392\"\\)", "tranCode\\(\"978\"\\)" }; for (int m = 0; m < sMoneyTypeList.Length; m++) { sGetHtml = GetCCBHtml(strResponse, sMoneyTypeList[m]).Replace("Verdana\">", "").Replace("</span></div>", "|").TrimEnd('|'); string[] sData = sGetHtml.Split('|'); for (int n = 0; n < sData.Length; n++) { if (n != 0 && n != 1) { continue; } else { arylst.Add(Convert.ToDouble(sData[n])*100); } } } return arylst; } #endregion #region 下载网页内容到本地 /// <summary> /// url是要访问的网站地址，charSet是目标网页的编码，如果传入的是null或者""，那就自动分析网页的编码 /// </summary> /// <param name="url"></param> /// <param name="charSet"></param> /// <returns></returns> private string GetPageData(string url, string charSet) { string strWebData = ""; if (url != null || url.Trim() != "") { WebClient myWebClient = new WebClient(); //创建WebClient实例myWebClient // 需要注意的： //有的网页可能下不下来，有种种原因比如需要cookie,编码问题等等 //这是就要具体问题具体分析比如在头部加入cookie // webclient.Headers.Add("Cookie", cookie); //这样可能需要一些重载方法。根据需要写就可以了 //获取或设置用于对向 Internet 资源的请求进行身份验证的网络凭据。 myWebClient.Credentials = CredentialCache.DefaultCredentials; //如果服务器要验证用户名,密码 //NetworkCredential mycred = new NetworkCredential(struser, strpassword); //myWebClient.Credentials = mycred; //从资源下载数据并返回字节数组。（加@是因为网址中间有"/"符号） byte[] myDataBuffer = myWebClient.DownloadData(url); strWebData = Encoding.Default.GetString(myDataBuffer); //获取网页字符编码描述信息 Match charSetMatch = Regex.Match(strWebData, "<meta([^<]*)charset=([^<]*)\"", RegexOptions.IgnoreCase | RegexOptions.Multiline); string webCharSet = charSetMatch.Groups[2].Value; if (charSet == null || charSet == "") { //如果未获取到编码，则设置默认编码 if (webCharSet == null || webCharSet == "") { charSet = "UTF-8"; } else { charSet = webCharSet; } } if (charSet != null && charSet != "" && Encoding.GetEncoding(charSet) != Encoding.Default) { strWebData = Encoding.GetEncoding(charSet).GetString(myDataBuffer); } } return strWebData; } /// <summary> /// 下载网页，url是要访问的网站地址 /// </summary> /// <param name="url"></param> /// <returns></returns> private string GetPageData(string url) { string responseData = ""; if (url != null || url != "") { try { HttpWebRequest Req = (HttpWebRequest)WebRequest.Create(new System.Uri(url)); Req.UserAgent = "Mozilla/4.0(compatible;MSIE 6.0;Windows NT 5.0; .NET CLR 1.1.4322)"; Req.Timeout = 30000; StreamReader responseReader = new StreamReader(Req.GetResponse().GetResponseStream(), Encoding.Default); responseData = responseReader.ReadToEnd(); responseReader.Close(); } catch(Exception ex) { Response.Write(ex.Message); } } return responseData; } #endregion }