网页抓取数据并分析，特别包括分页数据的抓取
[csharp]
包括8个按钮，每个按钮下的代码都可运行（第5、6个可能需要调试一下）。
[csharp]
有基本的页面抓取，不含分页数据的；
[csharp
有含分页数据，且【下一页】的链接是网址的；
[csharp]
有含分页数据，且【下一页】的链接是__doPostBack；
[csharp]
有含分页数据，且【下一页】的属性是.gif，可通过F12找到href的。
[csharp]
[csharp]
[csharp]
[csharp]
[csharp]
[csharp]
[csharp]
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.IO;
using System.Net;
namespace WindowsFormsApplication1
{
public partial class Form1 : Form
{
public Form1()
{
InitializeComponent();
}
//private void button1_Click(object sender, EventArgs e)
//{
// MessageBox.Show("hello world.");
//}
//WebBrowser web1 = new WebBrowser();
//web1.Navigate("http://www.xjflcp.com/ssc/");
//web.DocumentCompleted += new WebBrowserDocumentCompletedEventHandler(web_DocumentCompleted);
//void web_DocumentCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
//{
// WebBrowser web = (WebBrowser)sender;
// HtmlElementCollection ElementCollection = web.Document.GetElementsByTagName("Table");
// foreach (HtmlElement item in ElementCollection)
// {
// File.AppendAllText("Kaijiang_xj.txt", item.InnerText);
// }
//}
//根据Url地址得到网页的html源码
private string GetWebContent(string Url)
{
string strResult="";
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url); //声明一个HttpWebRequest请求
request.Timeout = 30000; //设置连接超时时间
request.Headers.Set("Pragma", "no-cache");
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Stream streamReceive = response.GetResponseStream();
Encoding encoding = Encoding.GetEncoding("GB2312");
StreamReader streamReader = new StreamReader(streamReceive, encoding);
strResult = streamReader.ReadToEnd();
}
catch
{
MessageBox.Show("出错");
}
return strResult;
}
//为了使用HttpWebRequest和HttpWebResponse，需填名字空间引用 using System.Net;
//以下是程序具体实现过程：
private void button1_Click(object sender, EventArgs e)
{
//要抓取的URL地址
string Url = "http://list.mp3.baidu.com/topso/mp3topsong.html?id=1#top2";
//得到指定Url的源码
string strWebContent = GetWebContent(Url);
richTextBox1.Text = strWebContent;
//取出和数据有关的那段源码
int iBodyStart = strWebContent.IndexOf("
int iStart = strWebContent.IndexOf("歌曲TOP500", iBodyStart);
int iTableStart = strWebContent.IndexOf("
int iTableEnd = strWebContent.IndexOf("", iTableStart);
string strWeb = strWebContent.Substring(iTableStart, iTableEnd - iTableStart + 8);
//生成HtmlDocument
WebBrowser webb = new WebBrowser();
webb.Navigate("about:blank");
HtmlDocument htmldoc = webb.Document.OpenNew(true);
htmldoc.Write(strWeb);
HtmlElementCollection htmlTR = htmldoc.GetElementsByTagName("TR");
foreach (HtmlElement tr in htmlTR)
{
string strID = tr.GetElementsByTagName("TD")[0].InnerText;
string strName = tr.GetElementsByTagName("TD")[1].InnerText;
string strSinger = tr.GetElementsByTagName("TD")[1].InnerText;
//插入DataTable
strID = strID.Replace(".", "");
//AddLine(strID, strName, strSinger, "0");
//string strID = tr.GetElementsByTagName("TD")[0].InnerText;
//string strName = SplitName(tr.GetElementsByTagName("TD")[1].InnerText, "MusicName");
//string strSinger = SplitName(tr.GetElementsByTagName("TD")[1].InnerText, "Singer");
插入DataTable
//strID = strID.Replace(".", "");
//AddLine(strID, strName, strSinger, "0");
//string strID1 = tr.GetElementsByTagName("TD")[2].InnerText;
//string strName1 = SplitName(tr.GetElementsByTagName("TD")[3].InnerText, "MusicName");
//string strSinger1 = SplitName(tr.GetElementsByTagName("TD")[3].InnerText, "Singer");
插入DataTable
//strID1 = strID1.Replace(".", "");
//AddLine(strID1, strName1, strSinger1, "0");
//string strID2 = tr.GetElementsByTagName("TD")[4].InnerText;
//string strName2 = SplitName(tr.GetElementsByTagName("TD")[5].InnerText, "MusicName");
//string strSinger2 = SplitName(tr.GetElementsByTagName("TD")[5].InnerText, "Singer");
插入DataTable
//strID2 = strID2.Replace(".", "");
//AddLine(strID2, strName2, strSinger2, "0");
}
//插入数据库
//InsertData(dt);
//dataGridView1.DataSource = dt.DefaultView;
}
private void button2_Click(object sender, EventArgs e)
{
try {
WebClient MyWebClient = new WebClient();
MyWebClient.Credentials = CredentialCache.DefaultCredentials;//获取或设置用于向Internet资源的请求进行身份验证的网络凭据
Byte[] pageData = MyWebClient.DownloadData("http://bbs.cup.edu.cn/cupbbs/ThreadList.aspx?fid=51"); //从指定网站 下载数据
//string pageHtml = Encoding.Default.GetString(pageData); //如果获取网站页面采用的是GB2312，则使用这句
string pageHtml = Encoding.UTF8.GetString(pageData); //如果获取网站页面采用的是UTF-8，则使用这句
Console.WriteLine(pageHtml);//在控制台输入获取的内容
using (StreamWriter sw = new StreamWriter("C:\\Users\\yuan\\Desktop\\ouput.html"))//将获取的内容写入文本
{
sw.Write(pageHtml);
}
Console.ReadLine(); //让控制台暂停,否则一闪而过了
}
catch(WebException webEx) {
Console.WriteLine(webEx.Message.ToString());
}
}
void web_DocumentCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
WebBrowser web = (WebBrowser)sender;
HtmlElementCollection ElementCollection = web.Document.GetElementsByTagName("Table");
foreach (HtmlElement item in ElementCollection)
{
File.AppendAllText("C:\\Users\\yuan\\Desktop\\ouputbutton3.txt", item.InnerText);
}
}
private void button3_Click(object sender, EventArgs e)
{
WebBrowser web = new WebBrowser();
web.Navigate("http://www.chinahighway.gov.cn/html/staticHtml/front/index_lkcx.html");
web.DocumentCompleted += new WebBrowserDocumentCompletedEventHandler(web_DocumentCompleted);
}
private void button4_Click(object sender, EventArgs e)
{
string _StrResponse = "";
HttpWebRequest _WebRequest = (HttpWebRequest)WebRequest.Create("http://bbs.cup.edu.cn/cupbbs/ThreadList.aspx?fid=51");
_WebRequest.UserAgent = "MOZILLA/4.0 (COMPATIBLE; MSIE 7.0; WINDOWS NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)";
_WebRequest.Method = "Get";
WebResponse _WebResponse = _WebRequest.GetResponse();
StreamReader _ResponseStream = new StreamReader(_WebResponse.GetResponseStream(), System.Text.Encoding.GetEncoding("gb2312"));
_StrResponse = _ResponseStream.ReadToEnd();
_WebResponse.Close();
_ResponseStream.Close();
File.AppendAllText("C:\\Users\\yuan\\Desktop\\ouputbutton4.txt", _StrResponse);
}
private void button5_Click(object sender, EventArgs e)
{
System.Net.WebClient WebClientObj = new System.Net.WebClient();
System.Collections.Specialized.NameValueCollection PostVars = new System.Collections.Specialized.NameValueCollection();
PostVars.Add("__VIEWSTATE", "dDwxNDMyMjQ3NTY4O3Q8O2w8aTwxPjs+O2w8dDw7bDxpPDM+O2k8Nz47PjtsPHQ8O2w8aTwwPjtpPDE+O2k8Mj47aTwzPjtpP........省略了，太长了.......");
//PostVars.Add("__EVENTVALIDATION", "此处是您需要提前得到的信息");
PostVars.Add("__EVENTTARGET", "grdThreadList:_ctl27:_ctl3");
//PostVars.Add("__EVENTARGUMENT", "");
//string s = "23$74$56";
//string[] str=new string[5];
//str=s.Split('$');
WebClientObj.Headers.Add("ContentType", "application/x-www-form-urlencoded");
try
{
byte[] byte1 = WebClientObj.UploadValues("http://bbs.cup.edu.cn/cupbbs/ThreadList.aspx?fid=51", "POST", PostVars);
string ResponseStr = Encoding.UTF8.GetString(byte1); //得到当前页面对应的html 文本字符串
//GetPostValue(ResponseStr);//得到当前页面对应的 __VIEWSTATE 等上面需要的信息，为抓取下一页面使用
//SaveMessage(ResponseStr);//保存自己关心的内容到 数据库中
File.AppendAllText("C:\\Users\\yuan\\Desktop\\ouputbutton5.txt", ResponseStr);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
private static string current__viewstate="";//保存当前页面对应的 __VIEWSTATE 等上面需要的信息，为再次点击按钮（抓取下一页面）使用
private string GetPostValue(string ResponseStr)
{
//......
return "";//略//解析ResponseStr，得到__VIEWSTATE的值
}
private void button6_Click(object sender, EventArgs e)
{
System.Net.WebClient WebClientObj = new System.Net.WebClient();
System.Collections.Specialized.NameValueCollection PostVars = new System.Collections.Specialized.NameValueCollection();
PostVars.Add("__VIEWSTATE", current__viewstate);
PostVars.Add("__EVENTVALIDATION", "此处是您需要提前得到的信息");
PostVars.Add("__EVENTTARGET", "grdThreadList:_ctl27:_ctl3");//通过for,改变其值，ctl1,ctl2,ctl3....
PostVars.Add("__EVENTARGUMENT", "");
string s = "23$74$56";
string[] str=new string[5];
str=s.Split('$');
WebClientObj.Headers.Add("ContentType", "application/x-www-form-urlencoded");
try
{
byte[] byte1 = WebClientObj.UploadValues("http://www.chinahighway.gov.cn/ html/staticHtml/front/index_lkcx.html", "POST", PostVars);
string ResponseStr = Encoding.UTF8.GetString(byte1); //得到当前页面对应的html 文本字符串
current__viewstate=GetPostValue(ResponseStr);//得到当前页面对应的 __VIEWSTATE 等上面需要的信息，为抓取下一页面使用
//SaveMessage(ResponseStr);//保存自己关心的内容到数据库中
File.AppendAllText("C:\\Users\\yuan\\Desktop\\ouputbutton6.txt", ResponseStr);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
private void button7_Click(object sender, EventArgs e)
{
try
{
WebClient MyWebClient = new WebClient();
MyWebClient.Credentials = CredentialCache.DefaultCredentials;//获取或设置用于向Internet资源的请求进行身份验证的网络凭据
Byte[] pageData = MyWebClient.DownloadData("http://gb.cri.cn/42071/2013/03/05/3245s4038640_2.htm"); //从指定网站下载数据
string pageHtml = Encoding.Default.GetString(pageData); //如果获取网站页面采用的是GB2312，则使用这句
//string pageHtml = Encoding.UTF8.GetString(pageData); //如果获取网站页面采用的是UTF-8，则使用这句
Console.WriteLine(pageHtml);//在控制台输入获取的内容
using (StreamWriter sw = new StreamWriter("C:\\Users\\yuan\\Desktop\\ouput7.html"))//将获取的内容写入文本
{
sw.Write(pageHtml);
}
Console.ReadLine(); //让控制台暂停,否则一闪而过了
}
catch (WebException webEx)
{
Console.WriteLine(webEx.Message.ToString());
}
}
//对分页数据抓取，当网页上【下一页】的属性是.gif格式的时候，而且鼠标停在【下一页】上面，页面下面显示的网址不完整（网址太长，只能看到部分），
//可以通过在网页上点击右键【查看源文件】,或F12【脚本】，找到对应的href，即可得到网址。
private void button8_Click(object sender, EventArgs e)
{
try
{
WebClient MyWebClient = new WebClient();
MyWebClient.Credentials = CredentialCache.DefaultCredentials;//获取或设置用于向Internet资源的请求进行身份验证的网络凭据
Byte[] pageData = MyWebClient.DownloadData("http://www.chinahighway.gov.cn/roadInfo/queryRoadInfo.do?queryType=map&startDate=&cantonName=&cantonCode=&infoType=3&endDate=&startPlanDate=&_page_size=50&roadName=&mapList=-1&endRealDate=&roadCode=&provinceList=-1&endPlanDate=&startRealDate=&page=3"); //从指定网站下载数据
string pageHtml = Encoding.Default.GetString(pageData); //如果获取网站页面采用的是GB2312，则使用这句
//string pageHtml = Encoding.UTF8.GetString(pageData); //如果获取网站页面采用的是UTF-8，则使用这句
Console.WriteLine(pageHtml);//在控制台输入获取的内容
using (StreamWriter sw = new StreamWriter("C:\\Users\\yuan\\Desktop\\ouput8.txt"))//将获取的内容写入文本 www.2cto.com
{
sw.Write(pageHtml);
}
Console.ReadLine(); //让控制台暂停,否则一闪而过了
}
catch (WebException webEx)
{
Console.WriteLine(webEx.Message.ToString());
}
}
}
}