C# 小爬虫
一个C# 小爬虫 用于爬取标书信息
using System;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using System.IO.Compression;
using System.Security.Cryptography.X509Certificates;
using System.Net.Security;
using System.Globalization;
namespace crawler
{
public partial class JunXiang : Form
{
public JunXiang()
{
InitializeComponent();
}
private void button1_Click(object sender, EventArgs e)
{
int page = 10;
try
{
page = int.Parse(textBox2.Text)+1;
if (page > 10000)
page = 9999;
}
catch {
}
webBrowser1.Document.OpenNew(false);
textBox1.Text = "";
HttpHelper http = new HttpHelper();
String str = "";
bool flag = true;
for(int i=1;iif (flag == false)
break;
DateTime start = dateTimePicker1.Value;
DateTime end = dateTimePicker2.Value;
HttpItem item = new HttpItem()
{
URL = "http://www.sczfcg.com/CmsNewsController.do?method=recommendBulletinList&moreType=provincebuyBulletinMore&channelCode=shiji_cggg&rp=25&page="+i,//URL 必需项
Method = "get",//URL 可选项 默认为Get
ContentType = "text/html",//返回类型 可选项有默认值
//ContentType = "application/x-www-form-urlencoded",//返回类型 可选项有默认值
};
//请求的返回值对象
HttpResult result = http.GetHtml(item);
//获取请请求的Html
string html = result.Html;
//string regex = "href=[\\\"\\\'](http:\\/\\/|\\.\\/|\\/)?\\w+(\\.\\w+)*(\\/\\w+(\\.\\w+)?)*(\\/|\\?\\w*=\\w*(&\\w*=\\w*)*)?[\\\"\\\']";
//Regex re = new Regex(regex);
//MatchCollection matches = re.Matches(html);
//System.Collections.IEnumerator enu = matches.GetEnumerator();
//while (enu.MoveNext() && enu.Current != null)
//{
// Match match = (Match)(enu.Current);
// Console.Write(match.Value + "\r\n");
//}
// Regex reg = new Regex(@"(?is)([^>]+?)");
Regex reg = new Regex(@"(.|\s)*? ", RegexOptions.IgnoreCase);
MatchCollection mc = reg.Matches(html);
foreach (Match m in mc)
{
if (m.ToString().IndexOf("采购公告") > -1)
{
string regex = "href=[\\\"\\\'](http:\\/\\/|\\.\\/|\\/)?\\w+(\\.\\w+)*(\\/\\w+(\\.\\w+)?)*(\\/|\\?\\w*=\\w*(&\\w*=\\w*)*)?[\\\"\\\']";
Regex reg1 = new Regex(regex);
MatchCollection matches = reg1.Matches(m.ToString());
Regex reg4 = new Regex(@"(?is)([^>]+?)" );
MatchCollection matches4 = reg.Matches(ToString());
if (matches.Count > 0 && (m.ToString().IndexOf(textBox3.Text) > -1 || textBox3.Text.Length == 0))
{
Regex reg3 = new Regex(@"(.|\s)*?", RegexOptions.IgnoreCase);
MatchCollection matches3 = reg3.Matches(m.ToString());
DateTimeFormatInfo dtFormat = new DateTimeFormatInfo();
dtFormat.ShortDatePattern = "yyyy-MM-dd";
String now = matches3[0].ToString();
string start_date = start.ToShortDateString().ToString();
string end_date = end.ToShortDateString().ToString();
now = now.Replace("", "").Replace("", "");
DateTime dt = Convert.ToDateTime(now, dtFormat);
start = Convert.ToDateTime(start_date, dtFormat);
end = Convert.ToDateTime(end_date, dtFormat);
if (DateTime.Compare(start, dt) > 0 || DateTime.Compare(dt, end) > 0)
{
break;
}
if((DateTime.Compare(start, dt) > 0))
flag = false;
textBox1.Text += matches3[0] + "\r\n";
str += m.ToString();
textBox1.Text += matches[0].Groups[0] + "\r\n";
Regex reg2 = new Regex(@" title=""(.|\s)*?"" target");
MatchCollection matches2 = reg2.Matches(m.ToString());
textBox1.Text += matches2[0].Value + "\r\n";
textBox1.Text += "***** " + "第"+(i)+"页"+" *****\r\n";
textBox1.Text = textBox1.Text.Replace("href=", "");
textBox1.Text = textBox1.Text.Replace("\"", "");
textBox1.Text = textBox1.Text.Replace(" title=", "");
textBox1.Text = textBox1.Text.Replace(" target", "");
textBox1.Text = textBox1.Text.Replace("", "");
textBox1.Text = textBox1.Text.Replace("", "");
}
}
}
}
str += "";
str = str.Replace("/view", "http://www.sczfcg.com/view");
textBox1.Text = textBox1.Text.Replace("/view", "http://www.sczfcg.com/view");
webBrowser1.Document.Write(str);
}
private void textBox1_TextChanged(object sender, EventArgs e)
{
}
private void Form1_Load(object sender, EventArgs e)
{
webBrowser1.Navigate("about:blank");
webBrowser1.Document.OpenNew(true);
}
private void textBox3_TextChanged(object sender, EventArgs e)
{
}
private void linkLabel1_LinkClicked(object sender, LinkLabelLinkClickedEventArgs e)
{
}
private void textBox2_TextChanged(object sender, EventArgs e)
{
}
private void button2_Click(object sender, EventArgs e)
{
webBrowser1.Document.OpenNew(false);
textBox1.Text = "";
HttpHelper http = new HttpHelper();
String str = "";
str += "";
webBrowser1.Document.Write(str);
}
}
public class HttpHelper
{
#region 预定义方变量
//默认的编码
private Encoding encoding = Encoding.Default;
//Post数据编码
private Encoding postencoding = Encoding.Default;
//HttpWebRequest对象用来发起请求
private HttpWebRequest request = null;
//获取影响流的数据对象
private HttpWebResponse response = null;
//设置本地的出口ip和端口
private IPEndPoint _IPEndPoint = null;
#endregion
#region Public
/// , RegexOptions.IgnoreCase);
string c = string.Empty;
if (meta != null && meta.Groups.Count > 0)
{
c = meta.Groups[1].Value.ToLower().Trim();
}
if (c.Length > 2)
{
try
{
encoding = Encoding.GetEncoding(c.Replace("\"", string.Empty).Replace("'", "").Replace(";", "").Replace("iso-8859-1", "gbk").Trim());
}
catch
{
if (string.IsNullOrEmpty(response.CharacterSet))
{
encoding = Encoding.UTF8;
}
else
{
encoding = Encoding.GetEncoding(response.CharacterSet);
}
}
}
else
{
if (string.IsNullOrEmpty(response.CharacterSet))
{
encoding = Encoding.UTF8;
}
else
{
encoding = Encoding.GetEncoding(response.CharacterSet);
}
}
}
}
/// 