新闻采集源码可自写规则
using System;
using System.Data;
using System.Configuration;
using System.Collections;
using System.Web;
using System.Web.Security;
using System.Web.UI;
using System.Web.UI.WebControls;
using System.Web.UI.WebControls.WebParts;
using System.Web.UI.HtmlControls;
using System.Text.RegularExpressions;
using System.Collections;
using System.IO;
using System.Net;
using System.Text;
namespace NewsCollection
{
public partial class SiteEdit : System.Web.UI.Page
{
string urlData = "";
protected void Page_Load(object sender, EventArgs e)
{
this.BtCollection.Attributes.Add("onclick", "Status.showInfo('加载中');");
if (!Page.IsPostBack)
{
AjaxAction();
BindData();
}
}
public string GetRequest(string key)
{
key = Convert.ToString(Request[key]??"");
key = key==null?(""):(key);
return key;
}
public void AjaxAction()
{
string isAjax = GetRequest("isAjax").ToLower();
if (isAjax == "true")
{
string state = "";
string action = this.GetRequest("action").ToLower();
string values = this.GetRequest("values");
if (action == "newsbody")
{
string modelstart = this.GetRequest("modelstart");
string modelend = this.GetRequest("modelend");
string modelbody = this.GetRequest("modelbody");
string siteUrl = this.GetRequest("siteUrl");
ArrayList al = this.GetModelData(modelstart, modelend, modelbody, siteUrl);
StringBuilder sb = new StringBuilder();
foreach (string s in al)
{
sb.Append(s);
}
state = sb.ToString();
}
else if (action == "newsdetail")
{
string modelstart = this.GetRequest("modelstart");
string modelend = this.GetRequest("modelend");
string modelbody = this.GetRequest("modelbody");
string siteUrl = this.GetRequest("siteUrl");
string newsTitleStart = this.GetRequest("newsTitleStart");
string newsTitleEnd = this.GetRequest("newsTitleEnd");
string newsContentStart = this.GetRequest("newsContentStart");
string newsContentEnd = this.GetRequest("newsContentEnd");
state = GetNews(GetModelData(modelstart, modelend, modelbody, siteUrl), newsTitleStart, newsTitleEnd, newsContentStart, newsContentEnd);
}
else
{
state = "test Ajax";
}
Response.Clear();
Response.Write(state);
Response.End();
}
}
public void BindData()
{
string Gid = Convert.ToString(Request["Gid"] ?? "");
if (Gid.Length > 0)
{
Beans.Sites sites = new Beans.Sites();
sites.Gid = Gid;
sites = sites.SelectById();
TbSiteName.Text = sites.SiteName;
TbSiteUrl.Text =sites.SiteUrl;
TbSiteModelStart.Text = sites.SiteModelStart;
TbSiteModelEnd.Text =sites.SiteModelEnd;
TbSiteModelBody.Text = sites.SiteModelBody;
TbNewsTitleStart.Text =sites.NewsTitleStart;
TbNewsTitleEnd.Text =sites.NewsTitleEnd;
TbNewsContentStart.Text =sites.NewsContentStart;
TbNewsContentEnd.Text =sites.NewsContentEnd;
}
}
protected void BtEdit_Click(object sender, EventArgs e)
{
string message = "系统错误请重试";
string script = "history.go(-1)";
string Gid = Convert.ToString(Request["Gid"]??"");
Beans.Sites sites = new Beans.Sites();
sites.SiteName = TbSiteName.Text.Trim();
sites.SiteUrl = TbSiteUrl.Text.Trim();
sites.SiteModelStart = TbSiteModelStart.Text.Trim();
sites.SiteModelEnd = TbSiteModelEnd.Text.Trim();
sites.SiteModelBody = TbSiteModelBody.Text.Trim();
sites.NewsTitleStart = TbNewsTitleStart.Text.Trim();
sites.NewsTitleEnd = TbNewsTitleEnd.Text.Trim();
sites.NewsContentStart = TbNewsContentStart.Text.Trim();
sites.NewsContentEnd = TbNewsContentEnd.Text.Trim();
if (Gid.Length > 0)
{
sites.Gid=Gid;
if (sites.Update())
{
message = "修改成功！";
}
}
else
{
if (sites.Add())
{
message = "添加成功！";
}
}
Response.Write("<script type='text/javascript'>alert('"+message+"');"+script+"</script>");
}
protected void BtCollection_Click(object sender, EventArgs e)
{
GetNews(GetModelData(TbSiteModelStart.Text.Replace("(", "\\(").Replace(")", "\\)").Replace(".", "\\."), TbSiteModelEnd.Text.Replace("(", "\\(").Replace(")", "\\)").Replace(".", "\\."), TbSiteModelBody.Text, TbSiteUrl.Text), TbNewsTitleStart.Text.Trim(), TbNewsTitleEnd.Text.Trim(), TbNewsContentStart.Text.Trim(), TbNewsContentEnd.Text.Trim());
// GetNews(GetModelData(TbSiteModelStart.Text.Replace("(", "\\(").Replace(")", "\\)").Replace(".", "\\."), TbSiteModelEnd.Text.Replace("(", "\\(").Replace(")", "\\)").Replace(".", "\\."), TbSiteModelBody.Text, TbSiteUrl.Text));
}
public ArrayList GetModelData(string modelstart,string modelend,string modelbody,string SiteUrl)
{
ArrayList al = new ArrayList();
string content = GetHttpData(SiteUrl,"gb2312");
Regex reg = new Regex(modelstart+"(?<newsBody>[\\s\\S]*)"+modelend, RegexOptions.IgnoreCase | RegexOptions.Multiline);
Match mat = reg.Match(content);
//TbContent.Text = mat.Groups["newsBody"].Value.ToString();
Regex regurl = new Regex(modelbody.Replace("_url_", "(?<url>[^\"'\\s]+)"), RegexOptions.IgnoreCase | RegexOptions.Singleline);
Match maturl = regurl.Match(mat.Groups["newsBody"].Value.ToString());
while (maturl.Success)
{
//z-zA-A0-9/\\.:
string temp = maturl.Groups["url"].Value;
al.Add(temp);
Response.Write(temp.StartsWith("http://") ? (temp) : (temp.Insert(0, "http://" + SiteUrl.Replace("http://","").Substring(0, SiteUrl.LastIndexOf("/")))) + "<br>");
maturl = maturl.NextMatch();
}
return al;
}
public string GetNews(ArrayList al,string titleStart,string titleEnd,string contentStart,string contetnEnd)
{
StringBuilder sb = new StringBuilder();
if (al != null)
{
foreach (string s in al)
{
Regex reg = new Regex(titleStart.Replace("(", "\\(").Replace(")", "\\)") + "(?<title>[^<]*)" + titleEnd.Replace("(", "\\(").Replace(")", "\\)") + "[\\s\\S]+" + contentStart.Replace("(", "\\(").Replace(")", "\\)") + "(?<content>[\\s\\S]+)" + contetnEnd.Replace("(", "\\(").Replace(")", "\\)"), RegexOptions.IgnoreCase | RegexOptions.Multiline);
Match mat = reg.Match(GetHttpData(s, "gb2312"));
//Response.Write(string.Format("news:{0}<br>content:{1}<br>",mat.Groups["title"].Value,mat.Groups["content"].Value));
Beans.News news = new Beans.News();
news.Title = mat.Groups["title"].Value;
news.Typeid = Convert.ToString(Request["Gid"]);//mat.Groups["title"].Value;
news.From = this.TbSiteName.Text;//mat.Groups["content"].Value;
news.Content = mat.Groups["content"].Value;
news.Add();
sb.AppendFormat("title:{0}", mat.Groups["title"].Value);
}
}
return sb.ToString();
}
public string GetHttpData(string sUrl, string encoding)
{
string sRslt = null;
WebResponse oWebRps = null;
WebRequest oWebRqst = WebRequest.Create(sUrl);
oWebRqst.Timeout = 50000;
try
{
oWebRps = oWebRqst.GetResponse();
}
finally
{
if (oWebRps != null)
{
StreamReader oStreamRd = new StreamReader(oWebRps.GetResponseStream(), System.Text.Encoding.GetEncoding(encoding));
sRslt = oStreamRd.ReadToEnd();
oStreamRd.Close();
oWebRps.Close();
}
}
return sRslt;
}
}
}