C#实现web信息自动抓取

C#实现web信息自动抓取

            NetBug--爬虫 V1.02开发笔记

背景

 

随着Internet的普及,网络信息正以极高的速度增长,在这么多数据中找到自己需要的信息是一件很繁琐的事情,找到需要的信息后如何获取也是件麻烦的事。这就需要Internet信息抓取程序来代替人工的操作。

所谓Internet信息抓取程序,就是程序会按照用户的关键词或关键网站来收集相应的信息,并提供给用户想要的信息格式。

 

信息量的增加会带来信息网站发布人员工作量的剧增,为实现信息发布系统实现信息自

动发布、减少工作人员工作量、即时跟踪最新信息,就需要自动信息提供程序,因此Internet信息抓取程序应运而生。

 

目标

 

实现自定义网站信息分类抓取,存入本地数据库、生成静态页面或其它用户定义的信息结构,并下载与信息相关的多媒体文件。

 

开发

 

l         目标站点结构分析

本步骤是准确抓取信息个关键。

首先要选择更新频率高的页面做为抓取地址,然后分析要抓取内容页面url特点。

然后分析要抓取信息页面的元素特性,比如标题位置,内容位置 等,得到定位标记点。

将以上信息写成自己的配置文件或存到数据库中。

每个网站都需要分析,写出单独的配置文件,供抓取程序使用。

 

l         信息提取

根据配置文件取得要抓取页面url,使用HttpWebRequest类获取内容:

//获取http页面函数

        public string Get_Http(string a_strUrl,int timeout)

        {

            string strResult ;        

 

            try

            {

HttpWebRequest myReq = (HttpWebRequest)HttpWebRequest.Create(a_strUrl) ;

                myReq.Timeout = timeout;

                HttpWebResponse HttpWResp = (HttpWebResponse)myReq.GetResponse();

           

                Stream myStream = HttpWResp.GetResponseStream () ;

 

                StreamReader sr = new StreamReader(myStream , Encoding.Default);

                StringBuilder strBuilder = new StringBuilder();

                while (-1 != sr.Peek())

                {

                    strBuilder.Append(sr.ReadLine()+"/r/n");

                }

 

                strResult = strBuilder.ToString();

            }

            catch(Exception exp)

            {

                strResult = "错误:" + exp.Message ;

            }

 

            return strResult ;

 

        }

获取页面内容后,分析页面中连接地址取到要抓取的url:

//处理页面标题和链接

        public string SniffWebUrl( string urlStr,string blockB,string blockE )

        {      

            string urlch1 = "";

            string urlch2 = "";                   

            int end_n1 = 0;

            int end_nums = 0;

            int end_nums1 = 0;

            int end_nums2 = 0;

            int end_nums3     = 0;           

            string reUTStr = "";

            string reTitle = "";

            string ret = "";          

            try

            {

                int pos01 = urlStr.IndexOf( "." );

                int pos02 = urlStr.LastIndexOf( "/" );

                if( pos01 < 0 )

                {

                    return "";

                }

                if( pos02 < 0 )

                {

                    return "";

                }

                int pos03 = urlStr.IndexOf( "/",pos01 );

                if ( pos03 < 0 )

                {

                    urlch1 = urlStr;

                    urlch2 = urlStr;

                }

                else

                {

                    urlch1 = urlStr.Substring( 0,pos03 );

                    urlch2 = urlStr.Substring( 0,pos02 );

                }

 

                string tmpAllStr = new PublicFun().Get_Http( urlStr ,time1);

 

                int pos1 = tmpAllStr.IndexOf( blockB );

                int pos2 = tmpAllStr.IndexOf( blockE,pos1 + blockB.Length );

                if ( pos1>0 && pos2>0 && pos2>pos1 )

                {

                    ret = tmpAllStr.Substring( pos1 + blockB.Length,pos2 - pos1 - blockB.Length );

                    ret = ret.Substring( ret.IndexOf( "<" ));

                    while( ret.IndexOf( "= 0 )

                    {

                        ret = ret.Substring( 0,ret.IndexOf( "

                    }

                    while( ret.IndexOf( "=0 )

                    {

                        ret = ret.Substring( 0,ret.IndexOf( "

                    }

                    while( ret.IndexOf( "Href=" ) >=0 )

                    {

                        ret = ret.Substring( 0,ret.IndexOf( "Href=" )) + "href=" + ret.Substring( ret.IndexOf( "Href=" ) + 5 );

                    }

                    while( ret.IndexOf( "HREF=" ) >=0 )

                    {

                        ret = ret.Substring( 0,ret.IndexOf( "HREF=" )) + "href=" + ret.Substring( ret.IndexOf( "HREF=" ) + 5 );

                    }

                    while( ret.IndexOf( "href='" ) >=0 )

                    {

                        ret = ret.Substring( 0,ret.IndexOf( "href='" )) + "href=/"" + ret.Substring( ret.IndexOf( "href='" ) + 6 );

                    }

                }      

                tmpAllStr = ret;     

                int begin_nums = tmpAllStr.IndexOf( "href=" );

 

                while ( begin_nums >= 0 )

                {              

                    string tmpStrA = "";

                    string tmpStrB = tmpAllStr.Substring( begin_nums + 5,1 );

                    if ( tmpStrB == "/"" )

                    {

                        end_n1 = begin_nums + 6;

                        if ( ( end_n1 + 1 ) > tmpAllStr.Length )

                        {

                            return "";

                        }

                        tmpStrA = tmpAllStr.Substring( begin_nums+6,1 );

                    }

                    else

                    {

                        end_n1 = begin_nums + 5;

                        tmpStrA = tmpStrB;

                    }

 

                    if ( tmpStrA == "#" )

                    {

                        tmpAllStr = tmpAllStr.Substring( end_n1 );

                        begin_nums = tmpAllStr.IndexOf( "href=" );

                    }

                    else

                    {                  

                        end_nums1 = tmpAllStr.IndexOf( " ",end_n1 );

                        end_nums2 = tmpAllStr.IndexOf( ">",end_n1 );

                        end_nums3 = tmpAllStr.IndexOf( "

 

                        if ( ( end_nums3 >= 0 ) && ( end_nums2 >= 0 ) )

                        {

                            reTitle = tmpAllStr.Substring( end_nums2 + 1,end_nums3 - end_nums2 - 1 );

 

                            if ( end_nums1 > end_nums2 )

                            {

                                end_nums = end_nums2;

                            }

                            else

                            {

                                if ( end_nums1 < 0 )

                                {

                                    end_nums = end_nums2;

                                }

                                else

                                {

                                    end_nums = end_nums1;

                                }

                            }

                            string str4 = tmpAllStr.Substring( end_nums-1, end_nums - end_nums + 1 );

 

                            if ( str4 =="/""  || str4 == "'" )

                            {

                                end_nums = end_nums - 1;

                            }

                            string sTotalOne = tmpAllStr.Substring( end_n1,end_nums - end_n1 );

 

                            if ( sTotalOne.IndexOf( "http://" ) <0 )

                            {

                                if ( sTotalOne.IndexOf( "/" ) == 0 )

                                {

                                    sTotalOne = urlch1 + sTotalOne;

                                }

                                else

                                {                              

                                    int linshiIntNum = 0;

                                    int flags = 0;

                                    string urlChange = urlStr;;

                                    while( sTotalOne.IndexOf( "../" ) >= 0 )

                                    {

                                        sTotalOne = sTotalOne.Substring( sTotalOne.IndexOf( "../" ) + 3 );

                                        linshiIntNum = linshiIntNum + 1;

                                        flags = flags +1;

                                    }

                                    while( ( urlChange.LastIndexOf( "/" ) >= 0 ) && ( linshiIntNum >= 0 ) )

                                    {

                                        urlChange = urlChange.Substring( 0,urlChange.LastIndexOf( "/" ) );

                                        linshiIntNum = linshiIntNum - 1;

                                    }

                                    if ( flags == 0 )

                                    {

                                        sTotalOne = urlch2 + "/" + sTotalOne;

                                    }

                                    else

                                    {

                                        sTotalOne = urlChange + "/" + sTotalOne;

                                    }

                                }

                            }

                            reUTStr = reUTStr + new PublicFun().RemoveHtmlCode( reTitle ) + sTotalOne;

 

                            tmpAllStr = tmpAllStr.Substring( end_nums3 + 4 );

                            begin_nums = tmpAllStr.IndexOf( "href=" );

                        }

                        else

                        {

                            begin_nums = -1;

                        }                   

                    }

                }

                return reUTStr;

            }

            catch( Exception e)

            {

                return "";

            }

        }

 

得到要抓取内容的url后,处理该页面:

//获取链接内容并分类处理

        public string GetWebContent( string gatherUrl,string subUrl,string subTitle,string b_Content,string e_Content,string b_Filter,string e_Filter,string root )

        {

            string tmpAllStr = "";           

            string dfStrB = "";

            string dfStrE = "";               

            string rePicStr = "";//图片返回路径   

            string reContentStr = "";

            string picHtml = "images"; //本地图片路径

           

            string urlch1 ="";

            string urlch2 ="";

            int pos1 = gatherUrl.IndexOf( "." );

            int pos2 = gatherUrl.LastIndexOf( "/" );

            if( pos1 < 0 )

            {

                return "";

            }

            if( pos2 < 0 )

            {               

                return "";

            }

            int pos3 = gatherUrl.IndexOf( "/",pos1 );

            if ( pos3 < 0 )

            {

                urlch1 = gatherUrl;

                urlch2 = gatherUrl;

            }

            else

            {

                urlch1 = gatherUrl.Substring( 0,pos3 );

                urlch2 = gatherUrl.Substring( 0,pos2 );

            }   

           

            tmpAllStr = new PublicFun().Get_Http( subUrl,time1 );

            //取稿源

            string docFromStr = "";

            if ( dfStrB != "" && dfStrE != "" )

            {

                if ( tmpAllStr != "" )

                {

                    int b_docF = tmpAllStr.IndexOf( dfStrB );

                    if ( b_docF > 0 )

                    {

                        int e_docF = tmpAllStr.IndexOf( dfStrE,b_docF + dfStrB.Length );

                        if ( e_docF > 0 && e_docF > b_docF && e_docF - b_docF < 20 )

                        {

                            docFromStr = tmpAllStr.Substring( b_docF + dfStrB.Length, e_docF - b_docF - dfStrB.Length );

                        }

                    }

                }

            }

            //取内容

            if ( tmpAllStr != "" )

            {               

                int begin_strnum = tmpAllStr.IndexOf( b_Content );

                if ( begin_strnum < 0 )

                {                  

                    return "";

                }

                int end_strnum = tmpAllStr.IndexOf( e_Content,begin_strnum + b_Content.Length );

                if ( end_strnum < 0 )

                {                  

                    return "";

                }

                string sTotalSubM = "";

                if ( end_strnum > begin_strnum )

                {

                    sTotalSubM = tmpAllStr.Substring ( begin_strnum,end_strnum - begin_strnum );

                }

               

                if ( sTotalSubM == "" )

                {                  

                    return "";

                }              

                //过滤无用信息

                int bfnum = sTotalSubM.IndexOf( b_Filter );

                if ( bfnum > -1 )

                {

                    int efnum = sTotalSubM.IndexOf( e_Filter,bfnum );

                    if ( efnum > -1 )

                    {

                        if ( efnum > bfnum )

                        {

                            sTotalSubM = sTotalSubM.Substring( 0,bfnum ) + sTotalSubM.Substring( efnum + e_Filter.Length );

                        }

                    }

                }

                //格式化图片标记

               

                while( sTotalSubM.IndexOf( "Src=" ) >= 0 )

                {

                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "Src=" ) ) + "src=" + sTotalSubM.Substring( sTotalSubM.IndexOf( "Src=" ) + 4 );

                }

                while( sTotalSubM.IndexOf( "SRC=" ) >= 0 )

                {

                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "SRC=" ) ) + "src=" + sTotalSubM.Substring( sTotalSubM.IndexOf( "SRC=" ) + 4 );

                }

                while( sTotalSubM.IndexOf( "src='" ) >= 0 )

                {

                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "src='" ) ) + "src=/"" + sTotalSubM.Substring( sTotalSubM.IndexOf( "src='" ) + 5 );

                }

 

                //取图片地址

                int end_n12 = 0;

                int end_nums2 = 0;

                int begin_nums2 = sTotalSubM.IndexOf( "src=" );

                while( begin_nums2 >= 0 )

                {

                    String tmpStr = sTotalSubM.Substring( begin_nums2 + 4,1 );

                    if ( tmpStr == "/"" )

                    {

                        end_n12 = begin_nums2 + 5;

                    }

                    else

                    {

                        end_n12 = begin_nums2 + 4;

                    }

                    int end_nums2a = sTotalSubM.IndexOf( " ",end_n12 );

                    int end_nums2b = sTotalSubM.IndexOf( ">",end_n12 );

                    if ( end_nums2b < 0 )

                    {

                        break;

                    }

                    if ( end_nums2a > end_nums2b )

                    {

                        end_nums2 = end_nums2b;

                    }

                    else

                    {

                        if (end_nums2a<0)

                        {

                            end_nums2 = end_nums2b;

                        }

                        else

                        {

                            end_nums2 = end_nums2a;

                        }

                    }

                    tmpStr = sTotalSubM.Substring( end_nums2-1,1 );

                    if ( tmpStr == "/"" || tmpStr == "'" )

                    {

                        end_nums2 = end_nums2 - 1;

                    }

                    string tmpPicStr = sTotalSubM.Substring( end_n12,end_nums2 - end_n12 );

 

                    if ( tmpPicStr.IndexOf( "http://" ) < 0 )

                    {

                        if ( tmpPicStr.IndexOf( "/" ) == 0 )

                        {

                            tmpPicStr = urlch1 + tmpPicStr;

                        }

                        else

                        {                           

                            int linshiIntNum = 0;

                            int flags = 0;

                            string urlChange = subUrl;

                            while( tmpPicStr.IndexOf( "../" ) >= 0 )

                            {

                                tmpPicStr = tmpPicStr.Substring( tmpPicStr.IndexOf("../") + 3 );

                                linshiIntNum = linshiIntNum + 1;

                                flags = flags + 1;

                            }

                            while( ( urlChange.LastIndexOf( "/" ) >= 0 ) && ( linshiIntNum >= 0 ) )

                            {

                                urlChange = urlChange.Substring( 0,urlChange.LastIndexOf( "/" ) );

                                linshiIntNum = linshiIntNum - 1;

                            }

                            if ( flags == 0 )

                            {

                                tmpPicStr = urlch2 + "/" + tmpPicStr;

                            }

                            else

                            {

                                tmpPicStr = urlChange + "/" + tmpPicStr;

                            }

                        }

                    }

                    //tmpPicStr = tmpPicStr.ToLower();

                    string tmpPicStrTmp = tmpPicStr.ToLower();

                    //if ( tmpPicStr.IndexOf( ".jpg" ) > 0 || tmpPicStr.IndexOf( ".gif" ) > 0 || tmpPicStr.IndexOf( ".bmp" ) > 0 )

                    if ( tmpPicStrTmp.IndexOf( ".jpg" ) > 0 || tmpPicStrTmp.IndexOf( ".gif" ) > 0 || tmpPicStrTmp.IndexOf( ".bmp" ) > 0 )

                    {

                        rePicStr = rePicStr + "||" + tmpPicStr ;

 

                        int flagN2 = tmpPicStr.LastIndexOf( "/" );

                        string fileN2 = picHtml + tmpPicStr.Substring( flagN2 );

                        sTotalSubM = sTotalSubM.Substring( 0,end_nums2 ) + ">******" + fileN2 + "******<" + sTotalSubM.Substring( end_nums2 );

 

                        begin_nums2 = sTotalSubM.IndexOf( "src=", end_nums2 + fileN2.Length + 22 );

                    }

                    else

                    {

                        begin_nums2 = sTotalSubM.IndexOf( "src=", end_nums2 + 4 );                       

                    }                  

                }

                if ( rePicStr.Length > 2 ) 

                    rePicStr =  rePicStr.Substring(2);              

 

                //内容处理 格式化关键标记

                while( sTotalSubM.IndexOf( "= 0 )

                {

                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "

                }

                while( sTotalSubM.IndexOf( "= 0 )

                {

                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "

                }

                while( sTotalSubM.IndexOf( "= 0 )

                {

                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "

                }

                while( sTotalSubM.IndexOf( "= 0 )

                {

                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "

                }

                while( sTotalSubM.IndexOf( "=0 )

                {

                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "

                }

                while( sTotalSubM.IndexOf( "= 0 )

                {

                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "

                }

                while( sTotalSubM.IndexOf( "= 0 )

                {

                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "

                }

                while( sTotalSubM.IndexOf( "= 0 )

                {

                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "

                }

 

                //去除html标记

                int linshiInt1 = sTotalSubM.IndexOf( "<" );

                int linshiInt2 = sTotalSubM.IndexOf( ">" );           

 

                if ( linshiInt2 < linshiInt1 )

                {

                    sTotalSubM = sTotalSubM.Substring( linshiInt2 + 1 );

                }

                int linshiInt11 = sTotalSubM.LastIndexOf( "<" );

                int linshiInt12 = sTotalSubM.LastIndexOf( ">" );

                if ( linshiInt12 < linshiInt11 )

                {

                    sTotalSubM = sTotalSubM.Substring( 0,linshiInt12 + 1 );

                }

                linshiInt1 = sTotalSubM.IndexOf( "<" );

                while ( linshiInt1 >= 0 )

                {

                    linshiInt2 = sTotalSubM.IndexOf( ">",linshiInt1 );

                    if ( linshiInt2 >= 0 )

                    {              

                        sTotalSubM = sTotalSubM.Substring( 0,linshiInt1 ) + sTotalSubM.Substring( linshiInt2 + 1 );

                    }

                    else

                    {

                        sTotalSubM = sTotalSubM.Substring( 0,linshiInt1 );

                    }

                    linshiInt1 = sTotalSubM.IndexOf("<");

                }

 

                //还原关键标记

                int linshiInt3 = 0;

                int linshiInt4 = 0;

 

                while( sTotalSubM.IndexOf( "+****+" ) >= 0 )

                {

                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "+****+" ) ) + "
/n" + sTotalSubM.Substring( sTotalSubM.IndexOf( "+****+" ) + 9 );

                }

                while( sTotalSubM.IndexOf( "|****|" ) >= 0 )

                {

                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "|****|" ) ) + "
/n" + sTotalSubM.Substring( sTotalSubM.IndexOf( "|****|" ) + 9 );

                }

                while( sTotalSubM.IndexOf( "******" ) >= 0 )

                {

                    linshiInt3 = sTotalSubM.IndexOf( "******" ) + 9;

                    linshiInt4 = sTotalSubM.IndexOf( "******",linshiInt3 );

                    if ( linshiInt4 >= 0 )

                    {

                        int tmpPos = sTotalSubM.IndexOf( "******" );

                        string tmpStr1 = sTotalSubM.Substring( 0,tmpPos );

                        string tmpStr2 = sTotalSubM.Substring( linshiInt3,linshiInt4 - linshiInt3 );

                        string tmpStr3 = sTotalSubM.Substring( linshiInt4 + 9 );

                        sTotalSubM = tmpStr1 + "" + tmpStr3;

                    }

                    else

                    {

                        break;

                    }

                }

                //去除内容中的标题

                if ( sTotalSubM.IndexOf( subTitle ) >= 0 )

                {

                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( subTitle ) ) + sTotalSubM.Substring( sTotalSubM.IndexOf( subTitle ) + subTitle.Length );

                }

                reContentStr = sTotalSubM;

                //调用下载图片功能

                //下载图片到指定目录

                string[] img_Url = new PublicFun().split( rePicStr,"||" );

                for ( int i=0;i

                {

                    if ( img_Url[i] != "" )

                    {

                        new PublicFun().Get_Img( img_Url[i],10000,root + "//images//" + img_Url[i].Substring( img_Url[i].LastIndexOf("/")+1 ) );

                    }

                }

            }

            return reContentStr;

 

        }

以上方法返回要取得的信息,包括标题内容,图片地址等。

 

下载页面中图片:

//下载图片

        public void Get_Img(string a_strUrl,int timeout,string filepath)

        {

            try

            {

HttpWebRequest myReq = (HttpWebRequest)HttpWebRequest.Create(a_strUrl) ;

                myReq.Timeout = timeout;

                HttpWebResponse HttpWResp = (HttpWebResponse)myReq.GetResponse();        

                Stream myStream = HttpWResp.GetResponseStream () ;         

                Bitmap map = new Bitmap( myStream );

                PictureBox picB = new PictureBox();

                picB.Image = (Image)map;

                string path = filepath.Substring( 0,filepath.LastIndexOf( "//" ) );

                if (!Directory.Exists(path))

                {

                    CreateDir( path );

                }              

                picB.Image.Save(filepath);               

            }

            catch(Exception exp)

            {

                string ss = exp.Message;

        WriteLog( filepath.Substring(0,filepath.LastIndexOf("//")) + "//error.log",a_strUrl + "--" + ss + "/r/n");       

            }

        }

l         保存文件或入库

上面取得的信息可以按自己的要求保存。

 

****设计的时候没有使用url按层次循环抓取,这样定义抓取url效率更高,速度更快。

 

 

测试程序下载:http://bjfile.focus.cn/file/15379/NetBugV102.rar

 

如有建议请发送EMail:[email protected] 或msn [email protected]

 

注:此版本只提供静态文件存储功能,不提供数据库接口,不提供自定义网站功能。

    本程序运行需要先安装.net 框架1.1。

 

 

 

                                         

你可能感兴趣的:(c#,c#,web,string,exception,internet,url)