proquest的采集程序demo测试截图

proquest的采集程序demo测试截图

 

程序C#.net开发，程序使用了 Castle Activerecord, Nhibernate ,Spring.net, Memcached分布式数据排重,

自动分页，错误自动重试，超时处理等自能纠错技术，，

配置 部分

 

  </configSections>

  <appSettings>
    <add key="startId" value="592" />
    <add key="endId" value="600" />
    <add key="startPId" value="0" />
    <add key="Cookies" value="" />
    <add key="DBID" value="10767" />
    <add key="SINDEX" value="10" />
    <add key="EINDEX" value="20" />
    <add key="PM" value="53153" />
    <add key="PC" value="6490441" />
    <add key="D" value="437140591" />
    <add key="backupData" value="53153,6490441,437140541,2315" />
  </appSettings>

 

部分代码

rtn = Zfr.Common.Helper.GetPage(url, cookies, Encoding.GetEncoding("iso-8859-1"), url, null);
        if (rtn.IndexOf("LOGIN ERROR MESSAGE") != -1) {
            Zfr.Common.ConsoleProgram.ShowError("线程->LOGIN ERROR,自动登录");
            System.Threading.Thread.Sleep(3000);
            GetCookies(rtn);
            goto p;
        }
           if (rtn.IndexOf("The remote name could not be resolved") != -1 ||
           rtn.IndexOf("has timed out") != -1 ||
           rtn.IndexOf("Bad Request") != -1)
            {
                error++;
                if (error < 4)
                {
                    Zfr.Common.ConsoleProgram.ShowError("线程->错误,暂停4秒,自动重试");
                    System.Threading.Thread.Sleep(4000);
                    goto p;
                }
                else
                {
                    Zfr.Common.ConsoleProgram.ShowError("线程->错误,超过3次,回车跳过,继续"); Console.ReadLine();
                }

 

---------------------------------

  if (MemcachedBench.KeyExists("" + obj.HASH))
                        {
                            Zfr.Common.ConsoleProgram.ShowError("KEY->重复:" + obj.HASH);
                        }
                        else
                        {

--------------------------------------------------------------------

 

  if (m.Success)
                             {
                                 obj.magLink = m.Groups[0].Value;
                             }
                            DB<ProQuestItem>.Create(obj);
                            Zfr.Common.ConsoleProgram.ShowSuccess("DATA->保存成功 ID:" + obj.ID);
                            MemcachedBench.Set("" + obj.HASH, null); successCount++;
                            SetConfig("backupData", string.Format("{0},{1},{2},{3}", pmid, pcid, did, obj.ID));

----------------------------------------------------------

void ShowSpeed()
        {
            TimeSpan ts = DateTime.Now - startTime;
            double s = ts.TotalSeconds;
            Zfr.Common.ConsoleProgram.Show(string.Format("INFO->{11}|{8} {9} {10}|{5} {6} {7}|C:{4} SP:{1} {2} {3} SPD:{0}", 24 * 60 * 60 * successCount / s, ts.Hours, ts.Minutes, ts.Seconds,successCount, pmid, pcid, did,sIndex,eIndex,cIndex,dbid));
        }

------------------------------------------------

测试程序demo截图pro-qq1163551688.jpg