多线程采集器的核心代码

自己写的一个采集程序，用来采集小说，将线程控制的代码发出来。

通过采集的数据数量来分配线程数，可以限制最大线程数量

采集数据采取自动分配的方式，为每个线程平均分配一定量的采集任务，具体采集部分用WebClient或者其他方式都可

ManualResetEvent 用来控制线程的挂起和恢复

通过建立委托来实现进度条的刷新和控件提示文本的刷新

 

  1  using System;
  2  using System.Collections.Generic;
  3  using System.Drawing;
  4  using System.Windows.Forms;
  5 
  6  using System.Threading;
  7  using System.Text.RegularExpressions;
  8 
  9  namespace GUIDemo
 10 {
 11      public  partial  class MainForm : Form
 12     {
 13          public MainForm()
 14         {
 15             InitializeComponent();
 16         }
 17         
 18         ManualResetEvent  mReset =  new ManualResetEvent( true);
 19         
 20         List<Thread> threads =  new List<Thread>();
 21         
 22          void BtnStartClick( object sender, EventArgs e)
 23         {
 24              if(workStatus == WorkStatus.Pause)
 25             {
 26                  if(threads.Count> 0)
 27                 {
 28                     mReset.Set();
 29                 }
 30                 workStatus = WorkStatus.Working;
 31                 SetButtonStatus();
 32                  return;
 33             }
 34             
 35             workStatus = WorkStatus.Working;
 36             SetButtonStatus();
 37             
 38             List<ALink> links =  new List<ALink>();
 39             
 40              for( int i= 0;i< 76;i++)
 41             {
 42                 links.Add( new ALink(i.ToString()));
 43             }
 44             
 45             progressBar1.Maximum =  76;
 46             progressBar1.Value =  0;
 47             th_cnt =  0;
 48             
 49              int dataCount = links.Count;
 50              int perCount =  25;
 51             
 52              int threadCount = dataCount % perCount == 0? dataCount/perCount:dataCount/perCount+ 1;
 53             
 54              int limitCount =  3;
 55              // 限制最大线程数量
 56               if(threadCount>limitCount)
 57             {
 58                 threadCount = limitCount;
 59                 perCount = dataCount % threadCount== 0?dataCount/threadCount:dataCount/threadCount+ 1;
 60             }
 61             
 62              int cnt =  0;  int i_cnt= 0;
 63             
 64             List<ALink> perLinks =  new List<ALink>();
 65              // 分配资源
 66               foreach(ALink link  in links)
 67             {
 68                  if(cnt%perCount== 0&&cnt> 0)
 69                 {
 70                     Thread thread =  new Thread( new ParameterizedThreadStart(DoWork));
 71                     thread.Name =  " thread "+i_cnt.ToString();
 72                     thread.Start(Clone(perLinks));
 73                     threads.Add(thread);
 74                     i_cnt++;
 75                     perLinks.Clear();
 76                 }
 77                 perLinks.Add(link);
 78                 cnt++;
 79             }
 80             
 81              if(perLinks.Count> 0&&i_cnt<threadCount)
 82             {
 83                 Thread thread =  new Thread( new ParameterizedThreadStart(DoWork));
 84                 thread.Name =  " thread "+i_cnt.ToString();
 85                 thread.Start(Clone(perLinks));
 86                 threads.Add(thread);
 87                 i_cnt++;
 88                 perLinks.Clear();
 89             }
 90             
 91              // Thread.Sleep(1000);
 92               // flagExit = true;
 93          }
 94         
 95          public  static  bool flagExit =  false;
 96         
 97          static  object locker =  new  object();
 98         
 99          static  int th_cnt =  0;
100         
101          void DoWork( object threadParam)
102         {
103             List<ALink> items = threadParam  as List<ALink>;
104              foreach (ALink link  in items.ToArray())
105             {
106                 mReset.WaitOne();
107                  if(flagExit)  break;
108                  lock (locker) {
109                     th_cnt++;
110                     UpdateProcess(th_cnt);
111                 }
112                 OutputText(Thread.CurrentThread.Name+ " : "+link.Url);
113                 ShowText(link.Url);
114                 
115                 Thread.Sleep( 50);
116             }
117              if(flagExit)
118             {
119                 OutputText(Thread.CurrentThread.Name+ " :exit ");
120             } else{
121                 OutputText(Thread.CurrentThread.Name+ " :finlished ");
122             }
123             OutputText(th_cnt.ToString());
124         }
125         
126          delegate  void ControlHandler( string text);
127          delegate  void ProcessBarHandler( int i);
128         
129          void ShowText( string text)
130         {
131              if(!label1.InvokeRequired)
132             {
133                 label1.Text = text;
134             }
135              else
136             {
137                 ControlHandler show =  new ControlHandler(ShowText);
138                 BeginInvoke(show, new  object[]{text});
139             }
140         }
141         
142          void UpdateProcess( int i)
143         {
144              if(!progressBar1.InvokeRequired)
145             {
146                 progressBar1.Value = i>progressBar1.Maximum ? progressBar1.Maximum:i;
147             }
148              else
149             {
150                 ProcessBarHandler show =  new ProcessBarHandler(UpdateProcess);
151                 BeginInvoke(show, new  object[]{i});
152             }
153         }
154         
155          static List<T> Clone<T>(IEnumerable<T> oldList) 
156         {     
157              return  new List<T>(oldList); 
158         }
159         
160          void OutputText( string text)
161         {
162             System.Diagnostics.Debug.WriteLine(text);
163         }
164         
165          void ReleaseThread()
166         {
167              if(threads.Count> 0)
168             {
169                  foreach(Thread th  in threads)
170                 {
171                     th.Abort();
172                 }
173                 threads.Clear();
174             }
175         }
176         
177          void MainFormFormClosing( object sender, FormClosingEventArgs e)
178         {
179             ReleaseThread();
180         }
181         
182          void BtnStopClick( object sender, EventArgs e)
183         {
184              if(workStatus== WorkStatus.Working || workStatus== WorkStatus.Pause)
185             {
186                 ReleaseThread();
187                 
188                 progressBar1.Value =  0;
189                 progressBar1.Refresh();
190                 
191                 th_cnt =  0;
192                 
193                 workStatus = WorkStatus.Stop;
194                 SetButtonStatus();
195             }
196         }
197         
198         WorkStatus workStatus = WorkStatus.Waitting;
199         
200          void BtnPauseClick( object sender, EventArgs e)
201         {
202              if(workStatus == WorkStatus.Working)
203             {
204                  if(threads.Count> 0)
205                 {
206                     mReset.Reset();
207                     workStatus = WorkStatus.Pause;
208                     SetButtonStatus();
209                 }
210             }
211         }
212         
213          void SetButtonStatus()
214         {
215              switch(workStatus)
216             {
217                  case WorkStatus.Working:
218                     btnStart.Enabled =  false;
219                     btnPause.Enabled =  true;
220                     btnStop.Enabled =  true;
221                      break;
222                  case WorkStatus.Pause:
223                     btnStart.Enabled =  true;
224                     btnPause.Enabled =  false;
225                     btnStop.Enabled =  true;
226                      break;
227                  case WorkStatus.Stop:
228                     btnStart.Enabled =  true;
229                     btnPause.Enabled =  false;
230                     btnStop.Enabled =  false;
231                      break;
232                  case WorkStatus.Waitting:
233                     btnStart.Enabled =  true;
234                     btnPause.Enabled =  false;
235                     btnStop.Enabled =  false;
236                      break;        
237             }
238         }
239         
240          void MainFormLoad( object sender, EventArgs e)
241         {
242             SetButtonStatus();
243         }
244     }
245     
246      public  class ALink
247     {
248          public  string Url= "";
249          public ALink( string url)
250         {
251              this.Url = url;
252         }
253     }
254     
255      enum WorkStatus
256     {
257         Pause,Stop,Working,Waitting
258     }
259 }