GNUGK（2.2.3）源代码分析之线程池结构

GNUGK2.2.3一改之前2.0版本版本的单线程结构。采用了作业（Job，Task）-代理（Agent）-执行者（Worker）三层控制结构 。使整个体系结构更加易于扩展和清晰。

在这里主要分析Job，Task，Agent，Worker他们之间的关系，以及Job的派生类Jobs与RegularJob时等。

理解Job，Agent，Worker三个类之间的关系，需要跟大家复习一下操作系统中的程序，进程，和CPU调度程序。

程序，是一个程序代码，数据等的集合。

进程，是一个正在运行的抢占CPU资源的程序。是程序的一个实例。

CPU调度程序，决定了当前那个进程可以运行，可以获得CPU的控制权限。

在这里的Job和操作系统中的程序是类似的，我们只要在Job的派生类中的Run函数编写我们要执行的程序代码即可。然后创建该Job的一个实例，并把该Job提交给Agent，Agent就会为该Job非配一个Worker来执行Job中的Run代码。因此他们之间和操作系统的程序，进程，CPU调度程序的对应关系是：

Job-----------程序

Agent------CPU调度进程

Worker----正在运行的进程

/////////////////////////////////////////////////////job.h//////////////////////////////////////////////////////////////////////////////

//////////////////////////////////////////////////////////////////
//
// job.h
//
// Abstraction of threads' jobs
//
//////////////////////////////////////////////////////////////////

#ifndef JOB_H
#define JOB_H "@(#) $Id: job.h,v 1.6 2005/04/24 16:39:44 zvision Exp $"

#include "name.h"

/** The base abstract class that represents job objects.
    This class implements the way to execute the job.
    Derived classes implement actual jobs (override Run()).
 Jobs are executed and managed by internal Job managment system
 consisting of the singleton Agent object and Worker threads
 that accept and execute new Jobs.
 
 Specialized Job examples are:
    Jobs - executes series of Tasks
    RegularJob - executes a Task again and again until stopped
    SimpleJob - calls some funtion with one pointer-type argument
    SimpleJobA - calls some funtion with two arguments : pointer-type and reference
    SimpleClassJob - calls a member function of some class that takes no arguments
    SimpleClassJobA - calls a member function of some class that takes one
                   argument of reference type
*/

// Job，代表一个具体要执行的任务
class Job : public NamedObject
{
public:
 virtual ~Job();

 /// Perform the actual job, return when it is done
 /* Job 真正要执行的代码,只要在派生类实现该函数即可*/
 virtual void Run() = 0;

 /// Stop a running job
 virtual void Stop();

 /** Execute the job in a first idle Worker thread.
  The function returns immediatelly and this object
  is delete automatically, when the job is finished.
  使用该函数，把此job提交到Agent，由agent分配执行者
 */
 void Execute();

 /// Stop all jobs being currently executed by Worker threads
 static void StopAll();
};

/** Similar to the Job, but is even more abstract. It does not contain
 any Task management routines. Main purpose of this class it to provide
 a way to represent a serie of Tasks that are to be executed one after another.

Task是一个比Job更小的执行任务，如果把Job当作操作系统的批处理任务，那么Task只是其中的一个小任务
*/

class Task
{
public:
 Task() : m_next(NULL), m_done(false) {}
 virtual ~Task();

 /// Perform the actual task and return when it is finished

//Task具体要执行的代码，在其派生类的函数中要实现Exec这个函数。在调用Job->Run的时候会调用该函数
 virtual void Exec() = 0;

 /** @return
     true if the task is done and a next task is being processed.
 */
 bool IsDone() const { return m_done; }
 
 /// Setup a task to be executed when this one is done

//Task是一个单向链表结构，设置下一个Task
 void SetNext(
  /// next task to be executed
  Task* next
  )
 {
  if (m_next != NULL && m_next != this)
   m_next->SetNext(next);
  else
   m_next = next;
 }

 /** @return
     true if this is not the last task.

是否还有可执行的Task
 */
 bool HasNext() const { return m_next != NULL; }

 /** Get a next task and mark this one as done.
     @return
     The task that is next or NULL if this is the last task.

 执行完成一个Task以后，设置下一个要执行的Task，并标识该Task已经执行完成，

done=true
 */
 Task* DoNext()
 {
  Task* next = m_next;
  if (next != this) // do not set m_done flag for circular task
   m_done = true;
  return next;
 }

private:
 /// next task to be executed when this one is done
 Task* m_next;
 /// true if the task is finished
 bool m_done;
};

/// Execute a task or a serie of tasks

// 重复不断执行的Job，不是循环的执行某一段共同的代码
class Jobs : public Job
{
public:
 Jobs(
  /// task to be executed
  Task* task
  ) : m_current(task) {}

 /// process the associated task (override from Job)
 virtual void Run();

private:
 /// task (or a serie of tasks) to be executed
 Task* m_current;
};

/** Regular job - executes the same task until it is stopped
 (by calling Stop()). The actual work is to be done in the virtual
 Exec() method in derived classes. RegularJob is an abstract class.

顺序的执行多个Job，Job与Job之间可以没有任何关系
*/
class RegularJob : public Job
{
public:
 RegularJob();

 /** @return
  true if the job has not been yet stopped
 */
 bool IsRunning() const { return !m_stop; }

 /// override from Job
 virtual void Run();

 /// repeated activity to be executed by this RegularJob

//真正要执行的代码，在派生类中继承该函数，就是重复Job要执行的具体代码
 virtual void Exec() = 0;
  
 /** Stop this job. NOTE: Acquire m_deletionPreventer mutex first
     to make sure this object is not deleted before the method that
  called Stop returns (if Stop is called from a derived class).
 */
 virtual void Stop();

protected:
 // new virtual function

 /// Callback function that is called before the job is started.
 virtual void OnStart();

 /** Callback function that is called when the job is stopped.
  NOTE: This function is called with m_deletionPreventer mutex acquired.
 */
 virtual void OnStop();

 /// Wait for a signal (Signal())
 void Wait() { m_sync.Wait(); }
 
 /** Wait for a signal (Signal()).
  
     @return
     true if the sync point has been signalled.
 */
 bool Wait(
  /// time to wait for the sync point to be signalled
  const PTimeInterval& timeout
  )
 {
  return m_sync.Wait(timeout) ? true : false;
 }

 /// Send a signal to the waiting task
 void Signal() { m_sync.Signal(); }

protected:
 /// can be used when calling Stop to prevent the job to be deleted
 /// (and invalid object being referenced) before the function
 /// that called Stop returns
 PMutex m_deletionPreventer;
 
private:
 /// used by Wait and Signal member functions
 PSyncPoint m_sync;
 /// true if the job should be stopped
 bool m_stop;
};

用于创建Job的模板，把某个函数或者某个类的成员函数当作一个线程执行

也就是为函数f，分配一个线程来执行其代码

template<class F, class T>
class SimpleJob : public Job {
public:
 SimpleJob(const F & _f, T *_t) : f(_f), t(_t) {}
 virtual void Run() { f(t); }

private:
 const F f;
 T *t;
};

template<class F, class T, class A>
class SimpleJobA : public Job {
public:
 SimpleJobA(const F & _f, T *_t, const A & _a) : f(_f), t(_t), a(_a) {}
 virtual void Run() { f(t, a); }

private:
 const F f;
 T *t;
 A a;
};

template<class T>
class SimpleClassJob : public Job {
public:
 SimpleClassJob(T *_t, void (T::*_j)()) : t(_t), j(_j) {}
 virtual void Run() { (t->*j)(); }

private:
 T *t;
 void (T::*j)();
};

template<class T, class A>
class SimpleClassJobA : public Job {
public:
 typedef void (T::*CJob)(A);
 SimpleClassJobA(T *_t, CJob _j, A _a) : t(_t), j(_j), a(_a) {}
 virtual void Run() { (t->*j)(a); }

private:
 T *t;
 CJob j;
 A a;
};

template<class T>
void CreateJob(T *t, void (T::*j)(), const char *n)
{
 Job *newjob = new SimpleClassJob<T>(t, j);
 newjob->SetName(n);
 newjob->Execute();
}

template<class T, class A>
void CreateJob(T *t, void (T::*j)(A), A a, const char *n)
{
 Job *newjob = new SimpleClassJobA<T, A>(t, j, a);
 newjob->SetName(n);
 newjob->Execute();
}

#endif // JOB_H
////////////////////////////////////////////////////////job.cxx//////////////////////////////////////////////////////////////////////////////////

//////////////////////////////////////////////////////////////////
//
// job.cxx
//
//////////////////////////////////////////////////////////////////
#include <list>
#include <ptlib.h>
#include "stl_supp.h"
#include "rwlock.h"
#include "singleton.h"
#include "job.h"
#include "h323util.h"

// timeout (seconds) for an idle Worker to be deleted
#define DEFAULT_WORKER_IDLE_TIMEOUT (10*60)

/** This class represents a thread that performs jobs. It has two states:
    idle and busy. When it accepts a new Job, it becomes busy. When the job
    is finished it becomes idle. Each idle Worker is stopped (deleted) after
    the specified timeout, so Workers that are not needed anymore do not use
    system resources. This makes passible to create dynamic sets of Workers.
*/
class Agent;
class Worker : public PThread
{
public:
 PCLASSINFO(Worker, PThread)

 /// create a new Worker thread and start it immediatelly
 Worker(
  /// pointer to the Agent instance that the worker is run under control of
  Agent* agent,
  /// timeout (seconds) for this Worker to be deleted, if idle
  long idleTimeout = DEFAULT_WORKER_IDLE_TIMEOUT
  );
 
 ~Worker();

 /** Tell this Worker to execute a new Job. The function returns
  immediatelly and the job is executed under control of the Worker thread.
  After the job is finished, its object is deleted.
  
  @return
  true if this Worker is idle and has taken the Job, false otherwise
  (on failuer the job object is not deleted).
  提交job到worker线程，该job是worker下一个要执行的任务
 */
 bool Exec(
  /// Job to be executed
  Job* job
  );
  
 /** Stop the Worker thread and any jobs being executed,
     wait for Worker thread termination and delete this object.
     销毁Worker对象，不能使用delete，需要自我delete
 */
 void Destroy();

 PThreadIdentifer GetWorkerId(){ return m_id;};

private:
 // override from class PThread
 virtual void Main();

 /// idle timeout (seconds), after which the Worker is destoyed
 PTimeInterval m_idleTimeout;
 /// signals that either a new Job is present or the Worker is destroyed
 PSyncPoint m_wakeupSync;
 /// true if the Worker is being destroyed
 bool m_closed;
 /// for atomic job insertion and deletion
 PMutex m_jobMutex;
 /// actual Job being executed, NULL if the Worker is idle
 Job* m_job;
 /// Worker thread identifier
 PThreadIdentifer m_id;
 /// Agent singleton pointer to avoid unnecessary Instance() calls
 Agent* m_agent;
};

/** Agent singleton manages a set of Worker threads. It creates
    new Workers if required. Idle Workers are deleted automatically
 after configured idle timeout.
*/
class Agent : public Singleton<Agent>
{
public:
 Agent();
 ~Agent();

 /** Execute the job by the first idle Worker or a new Worker.
  Delete the Job object after it is done.
 */
 void Exec(
  /// the job to be executed
  Job* job
  );
  
 /** Remove the Worker from busy and idle lists.
  Called by the Worker when it deletes itself.
 */
 void Remove(
  /// the worker to be removed from the lists
  Worker* worker
  );

 /** Move the Worker from the busy list to the idle list.
  Called by the Worker when it finishes each job.
 */
 void JobDone(
  /// the worker to be marked as idle
  Worker* worker
  );
  
private:
 /// mutual access to Worker lists
 PMutex m_wlistMutex;
 /// list of idle Worker threads
 std::list<Worker*> m_idleWorkers;
 /// list of Worker threads executing some Jobs
 std::list<Worker*> m_busyWorkers;
 /// flag preventing new workers to be registered during Agent destruction
 bool m_active;
};

Worker::Worker(
 /// pointer to the Agent instance that the worker is run under control of
 Agent* agent,
 /// timeout (seconds) for this Worker to be deleted, if idle
 long idleTimeout
 )
 : PThread(5000, AutoDeleteThread),
 m_idleTimeout(idleTimeout*1000), m_closed(false), m_job(NULL), m_id(0),
 m_agent(agent)
{
 // resume suspended thread (and run Main)
 Resume();
}

Worker::~Worker()
{
 PWaitAndSignal lock(m_jobMutex);
 if (m_job) {
  DEBUG_TEST(1, "JOB/tDestroying Worker " << m_id << " with active Job " << m_job->GetName());
  delete m_job;
  m_job = NULL;
 }
 DEBUG_TEST(5, "JOB/tWorker " << m_id << " destroyed");
}

void Worker::Main()
{
 m_id = GetThreadId();
 DEBUG_TEST(5, "JOB/tWorker " << m_id << " started ");
 

只要该worker没有停止，就会检测时候有Job提交到该Worker，如果有就执行之，

否则就睡觉去，等待Job的到来
 while (!m_closed) {
  bool timedout = false;
  // wait for a new job or idle timeout expiration
  if (m_job == NULL) {
   timedout = !m_wakeupSync.Wait(m_idleTimeout);
   if (timedout)
   { DEBUG_TEST(5, "JOB/tIdle timeout for Worker " << m_id);
   }
  }
  // terminate this worker if closed explicitly or idle timeout expired
  if (m_closed || (timedout && m_job == NULL)) {

程序处于空闲的状态，好久没有任务可以执行，先停止，回去休息一下，等忙的时候再由Agent调出来

执行Job
   m_closed = true;
   break;
  }
  
  if (m_job) {
   DEBUG_TEST(5, "JOB/tStarting Job " << m_job->GetName()
    << " at Worker thread " << m_id
    );

重点在这里，调用Job的Run函数，执行某个Job具体的指令，以完成其提交的任务。

   m_job->Run();

   {
    PWaitAndSignal lock(m_jobMutex);
    delete m_job;
    m_job = NULL;
   }
   
   m_agent->JobDone(this);
  }
 }

 DEBUG_TEST(5, "JOB/tWorker " << m_id << " closed");
 
 // remove this Worker from the list of workers
 m_agent->Remove(this);
 if (m_job)
 {
  DEBUG_TEST(1, "JOB/tActive Job " << m_job->GetName()
   << " left at closing Worker thread " << m_id
   );
 }
}

bool Worker::Exec(
 /// Job to be executed
 Job* job
 )
{
 // fast check if there is no job being executed
 if( GetThreadId()== 0  )
 {
  DEBUG_TEST(0,"thread id is zeor "<<GetThreadId());
  return false ;
 }
 if (m_job == 0 && !m_closed) {
  PWaitAndSignal lock(m_jobMutex);
  // check again there is no job being executed
  if (m_job == 0 && !m_closed) {
   m_job = job;
   m_wakeupSync.Signal();
   return true;
  }
 }
 return false;
}

void Worker::Destroy()
{
 // do not delete itself when the thread is stopped
 SetNoAutoDelete();
 
 m_jobMutex.Wait();
 if (m_job)
  m_job->Stop();
 m_jobMutex.Signal();

 m_closed = true;
 m_wakeupSync.Signal();
 
 DEBUG_TEST(5, "JOB/tWaiting for Worker thread " << m_id << " termination");
 WaitForTermination();
 /* 自我销毁*/
 delete this;
}

Agent::Agent() : Singleton<Agent>("Agent"), m_active(true)
{
}

Agent::~Agent()
{
 DEBUG_TEST(5, "JOB/tDestroying active Workers for the Agent");

 std::list<Worker*> workers;
#if PTRACING
 int numIdleWorkers = -1;
 int numBusyWorkers = -1;
#endif
 
 {
  // move all workers to the local list
  PWaitAndSignal lock(m_wlistMutex);
  m_active = false;
#if PTRACING
  numIdleWorkers = m_idleWorkers.size();
  numBusyWorkers = m_busyWorkers.size();
#endif
  while (!m_busyWorkers.empty()) {
   workers.push_front(m_busyWorkers.front());
   m_busyWorkers.pop_front();
  }
  while (!m_idleWorkers.empty()) {
   workers.push_front(m_idleWorkers.front());
   m_idleWorkers.pop_front();
  }
 }

#if TEST_
 DEBUG_TEST(5, "JOB/tWorker threads to cleanup: " << (numBusyWorkers+numIdleWorkers)
  << " total - " << numBusyWorkers << " busy, " << numIdleWorkers << " idle"
  );
#endif

 // destroy all workers
 ForEachInContainer(workers, mem_vfun(&Worker::Destroy));
 
 DEBUG_TEST(5, "JOB/tAgent and its Workers destroyed");
}

void Agent::Exec(
 /// the job to be executed
 Job* job
 )
{
 Worker* worker = NULL;
#if PTRACING
 int numIdleWorkers = -1;
 int numBusyWorkers = -1;
#endif
 // pop the first idle worker and move it to the busy list 
 if (job) {
  PWaitAndSignal lock(m_wlistMutex);
  // delete the job if the Agent is being destroyed
  if (!m_active) {
   DEBUG_TEST(5, "JOB/tAgent did not accept Job " << job->GetName());
   delete job;
   job = NULL;
   return;
  }
  /*如果有空闲的worker，从中其一个来执行worker*/
  if (!m_idleWorkers.empty()) {
   worker = m_idleWorkers.front();
   m_idleWorkers.pop_front();
   m_busyWorkers.push_front(worker);
#if PTRACING
   numIdleWorkers = m_idleWorkers.size();
   numBusyWorkers = m_busyWorkers.size();
#endif
  }
 } else
  return;
 
 bool destroyWorker = false;
  
 // if no idle worker has been found, create a new one
 // and put it on the list of busy workers
 /* 如果没有空闲的worker创建一个新的worker来执行job
 并把worker加入忙队列*/
 if (worker == NULL) {
  worker = new Worker(this);

  PWaitAndSignal lock(m_wlistMutex);
  if (m_active)
   m_busyWorkers.push_front(worker);
  else
   destroyWorker = true;
#if PTRACING
  numIdleWorkers = m_idleWorkers.size();
  numBusyWorkers = m_busyWorkers.size();
#endif
 }
 
 // execute the job by the worker
 // if worker is running the the worker id > 0 else == 0
 /*提交job到worker 如果提交失败，一般是worker  线程已经停止，
 就删除job，否则不能删除job*/
 if (!(m_active && worker->Exec(job))) {
  // should not ever happen, but...
  DEBUG_TEST(1,"can't execute the job,delete it "<<job->GetName());
  delete job;
  job = NULL;
  PWaitAndSignal lock(m_wlistMutex);
  m_busyWorkers.remove(worker);
  if (m_active)
   m_idleWorkers.push_front(worker);
  else
   destroyWorker = true;
#if PTRACING
  numIdleWorkers = m_idleWorkers.size();
  numBusyWorkers = m_busyWorkers.size();
#endif
 }

#if PTRACING
 PTRACE_IF(5, m_active, "JOB/tWorker threads: " << (numBusyWorkers+numIdleWorkers)
  << " total - " << numBusyWorkers << " busy, " << numIdleWorkers << " idle"
  );
#endif

 if (destroyWorker) {
  DEBUG_TEST(5, "JOB/tAgent did not accept Job " << job->GetName());
  worker->Destroy();
 }
}

void Agent::Remove(
 Worker* worker
 )
{
#if PTRACING
 int numIdleWorkers;
 int numBusyWorkers;
 {
#endif
  PWaitAndSignal lock(m_wlistMutex);
  // check both lists for the worker
  m_idleWorkers.remove(worker);
  m_busyWorkers.remove(worker);
#if PTRACING
  numIdleWorkers = m_idleWorkers.size();
  numBusyWorkers = m_busyWorkers.size();
 }
 PTRACE_IF(5, m_active, "JOB/tWorker threads: " << (numBusyWorkers+numIdleWorkers)
  << " total - " << numBusyWorkers << " busy, " << numIdleWorkers << " idle"
  );
#endif
}

void Agent::JobDone(
 /// the worker to be marked as idle
 Worker* worker
 )
{
#if PTRACING
 int numIdleWorkers;
 int numBusyWorkers;
 {
#endif
  PWaitAndSignal lock(m_wlistMutex);
  m_busyWorkers.remove(worker);
  if (m_active)
   m_idleWorkers.push_front(worker);
#if PTRACING
  numIdleWorkers = m_idleWorkers.size();
  numBusyWorkers = m_busyWorkers.size();
 }
 PTRACE_IF(5, m_active, "JOB/tWorker threads: " << (numBusyWorkers+numIdleWorkers)
  << " total - " << numBusyWorkers << " busy, " << numIdleWorkers << " idle"
  );
#endif
}

Task::~Task()
{
}

Job::~Job()
{
 DEBUG_TEST(5, "JOB/tJob " << GetName() << " deleted");
}

void Job::Execute()
{

提交任务到代理，由代理调度执行
 Agent::Instance()->Exec(this);
}

void Job::Stop()
{
}

void Job::StopAll()
{

停止整个程序的所有Job
 delete Agent::Instance();
}

void Jobs::Run()
{

执行Jobs
 while (m_current) {
  m_current->Exec();
  m_current = m_current->DoNext();
 }
}

RegularJob::RegularJob() : m_stop(false)
{
}

void RegularJob::OnStart()
{
}

void RegularJob::OnStop()
{
}

void RegularJob::Run()
{
 OnStart();
 
 while (!m_stop)
  Exec();
  
 // lock to allow a member function that is calling Stop
 // return before OnStop is called and the object is deleted
 PWaitAndSignal lock(m_deletionPreventer);
 OnStop();
}

void RegularJob::Stop()
{
 // signal stop flag and wake up job thread, if it is in the waiting state
 m_stop = true;
 m_sync.Signal();
}