Linux高性能服务器编程学习记录——十五、进程池与线程池

这一章实现了一个半同步/半异步的进程池（主进程只负责监听socket，所有连接socket，以及连接socket上的数据操作由子进程负责），和一个半同步/半反应堆的线程池，主线程负责监听socket和所有连接socket上的IO事件，子线程都阻塞在工作队列中，以竞争的方式从队列中取任务，然后处理。

半同步/半异步的进程池实现

processpool.h

#ifndef PROCESSPOOL_H
#define PROCESSPOOL_H

#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 

/* 描述一个子进程的类 */
struct process
{
	pid_t	m_pid;	//子进程pid
	int		m_pipefd[2];	//与父进程通信用的管道 父进程使用0端 子进程使用1端

	process():m_pid(-1){}
};

/* 进程池类 定义为模板是为了代码复用，模板参数是处理逻辑任务的类 */
template<typename T>
class processpool
{
private:
	/* 单例模式 故将构造私有化，使用静态函数create来创建实例*/
	processpool(int listenfd, int process_number = 8);

public:
	/*单例模式，保证程序最多创建一个processpool实例，这是程序正确处理信号的必要条件*/
	static processpool<T>* create(int listenfd, int process_number = 8)
	{
		if (!m_instance)
		{
			m_instance = new processpool<T>(listenfd, process_number);
		}
		return m_instance;
	}
	
	~processpool()
	{
		delete[] m_sub_process;
	}

	/* 运行，根据m_idx判断运行父进程还是子进程 */
	void run();
private:
	/* 统一事件源 */
	void setup_sig_pipe();
	/* 运行父进程 */
	void run_parent();
	/* 运行子进程 */
	void run_child();

private:
	/* 进程池允许的最大子进程数量 */
	static const int MAX_PROCESS_NUMBER = 16;
	/* 每个子进程最多能处理的客户数 */
	static const int USER_PER_PROCESS = 65536;
	/* epoll最多能处理的事件数 */
	static const int MAX_EVENT_NUMBER = 10000;
	/* 进程池中的进程总数 */
	int m_process_number;
	/* 子进程在池中的序号，从0开始 */
	int m_idx;
	/* 每个进程都有一个epoll内核事件表，用m_epollfd标识 */
	int m_epollfd;
	/* 监听socket */
	int m_listenfd;
	/* 是否停止允许（子进程使用）*/
	int m_stop;
	/* 保存所有子进程的描述信息 */
	process* m_sub_process;
	/* 进程池的静态实例 */
	static processpool<T>* m_instance;
};

/* 通常 静态成员的初始化在类外进行 且不需要再用static修饰 */
template<typename T>
processpool<T>* processpool<T>::m_instance = NULL;

/* 用于处理信号的管道，以实现统一事件源 */
static int sig_pipefd[2];

static int setnonblocking(int fd)
{
	int oldopt = fcntl(fd, F_GETFL);
	int newopt = oldopt | O_NONBLOCK;
	fcntl(fd, F_SETFL, newopt);
	return oldopt;
}

static void addfd(int epollfd, int fd)
{
	epoll_event event;
	event.data.fd = fd;
	event.events = EPOLLIN | EPOLLET;
	epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &event);
	setnonblocking(fd);
}

/*从epoll标识的epoll内核事件表中删除fd上的所有注册事件*/
static void removefd(int epollfd, int fd)
{
	epoll_ctl(epollfd, EPOLL_CTL_DEL, fd, 0);
	close(fd);
}

/*收到的信号塞到管道里让主循环去处理，统一事件源*/
static void sig_handler(int sig)
{
	int save_errno = errno;
	int msg = sig;
	send(sig_pipefd[1], (char*)&msg, 1, 0);
	errno = save_errno;
}

/*注册信号处理函数*/
static void addsig(int sig, void(handler)(int), bool restart = true)
{
	struct sigaction sa;
	memset(&sa, '\0', sizeof(sa));
	sa.sa_handler = handler;
	if (restart)
	{
		sa.sa_flags |= SA_RESTART;
	}
	sigfillset(&sa.sa_mask);
	assert(sigaction(sig, &sa, NULL) != -1);
}

/*进程池构造函数，listenfd必须在创建进程池之前被创建，否则子进程无法引用*/
template<typename T>
processpool<T>::processpool(int listenfd, int process_number)
	:m_listenfd(listenfd), m_process_number(process_number), m_idx(-1), m_stop(false)
{
	assert(process_number > 0 && process_number <= MAX_PROCESS_NUMBER);
	m_sub_process = new process[process_number];
	assert(m_sub_process);

	/*创建子进程，并建立它们与父进程之间的管道*/
	for (int i = 0; i < process_number; ++i)
	{
		int ret = socketpair(PF_UNIX, SOCK_STREAM, 0, m_sub_process[i].m_pipefd);
		assert(ret == 0);
		/*父进程中fork返回子进程pid 子进程中返回0*/
		m_sub_process[i].m_pid = fork();
		assert(m_sub_process[i].m_pid >= 0);
		if (m_sub_process[i].m_pid > 0)
		{
			/*父进程中关闭1端*/
			close(m_sub_process[i].m_pipefd[1]);
			continue;
		}
		else
		{
			/*子进程中关闭0端*/
			close(m_sub_process[i].m_pipefd[0]);
			m_idx = i;
			/*创建子进程的工作是父进程完成的，所以子进程这里break出循环*/
			break;
		}
	}
}

/*统一事件源*/
template<typename T>
void processpool<T>::setup_sig_pipe()
{
	/*创建epoll事件表和信号管道*/
	m_epollfd = epoll_create(5);
	assert(m_epollfd != -1);
	int ret = socketpair(PF_UNIX, SOCK_STREAM, 0, sig_pipefd);
	assert(ret != -1);

	setnonblocking(sig_pipefd[1]);
	addfd(m_epollfd, sig_pipefd[0]);

	/*设置信号处理函数*/
	addsig(SIGCHLD, sig_handler);	/*子进程终止给发进程发送的信号*/
	addsig(SIGTERM, sig_handler);	/*软件终止信号 */
	addsig(SIGINT, sig_handler);	/*中断进程信号*/	
	addsig(SIGPIPE, sig_handler);	/*向一个没有读进程的管道写数据 子进程已关闭，父进程还往与子进程关联的管道写数据会触发*/
}

template<typename T>
void processpool<T>::run()
{
	if (m_idx != -1)
	{
		run_child();
		return;
	}
	run_parent();
}

template<typename T>
void processpool<T>::run_child()
{
	setup_sig_pipe();
	/*找到与父进程通信的管道*/
	int pipefd = m_sub_process[m_idx].m_pipefd[1];
	/*父进程通过这个管道通知子进程有新的连接到来*/
	addfd(m_epollfd, pipefd);
	epoll_event events[MAX_EVENT_NUMBER];
	T* users = new T[USER_PER_PROCESS];
	assert(users);
	int number = 0;
	int ret = -1;

	while (!m_stop)
	{
		number = epoll_wait(m_epollfd, events, MAX_EVENT_NUMBER, -1);
		if (number < 0 && errno != EINTR)
		{
			printf("epoll failure\n");
			break;
		}

		for (int i = 0; i < number; ++i)
		{
			int sockfd = events[i].data.fd;
			/*父进程通知有新连接*/
			if (sockfd == pipefd && (events[i].events & EPOLLIN))
			{
				int client = 0;
				/*从管道中读取数据，若读取成功则说明有新连接到来*/
				ret = recv(pipefd, (char*)&client, sizeof(client), 0);
				if ((ret < 0 && errno != EAGAIN) || ret == 0)
				{
					printf("recv new conn failure\n");
					continue;
				}
				else
				{
					struct sockaddr_in client_address;
					socklen_t client_addrlength = sizeof(client_address);
					int connfd = accept(m_listenfd, (struct sockaddr*)&client_address, &client_addrlength);
					if (connfd < 0)
					{
						printf("accept failure, errno is:%d\n", errno);
						continue;
					}
					addfd(m_epollfd, connfd);
					/*模板类必须实现init函数以初始化一个客户链接*/
					users[connfd].init(m_epollfd, connfd, client_address);
				}
			}
			/*子进程收到信号*/
			else if (sockfd == sig_pipefd[0] && (events[i].events & EPOLLIN))
			{
				int sig;
				char signals[1024];
				ret = recv(sig_pipefd[0], signals, sizeof(signals), 0);
				if (ret <= 0)
				{
					continue;
				}
				else
				{
					for (int i = 0; i < ret; ++i)
					{
						switch (signals[i])
						{
						case SIGCHLD:
						{
							pid_t pid;
							int stat;
							/*回收子进程*/
							while ((pid = waitpid(-1, &stat, WNOHANG)) > 0)
							{
								continue;
							}
							break;
						}
						case SIGTERM:
						case SIGINT:
						{
							m_stop = true;
							break;
						}
						default:
							break;
						}
					}
				}
			}
			/*客户请求 模板类必须实现process函数以处理客户请求*/
			else if (events[i].events & EPOLLIN)
			{
				printf("recv client request\n");
				users[sockfd].process();
			}
			else
			{
				continue;
			}
		}
	}
}

template<typename T>
void processpool<T>::run_parent()
{
	setup_sig_pipe();
	/*父进程监听listenfd*/
	addfd(m_epollfd, m_listenfd);

	epoll_event events[MAX_EVENT_NUMBER];
	int sub_process_counter = 0;
	int new_conn = 1;
	int number = 0;
	int ret = -1;

	while (!m_stop)
	{
		number = epoll_wait(m_epollfd, events, MAX_EVENT_NUMBER, -1);
		if (number < 0 && errno != EINTR)
		{
			printf("epoll failure\n");
			break;
		}
		for (int i = 0; i < number; ++i)
		{
			int sockfd = events[i].data.fd;
			if (sockfd == m_listenfd)
			{
				/*采用RoundRobin方式将新连接分配给一个子进程*/
				int i = sub_process_counter;
				do 
				{
					if (m_sub_process[i].m_pid != -1)
					{
						break;
					}
					i = (i + 1) % sub_process_counter;
				} while (i != sub_process_counter);
				/*没有可用的子进程了*/
				if (m_sub_process[i].m_pid == -1)
				{
					m_stop = 1;
					break;
				}
				/*这里好像有问题??*/
				sub_process_counter = (i + 1) % m_process_number;
				/*通过管道通知子进程 有新连接*/
				send(m_sub_process[i].m_pipefd[0], (char*)&new_conn, sizeof(new_conn), 0);
				printf("send request to child %d\n", i);
			}
			/*收到信号*/
			else if (sockfd == sig_pipefd[0] && events[i].events & EPOLLIN)
			{
				int sig;
				char signals[1024];
				ret = recv(sockfd, signals, sizeof(signals), 0);
				if (ret <= 0)
				{
					continue;
				}
				else
				{
					for (int i = 0; i < ret; ++i)
					{
						switch (signals[i])
						{
						case SIGCHLD:
						{
							pid_t pid;
							int stat;
							while ((pid = waitpid(-1, &stat, WNOHANG)) > 0)
							{
								/*子进程pid退出，关闭通信管道，设置相应的m_pid为-1*/
								for (int i = 0; i < m_process_number; ++i)
								{
									if (m_sub_process[i].m_pid == pid)
									{
										printf("child %d quit\n", pid);
										close(m_sub_process[i].m_pipefd[0]);
										m_sub_process[i].m_pid = -1;
										/*这里不break?*/
									}
								}
							}
							/*如果所有子进程都退出了，父进程也退出*/
							m_stop = true;
							for (int i = 0; i < m_process_number; ++i)
							{
								if (m_sub_process[i].m_pid != -1)
								{
									m_stop = false;
									break;
								}
							}
							break;
						}
						case SIGTERM:
						case SIGINT:
						{
							/*父进程收到终止信号，先杀死所有子进程，等他们结束后，再结束自己，
							更好的杀死子进程的方式是通过管道发送特殊数据让子进程自己退出*/
							printf("kill all the child now\n");
							for (int i = 0; i < m_process_number; ++i)
							{
								int pid = m_sub_process[i].m_pid;
								if (pid != -1)
								{
									kill(pid, SIGTERM);
								}
							}
							break;
						}
						default:
							break;
						}
					}
				}
			}
			else
			{
				continue;
			}
		}
	}

	//close(m_listenfd) 应该由创建者关闭
	close(m_epollfd);
}

#endif

用进程池实现的简单CGI服务器
cgi_simple.cpp

#include "processpool.h"

/*用与处理客户CGI请求，它可以作为进程池类的模板参数*/
class cgi_conn
{
public:
	cgi_conn(){}
	~cgi_conn(){}
	/*初始化客户端连接，情况缓冲区*/
	void init(int epollfd, int sockfd, const sockaddr_in& client_addr)
	{
		m_epolled = epollfd;
		m_sockfd = sockfd;
		m_address = client_addr;
		memset(m_buf, '\0', sizeof(m_buf));
		m_read_idx = 0;
	}

	/*处理客户请求*/
	void process()
	{
		int idx = 0;
		int ret = -1;
		while (1)
		{
			idx = m_read_idx;
			ret = recv(m_sockfd, m_buf + idx, BUFFER_SIZE - idx - 1, 0);
			/*如果是读操作发送错误，则关闭连接，如果是暂时无数据可读，则退出循环*/
			if (ret < 0)
			{
				if (errno != EAGAIN)
				{
					/*读错误*/
					removefd(m_epolled, m_sockfd);
				}
				break;
			}
			/*若对方关闭连接，则服务器也关闭*/
			else if (ret == 0)
			{
				removefd(m_epolled, m_sockfd);
				break;
			}
			else
			{
				m_read_idx += ret;
				printf("user content is: %s\n", m_buf);
				/*如果遇到字符"\r\n"则开始处理请求*/
				for (; idx < m_read_idx; ++idx)
				{
					if (idx >= 1 && (m_buf[idx - 1] == '\r') && (m_buf[idx] == '\n'))
					{
						break;
					}
				}
				/*如果没有遇到“\r\n”则需要读取更多客户数据*/
				if (idx == m_read_idx)
				{
					continue;
				}
				m_buf[idx - 1] = '\0';
				char* file_name = m_buf;
				/*判断客户要执行的CGI程序是否存在*/
				if (access(file_name, F_OK) == -1)
				{
					removefd(m_epolled, m_sockfd);
					break;
				}
				/*创建子进程来执行CGI程序*/
				ret = fork();
				if (ret == -1)
				{
					removefd(m_epolled, m_sockfd);
					break;
				}
				else if (ret > 0)
				{
					/*父进程中关闭连接*/
					removefd(m_epolled, m_sockfd);
					break;
				}
				else
				{
					/*子进程将标准输出定向到m_sockfd,并执行CGI程序*/
					close(STDOUT_FILENO);
					dup(m_sockfd);
					execl(m_buf, m_buf, 0);
					exit(0);
				}
			}
		}
	}

private:
	/*读缓冲区大小*/
	static const int BUFFER_SIZE = 1024;
	static int m_epolled;
	int m_sockfd;
	sockaddr_in m_address;
	char m_buf[BUFFER_SIZE];
	/*标记读缓冲区中已经读入的客户数据的最后一个字节的下一个位置*/
	int m_read_idx;
};

int cgi_conn::m_epolled = -1;

int main(int argc, char* argv[])
{
	if (argc <= 2)
	{
		printf("usage: %s ip port\n", argv[0]);
		return 1;
	}
	const char* ip = argv[1];
	int port = atoi(argv[2]);

	struct sockaddr_in address;
	bzero(&address, sizeof(address));
	address.sin_family = AF_INET;
	address.sin_port = htons(port);
	inet_pton(AF_INET, ip, &address.sin_addr);

	int listenfd = socket(PF_INET, SOCK_STREAM, 0);
	assert(listenfd >= 0);
	int ret = bind(listenfd, (struct sockaddr*)&address, sizeof(address));
	assert(ret != -1);
	ret = listen(listenfd, 5);
	assert(ret != -1);

	processpool<cgi_conn>* pool = processpool<cgi_conn>::create(listenfd, 1);
	if (pool)
	{
		pool->run();
		delete pool;
	}
	close(listenfd);
	return 0;
}

半同步/半反应堆线程池实现

threadpool.h

#ifndef THREADPOOL_H
#define THREADPOOL_H

#include 
#include 
#include 
#include 

#include "locker.h"

template<typename T>
class threadpool
{
public:
	/*thread_number指定线程池中线程的数量，
	max_requests是请求队列中最多允许的、等待处理的请求的数量*/
	threadpool(int thread_number = 8, int max_requests = 10000);
	~threadpool();

	/*往请求队列中添加任务*/
	bool append(T* request);

private:
	/*工作线程运行的函数，它不断从工作队列中取出任务并执行
	worker是静态成员函数，它里面需要调用动态成员函数run，有两种实现方式
	1：通过类得静态对象来调用，如单例模式中，静态函数可以通过类得全局唯一实例来访问动态成员函数
	2：将类得对象作为参数传递给该静态成员函数，然后在静态函数中引用这个对象来调用其动态成员函数
	这里用的方式2*/
	static void* worker(void* arg);
	void run();

private:
	/*线程池中的线程数*/
	int m_thread_number;
	/*请求队列中允许的最大请求数*/
	int m_max_requests;
	/*描述线程池的数组*/
	pthread_t* m_threads;
	/*请求队列*/
	std::deque<T*> m_workqueue;
	/*保护请求队列的互斥锁*/
	locker m_queuelocker;
	/*是否有任务需要处理*/
	sem m_queuestat;
	/*是否结束线程*/
	bool m_stop;
};

template<typename T>
threadpool<T>::threadpool(int thread_number, int max_requests):
	m_thread_number(thread_number), m_max_requests(max_requests), m_stop(false), m_threads(NULL)
{
	if (thread_number <= 0 || max_requests <= 0)
	{
		throw std::exception();
	}

	m_threads = new pthread_t[thread_number];
	if (!m_threads)
	{
		throw std::exception();
	}

	/*创建线程，并设置为脱离线程 
	脱离线程指脱离与同个进程下的其他线程的同步，退出时自行释放资源*/
	for (int i = 0; i < thread_number; ++i)
	{
		printf("create %d thread\n", i);
		if (pthread_create(m_threads + i, NULL, worker, this) != 0)
		{
			delete[] m_threads;
			throw std::exception();
		}

		if (pthread_detach(m_threads[i]))
		{
			delete[] m_threads;
			throw std::exception();
		}
	}
}

template<typename T>
threadpool<T>::~threadpool()
{
	delete[] m_threads;
	m_stop = true;
}

template<typename T>
bool threadpool<T>::append(T* request)
{
	/*操作工作队列一定要加索，因为它被所有线程共享*/
	m_queuelocker.lock();
	if (m_workqueue.size() >= m_max_requests)
	{
		m_queuelocker.unlock();
		return false;
	}

	m_workqueue.push_back(request);
	m_queuelocker.unlock();
	m_queuestat.post();
	return true;
}

template<typename T>
void* threadpool<T>::worker(void* arg)
{
	threadpool* pool = (threadpool*)arg;
	pool->run();
	return pool;
}

template<typename T>
void threadpool<T>::run()
{
	while (!m_stop)
	{
		/*竞争任务队列的使用权*/
		m_queuestat.wait();
		/*成功竞得，先锁队列，再取数据*/
		m_queuelocker.lock();
		if (m_workqueue.empty())
		{
			m_queuelocker.unlock();
			continue;
		}
		T* request = m_workqueue.front();
		m_workqueue.pop_front();
		m_queuelocker.unlock();
		if (!request)
		{
			continue;
		}
		request->process();
	}
}

#endif

locker.h

#ifndef LOCKER_H
#define LOCKER_H

#include 
#include 
#include 

// 封装信号量的类
class sem
{
public:
	sem()
	{
		//初始化信号量
		if (sem_init(&m_sem, 0, 0) != 0)
		{
			//构造函数没有返回值，通过抛出异常来报告异常
			throw std::exception();
		}
	}
	~sem()
	{
		//销毁信号量
		sem_destroy(&m_sem);
	}
	//等待信号量
	bool wait()
	{
		return sem_wait(&m_sem) == 0;
	}
	//增加信号量
	bool post()
	{
		return sem_post(&m_sem) == 0;
	}

private:
	sem_t m_sem;
};

// 封装互斥锁的类
class locker
{
public:
	locker()
	{
		if (pthread_mutex_init(&m_mutex, NULL) != 0)
		{
			throw std::exception();
		}
	}
	~locker()
	{
		pthread_mutex_destroy(&m_mutex);
	}
	//获取互斥锁
	bool lock()
	{
		return pthread_mutex_lock(&m_mutex) == 0;
	}
	//释放互斥锁
	bool unlock()
	{
		return pthread_mutex_unlock(&m_mutex) == 0;
	}
private:
	pthread_mutex_t m_mutex;
};

// 封装条件变量的类
class cond
{
public:
	cond()
	{
		if (pthread_mutex_init(&m_mutex, NULL) != 0)
		{
			throw std::exception();
		}
		if (pthread_cond_init(&m_cond, NULL) != 0)
		{
			//释放已经申请的资源
			pthread_mutex_destroy(&m_mutex);
			throw std::exception();
		}
	}
	~cond()
	{
		pthread_mutex_destroy(&m_mutex);
		pthread_cond_destroy(&m_cond);
	}
	//等待条件变量
	bool wait()
	{
		int ret = 0;
		pthread_mutex_lock(&m_mutex);
		ret = pthread_cond_wait(&m_cond, &m_mutex);
		pthread_mutex_unlock(&m_mutex);
		return ret == 0;
	}
	// 唤醒等待条件变量的线程
	bool signal()
	{
		return pthread_cond_signal(&m_cond);
	}

private:
	pthread_mutex_t m_mutex;
	pthread_cond_t m_cond;
};

#endif

用线程池实现的简单web服务器
http_conn.h

#ifndef HTTPCONNECTION_H
#define HTTPCONNECTION_H

#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include "locker.h"

class http_conn
{
public:
	/*文件最大长度*/
	static const int FILENAME_LEN = 200;
	/*读缓冲区大小*/
	static const int READ_BUFFER_SIZE = 2048;
	/*写缓冲区大小*/
	static const int WRITE_BUFFER_SIZE = 1024;
	/*HTTP请求方法，仅支持GET*/
	enum METHOD
	{
		GET = 0,
		POST,
		HEAD,
		PUT,
		DELETE,
		TRACE,
		OPTION,
		CONNECT,
		PATCH
	};
	/*解析客户请求时，主状态机所处状态*/
	enum CHECK_STATE
	{
		CHECK_STATE_REQUESTLINE = 0,		//当前正在分析请求行
		CHECK_STATE_HEADER,					//当前正在分析请求头部
		CHECK_STATE_CONTENT
	};
	/*服务器处理HTTP请求的可能结果*/
	enum HTTP_CODE
	{
		NO_REQUEST,					//数据不完整，需要继续读取数据
		GET_REQUEST,					//获得了一个完整的客户请求
		BAD_REQUEST,					//请求中有语法错误
		NO_RESOURCE,				//目标文件不存在
		FORBIDDEN_REQUEST,			//无访问权限
		FILE_REQUEST,
		INTERNAL_ERROR,				//服务器内部错误
		CLOSED_CONNECTION,			//连接已断开
	};
	/*行的读取状态*/
	enum LINE_STATUS
	{
		LINE_OK = 0,		//读取到一个完整的行
		LINE_BAD,			//行出错
		LINE_OPEN			//行数据不完整
	};

public:
	http_conn(){}
	~http_conn(){}

	/*初始化新接收的连接*/
	void init(int sockfd, const sockaddr_in& addr);
	/*关闭连接*/
	void close_conn(bool real_close = true);
	/*处理客户请求*/
	void process();
	/*非阻塞读*/
	bool read();
	/*非阻塞写*/
	bool write();

private:
	/*初始化连接*/
	void init();
	/*解析http请求*/
	HTTP_CODE process_read();
	/*填充http应答*/
	bool process_write(HTTP_CODE ret);

	/*下面这一组函数被process_read调用以分析http请求*/
	HTTP_CODE parse_request_line(char* text);
	HTTP_CODE parse_headers(char* text);
	HTTP_CODE parse_content(char* text);
	HTTP_CODE do_request();
	char* get_line() { return m_read_buf + m_start_line; }
	LINE_STATUS parse_line();

	/*下面这一组函数被process_write调用以填充http应答*/
	void unmap();
	bool add_response(const char* format, ...);
	bool add_content(const char* content);
	bool add_status_line(int status, const char* title);
	bool add_headers(int content_length);
	bool add_content_length(int content_length);
	bool add_linger();
	bool add_blank_line();

public:
	/*所有socket上的事件都被注册到同一个epoll内核事件表中，所以将epoll文件描述符设置为静态的*/
	static int m_epollfd;
	/*统计用户数量*/
	static int m_user_count;

private:
	/*该http连接的socket和对方的socket地址*/
	int m_sockfd;
	sockaddr_in m_address;
	/*读缓冲区*/
	char m_read_buf[READ_BUFFER_SIZE];
	/*标识读缓冲中已经读入的客户数据的最后一个字节的下一个位置*/
	int m_read_idx;
	/*当前正在分析的字符在读缓冲区中的位置*/
	int m_checked_idx;
	/*当前正在解析的行的起始位置*/
	int m_start_line;
	/*写缓冲区*/
	char m_write_buf[WRITE_BUFFER_SIZE];
	/*写缓冲区中待发送的字节数*/
	int m_write_idx;

	/*主状态机当前所处状态*/
	CHECK_STATE m_check_state;
	/*请求方法*/
	METHOD m_method;

	/*客户请求的目标文件的完整路径，其内容等于doc_root + m_url, doc_root是网站根目录*/
	char m_real_file[FILENAME_LEN];
	/*客户请求的目标文件的文件名*/
	char* m_url;
	/*HTTP协议版本号，仅支持HTTP/1.1*/
	char* m_version;
	/*主机名*/
	char* m_host;
	/*http请求的消息体的长度*/
	int m_content_length;
	/*http请求是否要求保持连接*/
	bool m_linger;

	/*客户请求的文件被mmap到内存中的起始位置*/
	char* m_file_address;
	/*目标文件状态，通过它我们可以判断文件是否存在、是否为目录、是否可读，并获取文件大小等信息*/
	struct stat m_file_stat;
	/*采用writev来执行写操作，所以定义下面两个成员，
	其中m_iv是分散的内存块数组，可实现集中写
	m_iv_count表示被写内存块的数量*/
	struct iovec m_iv[2];
	int m_iv_count;
};

#endif

http_conn.cpp

#include "http_conn.h"

/*定义http响应的一些状态信息*/
const char* ok_200_title = "OK";
const char* error_400_title = "Bad Request";
const char* error_400_form = "Your request has bad syntax or is inherently impossible to satisfy.\n";
const char* error_403_title = "Forbidden";
const char* error_403_form = "You do not have permission to get file from this server.\n";
const char* error_404_title = "Not Found";
const char* error_404_form = "The requested file was not found on this server.\n";
const char* error_500_title = "Internal Error";
const char* error_500_form = "There was an unusual problem serving the requested file.\n";

/*网站根目录*/
const char* doc_root = "/var/www/html";

int setnonblocking(int fd)
{
	int oldopt = fcntl(fd, F_GETFL);
	int newopt = oldopt | O_NONBLOCK;
	fcntl(fd, F_SETFL, newopt);
	return oldopt;
}

void addfd(int epollfd, int fd, bool one_shot = false)
{
	epoll_event event;
	event.data.fd = fd;
	event.events = EPOLLIN | EPOLLET | EPOLLRDHUP;		// EPOLLRDHUP事件是TCP连接被对方关闭，或者对方关闭了写操作
	if (one_shot)
	{
		event.events |= EPOLLONESHOT;
	}
	epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &event);
	setnonblocking(fd);
}

void removefd(int epollfd, int fd)
{
	epoll_ctl(epollfd, EPOLL_CTL_DEL, fd, 0);
	close(fd);
}

void modfd(int epollfd, int fd, int ev)
{
	epoll_event event;
	event.data.fd = fd;
	event.events = ev | EPOLLIN | EPOLLET | EPOLLRDHUP;
	epoll_ctl(epollfd, EPOLL_CTL_MOD, fd, &event);
}

//类外初始化静态成员
int http_conn::m_user_count = 0;
int http_conn::m_epollfd = -1;

void http_conn::close_conn(bool real_close)
{
	if (real_close && m_sockfd != -1)
	{
		removefd(m_epollfd, m_sockfd);
		m_sockfd = -1;
		m_user_count--;
	}
}

/*public成员 接收到新连接时调用*/
void http_conn::init(int sockfd, const sockaddr_in& addr)
{
	m_sockfd = sockfd;
	m_address = addr;
	/*下面两行是为了避免TIME_WAIT状态，仅用于调试，实际使用时应该去掉*/
	int reuse = 1;
	setsockopt(m_sockfd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse));
	//添加到epoll中进行监听
	addfd(m_epollfd, m_sockfd);
	m_user_count++;

	init();
}

/*private成员 由public的init调用*/
void http_conn::init()
{
	m_check_state = CHECK_STATE_REQUESTLINE;
	m_linger = false;
	m_method = GET;
	m_url = 0;
	m_version = 0;
	m_content_length = 0;
	m_host = 0;
	m_start_line = 0;
	m_checked_idx = 0;
	m_read_idx = 0;
	m_write_idx = 0;

	memset(m_read_buf, '\0', READ_BUFFER_SIZE);
	memset(m_write_buf, '\0', WRITE_BUFFER_SIZE);
	memset(m_real_file, '\0', FILENAME_LEN);
}

/*从状态机 
用于解析出一行内容 一个完整的行结尾必是一个回车符+一个换行符，即"\r\n"*/
http_conn::LINE_STATUS http_conn::parse_line()
{
	char temp;
	for (; m_checked_idx < m_read_idx; ++m_checked_idx)
	{
		temp = m_read_buf[m_checked_idx];
		if (temp == '\r')
		{
			if ((m_checked_idx + 1) == m_read_idx)
			{
				return LINE_OPEN;
			}
			if (m_read_buf[m_checked_idx + 1] == '\n')
			{
				m_read_buf[m_checked_idx++] = '\0';
				m_read_buf[m_checked_idx++] = '\0';
				return LINE_OK;
			}
			return LINE_BAD;
		}
		
		if (temp == '\n')
		{
			/*这种情况对应第一个if中LINE_OPEN的情况*/
			if (m_checked_idx > 1 && m_read_buf[m_checked_idx - 1] == '\r')
			{
				return LINE_OK;
			}
			return LINE_BAD;
		}
	}

	return LINE_OPEN;
}

/*循环读取数据，直到无数据可读或对方关闭连接*/
bool http_conn::read()
{
	if (m_read_idx >= READ_BUFFER_SIZE)
	{
		return false;
	}

	int bytes_read = 0;
	while (1)
	{
		bytes_read = recv(m_sockfd, m_read_buf + m_read_idx, READ_BUFFER_SIZE - m_read_idx, 0);
		if (bytes_read == -1)
		{
			if (errno == EAGAIN || errno == EWOULDBLOCK)
			{
				// 无数据可读了
				break;
			}
			// recv出错了
			return false;
		}
		if (bytes_read == 0)
		{
			// 对方关闭连接
			return false;
		}
		m_read_idx += bytes_read;
	}
	return true;
}

/*解析http请求行 获得请求方法、目标URL,以及http版本号
一个正常的http请求行示例："GET http://www.xxx.xx/xx HTTP/1.1"*/
http_conn::HTTP_CODE http_conn::parse_request_line(char* text)
{
	/*c库函数 strpbrk 检索temp中第一个出现" \t"中字符的字符，
	返回指向检索到的哪个字符的指针本例中是检索temp中第一次出现空格或者'\t'的位置  
	如temp为"abc def",返回的url将指向c后面的空格*/
	m_url = strpbrk(text, " \t");
	if (!m_url)
	{
		return BAD_REQUEST;
	}
	*m_url++ = '\0';
	char* method = text;
	/*strcasecmp忽略大小写比较字符串*/
	if (strcasecmp(method, "GET") != 0)
	{
		return BAD_REQUEST;
	}
	m_method = GET;
	/*C 库函数 size_t strspn(const char *str1, const char *str2) 检索字符串 str1 中第一个不在字符串 str2 中出现的字符下标。
	这一步过滤多余的空格和'\t'，确保url指向'h'*/
	m_url += strspn(m_url, " \t");
	m_version = strpbrk(m_url, " \t");
	if (!m_version)
	{
		return BAD_REQUEST;
	}

	*m_version++ = '\0';
	m_version += strspn(m_version, " \t");
	if (strcasecmp(m_version, "HTTP/1.1") != 0)
	{
		return BAD_REQUEST;
	}

	if (strncasecmp(m_url, "http://", 7) == 0)
	{
		m_url += 7;
		/*C 库函数 char* strchr(const char* str, int c) 在参数 str 所指向的字符串中搜索第一次出现字符 c（一个无符号字符）的位置*/
		m_url = strchr(m_url, '/');
	}
	
	if (!m_url || m_url[0] != '/')
	{
		return BAD_REQUEST;
	}
	/*状态转换*/
	m_check_state = CHECK_STATE_HEADER;
	return NO_REQUEST;
}

/*解析http请求的一个头部信息*/
http_conn::HTTP_CODE http_conn::parse_headers(char* text)
{
	/*遇到空行，表示头部字段解析完毕*/
	if (text[0] == '\0')
	{
		/*如果请求有消息体，则还需要读取m_content_length字节的消息体，状态机转移到CHECK_STATE_CONTENT状态*/
		if (m_content_length != 0)
		{
			m_check_state = CHECK_STATE_CONTENT;
			return NO_REQUEST;
		}
		/*否则说明已经得到了一个完整的http请求*/
		return GET_REQUEST;
	}

	/*处理connection头部字段*/
	if (strncasecmp(text, "Connection:", 11) == 0)
	{
		text += 11;
		text += strspn(text, " \t");
		if (strcasecmp(text, "keep-alive") == 0)
		{
			/*长连接*/
			m_linger = true;
		}
	}
	else if (strncasecmp(text, "Content-Length:", 15) == 0)
	{
		text += 15;
		text += strspn(text, " \t");
		m_content_length = atol(text);
	}
	else if (strncasecmp(text, "Host:", 5) == 0)
	{
		text += 5;
		text += strspn(text, " \t");
		m_host = text;
	}
	else
	{
		printf("oop! unknow header %s\n", text);
	}
	return NO_REQUEST;
}

/*我们没有真正解析http请求的消息体，只是判断它是否被完整读入了*/
http_conn::HTTP_CODE http_conn::parse_content(char* text)
{
	if (m_read_idx >= (m_content_length + m_checked_idx))
	{
		text[m_content_length] = '\0';
		return GET_REQUEST;
	}
	return NO_REQUEST;
}

/*主状态机
从buffer中读出所有的完整的行*/
http_conn::HTTP_CODE http_conn::process_read()
{
	LINE_STATUS line_status = LINE_OK;
	HTTP_CODE ret = NO_REQUEST;
	char* text = 0;

	while ((m_check_state == CHECK_STATE_CONTENT && line_status == LINE_OK)
		|| (line_status = parse_line()) == LINE_OK)
	{
		text = get_line();
		m_start_line = m_checked_idx;
		printf("got 1 http line: %s\n", text);

		switch (m_check_state)
		{
		case http_conn::CHECK_STATE_REQUESTLINE:
		{
			ret = parse_request_line(text);
			if (ret == BAD_REQUEST)
			{
				return BAD_REQUEST;
			}
			break;
		}
		break;
		case http_conn::CHECK_STATE_HEADER:
		{
			ret = parse_headers(text);
			if (ret == BAD_REQUEST)
			{
				return BAD_REQUEST;
			}
			if (ret == GET_REQUEST)
			{
				return do_request();
			}
			break;
		}
		break;
		case http_conn::CHECK_STATE_CONTENT:
		{
			ret = parse_content(text);
			if (ret == GET_REQUEST)
			{
				return do_request();
			}
			line_status = LINE_OPEN;
			break;
		}
		break;
		default:
			return INTERNAL_ERROR;
		}
	}
	return NO_REQUEST;
}

/*当得到一个完整、正确的http请求时就分析目标文件的属性。如果目标文件存在、对所有用户可读，且不是目录，
则使用mmap将其映射到内存地址m_file_address处，并告诉调用者获取文件成功*/
http_conn::HTTP_CODE http_conn::do_request()
{
	strcpy(m_real_file, doc_root);
	int len = strlen(doc_root);
	strncpy(m_real_file + len, m_url, FILENAME_LEN - len - 1);
	if (stat(m_real_file, &m_file_stat) < 0)
	{
		printf("real file:%s\n", m_real_file);
		return NO_RESOURCE;
	}
	/*S_IRUSR：用户读权限
	S_IWUSR：用户写权限
	S_IRGRP：用户组读权限
	S_IWGRP：用户组写权限
	S_IROTH：其他组读权限
	S_IWOTH：其他组写权限*/
	if (!(m_file_stat.st_mode & S_IROTH))
	{
		return FORBIDDEN_REQUEST;
	}

	if (S_ISDIR(m_file_stat.st_mode))
	{
		/*请求的是个目录*/
		return BAD_REQUEST;
	}

	int fd = open(m_real_file, O_RDONLY);
	/*mmap看这里：https://blog.csdn.net/qq_20363225/article/details/121653263?spm=1001.2014.3001.5501*/
	m_file_address = (char*)mmap(0, m_file_stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
	close(fd);
	return FILE_REQUEST;
}

/*对内存映射区执行munmap*/
void http_conn::unmap()
{
	if (m_file_address)
	{
		munmap(m_file_address, m_file_stat.st_size);
		m_file_address = 0;
	}
}

/*写http响应*/
bool http_conn::write()
{
	int temp = 0;
	int bytes_have_send = 0;
	/*需要发送的字节长度*/
	int bytes_to_send = m_write_idx;
	if (bytes_to_send == 0)
	{
		modfd(m_epollfd, m_sockfd, EPOLLIN);
		init();
		return true;
	}

	while (1)
	{
		temp = writev(m_sockfd, m_iv, m_iv_count);
		if (temp <= -1)
		{
			/*如果tcp写缓冲没有空间，则等待下一轮epollout事件，虽然在此期间，
			服务器无法立即接收到同一客户的下一个请求（因为EPOLLONESHOT）。但这可以保证连接的完整性*/
			if (errno == EAGAIN)
			{
				modfd(m_epollfd, m_sockfd, EPOLLOUT);
				return true;
			}
			/*出错*/
			unmap();
			return false;
		}

		bytes_to_send -= temp;
		bytes_have_send += temp;
		/*为什么这里就发完了？*/
		if (bytes_to_send <= bytes_have_send)
		{
			/*发送http响应成功，根据http请求中的connection字段决定是否立即关闭连接*/
			unmap();
			if (m_linger)
			{
				init();
				modfd(m_epollfd, m_sockfd, EPOLLIN);
				return true;
			}
			else
			{
				modfd(m_epollfd, m_sockfd, EPOLLIN);
				return false;
			}
		}
	}
}

/*往写缓冲区写入待发送的数据*/
bool http_conn::add_response(const char* format, ...)
{
	if (m_write_idx >= WRITE_BUFFER_SIZE)
	{
		return false;
	}
	/*VA_LIST 是在C语言中解决变参问题的一组宏，所在头文件：#include ，用于获取不确定个数的参数。*/
	va_list arg_list;
	/*VA_START宏，获取可变参数列表的第一个参数的地址（ap是类型为va_list的指针，v是可变参数最左边的参数）
	#define va_start(ap, v) (ap = (va_list)&v + _INTSIZEOF(v))*/
	va_start(arg_list, format);
	/*vsnprintf 将可变参数格式化输出到字符数组*/
	int len = vsnprintf(m_write_buf + m_write_idx, WRITE_BUFFER_SIZE - 1 - m_write_idx, format, arg_list);
	if (len >= (WRITE_BUFFER_SIZE - 1 - m_write_idx))
	{
		return false;
	}
	m_write_idx += len;
	/*va_end清空va_list可变参数列表*/
	va_end(arg_list);
	return true;
}

/*请求行*/
bool http_conn::add_status_line(int status, const char* title)
{
	return add_response("%s %d %s\r\n", "HTTP/1.1", status, title);
}

/*头部信息*/
bool http_conn::add_headers(int content_length)
{
	add_content_length(content_length);
	add_linger();
	add_blank_line();
	return true;
}

/*长度*/
bool http_conn::add_content_length(int content_len)
{
	return add_response("Content-Length: %d\r\n", content_len);
}

/*Connection信息 长连接或短连接*/
bool http_conn::add_linger()
{
	return add_response("Connection: %s\r\n", m_linger ? "keep-alive" : "close");
}

/*头部信息结束后的空行*/
bool http_conn::add_blank_line()
{
	return add_response("%s", "\r\n");
}

/*内容*/
bool http_conn::add_content(const char* content)
{
	return add_response("%s", content);
}

/*根据服务器处理http请求的结果，决定返回给客户端的内容*/
bool http_conn::process_write(HTTP_CODE ret)
{
	switch (ret)
	{
	case http_conn::BAD_REQUEST:
	{
		add_status_line(400, error_400_title);
		add_headers(strlen(error_400_form));
		if (!add_content(error_400_form))
		{
			return false;
		}
	}
	break;
	case http_conn::NO_RESOURCE:
	{
		add_status_line(404, error_404_title);
		add_headers(strlen(error_404_form));
		if (!add_content(error_404_form))
		{
			return false;
		}
	}
	break;
	case http_conn::FORBIDDEN_REQUEST:
	{
		add_status_line(403, error_403_title);
		add_headers(strlen(error_403_form));
		if (!add_content(error_403_form))
		{
			return false;
		}
	}
	break;
	case http_conn::INTERNAL_ERROR:
	{
		add_status_line(500, error_500_title);
		add_headers(strlen(error_500_form));
		if (!add_content(error_500_form))
		{
			return false;
		}
	}
	break;
	case http_conn::FILE_REQUEST:
	{
		add_status_line(200, ok_200_title);
		if (m_file_stat.st_size != 0)
		{
			add_headers(m_file_stat.st_size);
			m_iv[0].iov_base = m_write_buf;
			m_iv[0].iov_len = m_write_idx;
			m_iv[1].iov_base = m_file_address;
			m_iv[1].iov_len = m_file_stat.st_size;
			m_iv_count = 2;
			return true;
		}
		else
		{
			const char* ok_string = "Hello World!";
			add_headers(strlen(ok_string));
			if (!add_content(ok_string))
			{
				return false;
			}
		}
	}
	break;
	default:
		return false;
	}

	m_iv[0].iov_base = m_write_buf;
	m_iv[0].iov_len = m_write_idx;
	m_iv_count = 1;
	return true;
}

/*由线程池中的工作线程调用，这是处理http请求的入口函数*/
void http_conn::process()
{
	HTTP_CODE read_ret = process_read();
	if (read_ret == NO_REQUEST)
	{
		modfd(m_epollfd, m_sockfd, EPOLLIN);
		return;
	}

	bool write_ret = process_write(read_ret);
	if (!write_ret)
	{
		close_conn();
	}
	modfd(m_epollfd, m_sockfd, EPOLLOUT);
}

webserver.cpp

#include 

#include "locker.h"	//locker类
#include "threadpool.h"
#include "http_conn.h"

#define  MAX_FD 65536
#define  MAX_EVENT_NUMBER	10000

extern int addfd(int epollfd, int fd, bool one_shot);
extern int removefd(int epollfd, int fd);

void addsig(int sig, void(handler)(int), bool restart = true)
{
	struct sigaction sa;
	memset(&sa, '\0', sizeof(sa));
	sa.sa_handler = handler;
	if (restart)
	{
		sa.sa_flags |= SA_RESTART;
	}
	sigfillset(&sa.sa_mask);
	assert(sigaction(sig, &sa, NULL) != -1);
}

void show_error(int connfd, const char* info)
{
	printf("%s\n", info);
	send(connfd, info, strlen(info), 0);
	close(connfd);
}

int main(int argc, char* argv[])
{
	if (argc <=2)
	{
		printf("usage: %s ip port\n", argv[0]);
		return 1;
	}

	const char* ip = argv[1];
	int port = atoi(argv[2]);

	/*忽略SIGPIPE信号 在向已经收到RST的socket执行写操作时，内核会向进程发送SIGPIPE信号，告知进程连接对端已关闭
	SIGPIPE默认处理方式是终止进程 所以需要对SIGPIPE信号进行处理*/
	addsig(SIGPIPE, SIG_IGN);

	/*创建线程池*/
	threadpool<http_conn>* pool = NULL;
	try
	{
		pool = new threadpool<http_conn>;
	}
	catch (...)
	{
		printf("new threadpoll failure\n");
		return 1;
	}

	/*预先为每个可能的客户连接分配一个http_conn对象*/
	http_conn* users = new http_conn[MAX_FD];
	assert(users);
	int user_count = 0;

	int listenfd = socket(PF_INET, SOCK_STREAM, 0);
	assert(listenfd >= 0);
	/*SO_LINGER决定close行为，具体看这里：https://blog.csdn.net/qq_20363225/article/details/122352713?spm=1001.2014.3001.5501*/
	struct linger tmp = {1, 0};
	setsockopt(listenfd, SOL_SOCKET, SO_LINGER, &tmp, sizeof(tmp));

	int ret = 0;
	struct sockaddr_in address;
	bzero(&address, sizeof(address));
	address.sin_family = AF_INET;
	address.sin_port = htons(port);
	inet_pton(AF_INET, ip, &address.sin_addr);

	ret = bind(listenfd, (struct sockaddr*)&address, sizeof(address));
	assert(ret != -1);
	ret = listen(listenfd, 5);
	assert(ret != -1);

	epoll_event events[MAX_EVENT_NUMBER];
	int epollfd = epoll_create(5);
	assert(epollfd != -1);
	addfd(epollfd, listenfd, false);
	http_conn::m_epollfd = epollfd;

	while (1)
	{
		int number = epoll_wait(epollfd, events, MAX_EVENT_NUMBER, -1);
		if (number < 0 && errno != EINTR)
		{
			printf("epoll failure\n");
			break;
		}

		for (int i = 0; i < number; ++i)
		{
			int sockfd = events[i].data.fd;
			if (sockfd == listenfd)
			{
				struct sockaddr_in client_address;
				socklen_t client_addrlen = sizeof(client_address);
				int connfd = accept(sockfd, (struct sockaddr*)&client_address, &client_addrlen);
				if (connfd < 0)
				{
					printf("accept failure errnor %d\n", errno);
					continue;
				}
				if (http_conn::m_user_count >= MAX_FD)
				{
					show_error(connfd, "Internal Server busy");
					continue;
				}
				/*初始化客户链接*/
				users[connfd].init(connfd, client_address);
			}
			else if (events[i].events & (EPOLLRDHUP | EPOLLHUP | EPOLLERR))
			{
				/*有异常，直接关闭客户端*/
				users[sockfd].close_conn();
			}
			else if (events[i].events & EPOLLIN)
			{
				/*根据读的结果决定是将任务添加到线程池还是关闭连接*/
				if (users[sockfd].read())
				{
					/*也可以这样写：&(uses[sockfd])*/
					pool->append(users + sockfd);
				}
				else
				{
					users[sockfd].close_conn();
				}
			}
			else if (events[i].events & EPOLLOUT)
			{
				/*根据写的结果，决定是否关闭连接*/
				if (!users[sockfd].write())
				{
					users[sockfd].close_conn();
				}
			}
		}
	}

	close(epollfd);
	close(listenfd);
	delete[] users;
	delete pool;
	return 0;
}