memcached源码分析之线程池机制（二）

    在上一篇中已分析了memcached线程池的创建流程，由于上篇篇幅较长，因此将memcached线程池中线程的调度流程另立一篇。

先让我们把目光转到主函数中，主线程在调用thread_init函数创建好线程池后，就开始创建监听套接字，memcached支持TCP，UDP，UNIX域套接字，因此相应的要创建三种监听套接字

这里我们只分析TCP listening socket的创建（UDP与TCP的创建采用统一的接口），函数入口为：

1  errno = 0;

2         if (settings.port && server_sockets(settings.port, tcp_transport,

3                                            portnumber_file)) {

4             vperror("failed to listen on TCP port %d", settings.port);

5             exit(EX_OSERR);

6         }

server_sockets函数即为创建TCP listening socket的入口函数。在server_sockets主要调用server_socket函数来实现，

 1 /**

 2  * Create a socket and bind it to a specific port number

 3  * @param interface the interface to bind to

 4  * @param port the port number to bind to

 5  * @param transport the transport protocol (TCP / UDP)

 6  * @param portnumber_file A filepointer to write the port numbers to

 7  *        when they are successfully added to the list of ports we

 8  *        listen on.

 9  */

10 static int server_socket(const char *interface,

11                          int port,

12                          enum network_transport transport,

13                          FILE *portnumber_file)；

server_socket函数实现源码较长，以下只列出部分：

 1 static int server_socket(const char *interface,

 2                          int port,

 3                          enum network_transport transport,

 4                          FILE *portnumber_file) {

 5     int sfd;

 6     struct linger ling = {0, 0};

 7     struct addrinfo *ai;

 8     struct addrinfo *next;

 9     struct addrinfo hints = { .ai_flags = AI_PASSIVE,

10                               .ai_family = AF_UNSPEC };

11     //套接字的创建过程

12     ..................

13     ..................

14         if (IS_UDP(transport)) {

15             int c;

16 

17             for (c = 0; c < settings.num_threads_per_udp; c++) {

18                 /* this is guaranteed to hit all threads because we round-robin */

19                 dispatch_conn_new(sfd, conn_read, EV_READ | EV_PERSIST,

20                                   UDP_READ_BUFFER_SIZE, transport);

21             }

22         } else {

23             if (!(listen_conn_add = conn_new(sfd, conn_listening,

24                                              EV_READ | EV_PERSIST, 1,

25                                              transport, main_base))) {

26                 fprintf(stderr, "failed to create listening connection\n");

27                 exit(EXIT_FAILURE);

28             }

29             listen_conn_add->next = listen_conn;

30             listen_conn = listen_conn_add;

31         }

32     }

33 

34     freeaddrinfo(ai);

35 

36     /* Return zero iff we detected no errors in starting up connections */

37     return success == 0;

38 }

在server_socket中，我们只关注两个函数：

（1）dispatch_conn_new(sfd, conn_read, EV_READ | EV_PERSIST,UDP_READ_BUFFER_SIZE, transport);

当创建的是UDP套接字时，使用这个函数，由于UDP是无连接的，因此直接启动settings.num_threads_per_udp个线程来服务于UDP端口。

（2）listen_conn_add = conn_new(sfd, conn_listening,EV_READ | EV_PERSIST, 1,transport, main_base);

当创建的是TCP套接字时，调用conn_new函数，源码如下：

 1 conn *conn_new(const int sfd, enum conn_states init_state,

 2                 const int event_flags,

 3                 const int read_buffer_size, enum network_transport transport,

 4                 struct event_base *base) {

 5     conn *c = conn_from_freelist();

 6 

 7     ........................//略去部分源码

 8 

 9     event_set(&c->event, sfd, event_flags, event_handler, (void *)c);

10     event_base_set(base, &c->event);

11     c->ev_flags = event_flags;

12 

13     if (event_add(&c->event, 0) == -1) {

14         if (conn_add_to_freelist(c)) {

15             conn_free(c);

16         }

17         perror("event_add");

18         return NULL;

19     }

20 

21     STATS_LOCK();

22     stats.curr_conns++;

23     stats.total_conns++;

24     STATS_UNLOCK();

25 

26     MEMCACHED_CONN_ALLOCATE(c->sfd);

27 

28     return c;

29 }

该函数对套接字设置conn_listening监听事件，回调函数为event_handler，在事件响应函数中调用状态机。

 1 void event_handler(const int fd, const short which, void *arg) {

 2     conn *c;

 3 

 4     c = (conn *)arg;

 5     assert(c != NULL);

 6 

 7     c->which = which;

 8 

 9     /* sanity */

10     if (fd != c->sfd) {

11         if (settings.verbose > 0)

12             fprintf(stderr, "Catastrophic: event fd doesn't match conn fd!\n");

13         conn_close(c);

14         return;

15     }

16 

17     drive_machine(c);

18 

19     /* wait for next event */

20     return;

21 }

memcached中的状态机是memcached运转发动机，它根据链接的不同状态而采取不同的行为，状态枚举如下：

 1 enum conn_states {

 2     conn_listening,  /**< the socket which listens for connections */

 3     conn_new_cmd,    /**< Prepare connection for next command */

 4     conn_waiting,    /**< waiting for a readable socket */

 5     conn_read,       /**< reading in a command line */

 6     conn_parse_cmd,  /**< try to parse a command from the input buffer */

 7     conn_write,      /**< writing out a simple response */

 8     conn_nread,      /**< reading in a fixed number of bytes */

 9     conn_swallow,    /**< swallowing unnecessary bytes w/o storing */

10     conn_closing,    /**< closing this connection */

11     conn_mwrite,     /**< writing out many items sequentially */

12     conn_max_state   /**< Max state value (used for assertion) */

13 };

 状态机的实现函数为drive_machine，由于该函数的源码实现过长，这里只分析对conn_listening状态的响应。

 1 static void drive_machine(conn *c) {

 2     bool stop = false;

 3     int sfd, flags = 1;

 4     socklen_t addrlen;

 5     struct sockaddr_storage addr;

 6     int nreqs = settings.reqs_per_event;

 7     int res;

 8 

 9     assert(c != NULL);

10 

11     while (!stop) {

12 

13         switch(c->state) {

14             //监听套接字发生事件

15         case conn_listening:

16             addrlen = sizeof(addr);

17             if ((sfd = accept(c->sfd, (struct sockaddr *)&addr, &addrlen)) == -1) {

18                 if (errno == EAGAIN || errno == EWOULDBLOCK) {

19                     /* these are transient, so don't log anything */

20                     stop = true;

21                 } else if (errno == EMFILE) {

22                     if (settings.verbose > 0)

23                         fprintf(stderr, "Too many open connections\n");

24                     accept_new_conns(false);

25                     stop = true;

26                 } else {

27                     perror("accept()");

28                     stop = true;

29                 }

30                 break;

31             }

32             //新套接字设置非阻塞

33             if ((flags = fcntl(sfd, F_GETFL, 0)) < 0 ||

34                 fcntl(sfd, F_SETFL, flags | O_NONBLOCK) < 0) {

35                 perror("setting O_NONBLOCK");

36                 close(sfd);

37                 break;

38             }

39             //调度线程来处理连接

40             dispatch_conn_new(sfd, conn_new_cmd, EV_READ | EV_PERSIST,

41                                      DATA_BUFFER_SIZE, tcp_transport);

42             stop = true;

43             break;

44 ...........................//略去其他状态的处理

45 }

从源码中我们可以看到，当监听套接字建立新连接时，通过事件响应函数event_handler来触发状态机，再调用dispatch_conn_new调度新线程来处理这个连接的读写事件。

 1 /*

 2  * Dispatches a new connection to another thread. This is only ever called

 3  * from the main thread, either during initialization (for UDP) or because

 4  * of an incoming connection.

 5  */

 6 void dispatch_conn_new(int sfd, enum conn_states init_state, int event_flags,

 7                        int read_buffer_size, enum network_transport transport) {

 8     CQ_ITEM *item = cqi_new();

 9     int tid = (last_thread + 1) % settings.num_threads;

10 

11     //以此种方式来取出线程

12     LIBEVENT_THREAD *thread = threads + tid;

13 

14     last_thread = tid;

15 

16     item->sfd = sfd;

17     item->init_state = init_state;

18     item->event_flags = event_flags;

19     item->read_buffer_size = read_buffer_size;

20     item->transport = transport;

21 

22     //将新item放至threads的new_conn_queue队列中

23     cq_push(thread->new_conn_queue, item);

24 

25     MEMCACHED_CONN_DISPATCH(sfd, thread->thread_id);

26     //写一个字节启动新的线程

27     if (write(thread->notify_send_fd, "", 1) != 1) {

28         perror("Writing to thread notify pipe");

29     }

30 }

至此，memcached的线程池调度机制已分析完毕了。