本文讲述gateserver的流程，同时讲解socket的流程
以watchdog.lua开始讲解
在main.lua函数
--main.lua
skynet.start(function()
skynet.error("Server start")
skynet.uniqueservice("protoloader")
if not skynet.getenv "daemon" then
local console = skynet.newservice("console")
end
skynet.newservice("debug_console",8000)
skynet.newservice("simpledb")
local watchdog = skynet.newservice("watchdog")
skynet.call(watchdog, "lua", "start", {
port = 8888,
maxclient = max_client,
nodelay = true,
})
skynet.error("Watchdog listen on", 8888)
skynet.exit()
end)
在main.lua中启动watchdog服务，启动之后给watchdog发送start协议。
--watchdog.lua
skynet.start(function()
skynet.dispatch("lua", function(session, source, cmd, subcmd, ...)
if cmd == "socket" then
local f = SOCKET[subcmd]
f(...)
-- socket api don't need return
else
local f = assert(CMD[cmd])
skynet.ret(skynet.pack(f(subcmd, ...)))
end
end)
gate = skynet.newservice("gate")---------------①
end)
watchdog启动的时候会在①处启动gate服务。gate.lua和gateserver.lua两个文件是属于同一个服务。在gate.lua中执行gateserver.start(handler)。在gateserver.lua中
function gateserver.start(handler)
assert(handler.message)
assert(handler.connect)
function CMD.open( source, conf ) -------------②
assert(not socket)
local address = conf.address or "0.0.0.0"
local port = assert(conf.port)
maxclient = conf.maxclient or 1024
nodelay = conf.nodelay
skynet.error(string.format("Listen on %s:%d", address, port))
socket = socketdriver.listen(address, port) ----------------③
socketdriver.start(socket)-----------------------④
if handler.open then
return handler.open(source, conf)------------------⑤
end
end
function CMD.close()
assert(socket)
socketdriver.close(socket)
end
local MSG = {}
local function dispatch_msg(fd, msg, sz)
if connection[fd] then
handler.message(fd, msg, sz)
else
skynet.error(string.format("Drop message from fd (%d) : %s", fd, netpack.tostring(msg,sz)))
end
end
MSG.data = dispatch_msg
local function dispatch_queue()
local fd, msg, sz = netpack.pop(queue)
if fd then
-- may dispatch even the handler.message blocked
-- If the handler.message never block, the queue should be empty, so only fork once and then exit.
skynet.fork(dispatch_queue)
dispatch_msg(fd, msg, sz)
for fd, msg, sz in netpack.pop, queue do
dispatch_msg(fd, msg, sz)
end
end
end
MSG.more = dispatch_queue
function MSG.open(fd, msg)
if client_number >= maxclient then
socketdriver.close(fd)
return
end
if nodelay then
socketdriver.nodelay(fd)
end
connection[fd] = true
client_number = client_number + 1
handler.connect(fd, msg)
end
local function close_fd(fd)
local c = connection[fd]
if c ~= nil then
connection[fd] = nil
client_number = client_number - 1
end
end
function MSG.close(fd)
if fd ~= socket then
if handler.disconnect then
handler.disconnect(fd)
end
close_fd(fd)
else
socket = nil
end
end
function MSG.error(fd, msg)
if fd == socket then
socketdriver.close(fd)
skynet.error("gateserver close listen socket, accpet error:",msg)
else
if handler.error then
handler.error(fd, msg)
end
close_fd(fd)
end
end
function MSG.warning(fd, size)
if handler.warning then
handler.warning(fd, size)
end
end
skynet.register_protocol {
name = "socket",
id = skynet.PTYPE_SOCKET, -- PTYPE_SOCKET = 6
unpack = function ( msg, sz )
return netpack.filter( queue, msg, sz)
end,
dispatch = function (_, _, q, type, ...)
queue = q
if type then
MSG[type](...)
end
end
}
skynet.start(function() ------①
skynet.dispatch("lua", function (_, address, cmd, ...)
local f = CMD[cmd]
if f then
skynet.ret(skynet.pack(f(address, ...)))
else
skynet.ret(skynet.pack(handler.command(cmd, address, ...)))
end
end)
end)
end
最后，启动watchdog服务最后会调用①。
当我们给watchdog发送start消息的时候，会给gate服务发送open消息。当gate手动open消息之后，调用②处函数，启动socket。依次顺序如下：
③中 socketdriver.listen会调用lua-socket.c中的llisten()。④start⑤open
//lua-socket.c
static int
llisten(lua_State *L) {
const char * host = luaL_checkstring(L,1);
int port = luaL_checkinteger(L,2);
int backlog = luaL_optinteger(L,3,BACKLOG);
struct skynet_context * ctx = lua_touserdata(L, lua_upvalueindex(1));
int id = skynet_socket_listen(ctx, host,port,backlog);//------a
if (id < 0) {
return luaL_error(L, "Listen error");
}
lua_pushinteger(L,id);
return 1;
}
a:绑定监听ip和端口
//创建用于监听的socket句柄等
static int
do_bind(const char *host, int port, int protocol, int *family) {
int fd;
int status;
int reuse = 1;
struct addrinfo ai_hints;
struct addrinfo *ai_list = NULL;
char portstr[16];
if (host == NULL || host[0] == 0) {
host = "0.0.0.0"; // INADDR_ANY
}
sprintf(portstr, "%d", port);
memset( &ai_hints, 0, sizeof( ai_hints ) );
ai_hints.ai_family = AF_UNSPEC;
if (protocol == IPPROTO_TCP) {
ai_hints.ai_socktype = SOCK_STREAM;
} else {
assert(protocol == IPPROTO_UDP);
ai_hints.ai_socktype = SOCK_DGRAM;
}
ai_hints.ai_protocol = protocol;
status = getaddrinfo( host, portstr, &ai_hints, &ai_list );
if ( status != 0 ) {
return -1;
}
*family = ai_list->ai_family;
fd = socket(*family, ai_list->ai_socktype, 0);
if (fd < 0) {
goto _failed_fd;
}
if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(int))==-1) {
goto _failed;
}
status = bind(fd, (struct sockaddr *)ai_list->ai_addr, ai_list->ai_addrlen); //服务器端绑定IP和端口
if (status != 0)
goto _failed;
freeaddrinfo( ai_list );
return fd;
_failed:
close(fd);
_failed_fd:
freeaddrinfo( ai_list );
return -1;
}
static int
do_listen(const char * host, int port, int backlog) {
int family = 0;
int listen_fd = do_bind(host, port, IPPROTO_TCP, &family); //绑定IP和端口
if (listen_fd < 0) {
return -1;
}
if (listen(listen_fd, backlog) == -1) { //监听fd
close(listen_fd);
return -1;
}
return listen_fd;
}
int
socket_server_listen(struct socket_server *ss, uintptr_t opaque, const char * addr, int port, int backlog) {
int fd = do_listen(addr, port, backlog); //----------b
if (fd < 0) {
return -1;
}
struct request_package request;
int id = reserve_id(ss);
if (id < 0) {
close(fd);
return id;
}
request.u.listen.opaque = opaque;
request.u.listen.id = id;
request.u.listen.fd = fd;
send_request(ss, &request, 'L', sizeof(request.u.listen)); //-----d
return id;
}
d处 给socket_server服务发送Listen消息。
static void
send_request(struct socket_server *ss, struct request_package *request, char type, int len) {
request->header[6] = (uint8_t)type;
request->header[7] = (uint8_t)len;
for (;;) {
ssize_t n = write(ss->sendctrl_fd, &request->header[6], len+2);
if (n<0) {
if (errno != EINTR) {
fprintf(stderr, "socket-server : send ctrl command error %s.\n", strerror(errno));
}
continue;
}
assert(n == len+2);
return;
}
}
会将type 和len写到管道里面去。接下来讲seocket_server服务的接收过程。
///////
①
//skynet_start.c
static void *
thread_socket(void *p) {
struct monitor * m = p;
skynet_initthread(THREAD_SOCKET);
for (;;) {
int r = skynet_socket_poll(); -----A
if (r==0)
break;
if (r<0) {
CHECK_ABORT
continue;
}
wakeup(m,0);
}
return NULL;
}
A:服务器启动的时候，会创建一个线程专门处理socket消息。线程回调函数是thread_socket（）。里面会一直循环。
//skynet_socket.c
int
skynet_socket_poll() {
struct socket_server *ss = SOCKET_SERVER;
assert(ss);
struct socket_message result;
int more = 1;
int type = socket_server_poll(ss, &result, &more); //B
switch (type) {
case SOCKET_EXIT:
return 0;
case SOCKET_DATA:
forward_message(SKYNET_SOCKET_TYPE_DATA, false, &result);
break;
case SOCKET_CLOSE:
forward_message(SKYNET_SOCKET_TYPE_CLOSE, false, &result);
break;
case SOCKET_OPEN:
forward_message(SKYNET_SOCKET_TYPE_CONNECT, true, &result);
break;
case SOCKET_ERR:
forward_message(SKYNET_SOCKET_TYPE_ERROR, true, &result);
break;
case SOCKET_ACCEPT:
forward_message(SKYNET_SOCKET_TYPE_ACCEPT, true, &result);
break;
case SOCKET_UDP:
forward_message(SKYNET_SOCKET_TYPE_UDP, false, &result);
break;
case SOCKET_WARNING:
forward_message(SKYNET_SOCKET_TYPE_WARNING, false, &result);
break;
default:
skynet_error(NULL, "Unknown socket message type %d.",type);
return -1;
}
if (more) {
return -1;
}
return 1;
}
B:
//socket_server.c
int
socket_server_poll(struct socket_server *ss, struct socket_message * result, int * more) {
for (;;) {
if (ss->checkctrl) {
if (has_cmd(ss)) { //判断监听的读管道里是否有数据
int type = ctrl_cmd(ss, result); //C
if (type != -1) {
clear_closed_event(ss, result, type);
return type;
} else
continue;
} else {
ss->checkctrl = 0;
}
}
if (ss->event_index == ss->event_n) {
ss->event_n = sp_wait(ss->event_fd, ss->ev, MAX_EVENT);
ss->checkctrl = 1;
if (more) {
*more = 0;
}
ss->event_index = 0;
if (ss->event_n <= 0) {
ss->event_n = 0;
if (errno == EINTR) {
continue;
}
return -1;
}
}
struct event *e = &ss->ev[ss->event_index++];
struct socket *s = e->s;
if (s == NULL) {
// dispatch pipe message at beginning
continue;
}
struct socket_lock l;
socket_lock_init(s, &l);
switch (s->type) {
case SOCKET_TYPE_CONNECTING:
return report_connect(ss, s, &l, result);
case SOCKET_TYPE_LISTEN: {
int ok = report_accept(ss, s, result);
if (ok > 0) {
return SOCKET_ACCEPT;
} if (ok < 0 ) {
return SOCKET_ERR;
}
// when ok == 0, retry
break;
}
case SOCKET_TYPE_INVALID:
fprintf(stderr, "socket-server: invalid socket\n");
break;
default:
if (e->read) {
int type;
if (s->protocol == PROTOCOL_TCP) {
type = forward_message_tcp(ss, s, &l, result);
} else {
type = forward_message_udp(ss, s, &l, result);
if (type == SOCKET_UDP) {
// try read again
--ss->event_index;
return SOCKET_UDP;
}
}
if (e->write && type != SOCKET_CLOSE && type != SOCKET_ERR) {
// Try to dispatch write message next step if write flag set.
e->read = false;
--ss->event_index;
}
if (type == -1)
break;
return type;
}
if (e->write) {
int type = send_buffer(ss, s, &l, result);
if (type == -1)
break;
return type;
}
if (e->error) {
// close when error
int error;
socklen_t len = sizeof(error);
int code = getsockopt(s->fd, SOL_SOCKET, SO_ERROR, &error, &len);
const char * err = NULL;
if (code < 0) {
err = strerror(errno);
} else if (error != 0) {
err = strerror(error);
} else {
err = "Unknown error";
}
force_close(ss, s, &l, result);
result->data = (char *)err;
return SOCKET_ERR;
}
if(e->eof) {
force_close(ss, s, &l, result);
return SOCKET_CLOSE;
}
break;
}
}
}
C:各种socket消息的处理函数
// return type
static int
ctrl_cmd(struct socket_server *ss, struct socket_message *result) {
int fd = ss->recvctrl_fd;
// the length of message is one byte, so 256+8 buffer size is enough.
uint8_t buffer[256];
uint8_t header[2];
block_readpipe(fd, header, sizeof(header));
int type = header[0];
int len = header[1];
block_readpipe(fd, buffer, len); //读取管道的数据
// ctrl command only exist in local fd, so don't worry about endian.
switch (type) {
case 'S':
return start_socket(ss,(struct request_start *)buffer, result);
case 'B':
return bind_socket(ss,(struct request_bind *)buffer, result);
case 'L':
return listen_socket(ss,(struct request_listen *)buffer, result);
case 'K':
return close_socket(ss,(struct request_close *)buffer, result);
case 'O':
return open_socket(ss, (struct request_open *)buffer, result);
case 'X':
result->opaque = 0;
result->id = 0;
result->ud = 0;
result->data = NULL;
return SOCKET_EXIT;
case 'D':
case 'P': {
int priority = (type == 'D') ? PRIORITY_HIGH : PRIORITY_LOW;
struct request_send * request = (struct request_send *) buffer;
int ret = send_socket(ss, request, result, priority, NULL);
dec_sending_ref(ss, request->id);
return ret;
}
case 'A': {
struct request_send_udp * rsu = (struct request_send_udp *)buffer;
return send_socket(ss, &rsu->send, result, PRIORITY_HIGH, rsu->address);
}
case 'C':
return set_udp_address(ss, (struct request_setudp *)buffer, result);
case 'T':
setopt_socket(ss, (struct request_setopt *)buffer);
return -1;
case 'U':
add_udp_socket(ss, (struct request_udp *)buffer);
return -1;
default:
fprintf(stderr, "socket-server: Unknown ctrl %c.\n",type);
return -1;
};
return -1;
}
static int
listen_socket(struct socket_server *ss, struct request_listen * request, struct socket_message *result) {
int id = request->id;
int listen_fd = request->fd;
struct socket *s = new_fd(ss, id, listen_fd, PROTOCOL_TCP, request->opaque, false);
if (s == NULL) {
goto _failed;
}
s->type = SOCKET_TYPE_PLISTEN;
return -1;
_failed:
close(listen_fd);
result->opaque = request->opaque;
result->id = id;
result->ud = 0;
result->data = "reach skynet socket number limit";
ss->slot[HASH_ID(id)].type = SOCKET_TYPE_INVALID;
return SOCKET_ERR;
}
static struct socket *
new_fd(struct socket_server *ss, int id, int fd, int protocol, uintptr_t opaque, bool add) {
struct socket * s = &ss->slot[HASH_ID(id)];
assert(s->type == SOCKET_TYPE_RESERVE);
if (add) {
if (sp_add(ss->event_fd, fd, s)) {
s->type = SOCKET_TYPE_INVALID;
return NULL;
}
}
s->id = id;
s->fd = fd;
s->sending = ID_TAG16(id) << 16 | 0;
s->protocol = protocol;
s->p.size = MIN_READ_BUFFER;
s->opaque = opaque;
s->wb_size = 0;
s->warn_size = 0;
check_wb_list(&s->high);
check_wb_list(&s->low);
s->dw_buffer = NULL;
s->dw_size = 0;
memset(&s->stat, 0, sizeof(s->stat));
return s;
}
static int
sp_add(int efd, int sock, void *ud) {
struct epoll_event ev;
ev.events = EPOLLIN;
ev.data.ptr = ud;
if (epoll_ctl(efd, EPOLL_CTL_ADD, sock, &ev) == -1) {
return 1;
}
return 0;
}