pgpool-II的master-slave模式的分析

磨砺技术珠矶，践行数据之道，追求卓越价值

回到上一级页面： PostgreSQL集群方案相关索引页     回到顶级页面：PostgreSQL索引页

现象描述：

客户来邮件，问：为何Pgpool-II在master-slave模式的时候，发生：

pgpool-II的某子进程与slave db节点间的连接因为长时间无联系被L4SW切断，却不发生failover，而此时向master db节点的commit已经生效，但是马上返回出错信息？

简单言之，那是因为，Pgpool-II开发的时候，没有考虑到这种进程的单独的连接被刻意切断的情形。

此时，如果fail_over_on_backend_error为ture，那么也会激发failover过程。

如果fail_over_on_backend_error为false，而pgpool-II的主进程此时还不断地进行healthcheck，可以正常检测到slave db节点，那么failover过程不会被激发。

上代码：

源代码概要A:

/* * child main loop */                                    
void do_child(int unix_fd, int inet_fd) { … for (;;) { … /* perform accept() */ frontend = do_accept(unix_fd, inet_fd, &timeout); if (frontend =/=* N cUonLLn)ection request from frontend timed out */ { /* check select() timeout */                        
            if (connected && pool_config->child_life_time > 0 && timeout.tv_sec == 0 && timeout.tv_usec == 0) { pool_debug("child life %d seconds expired", pool_config->child_life_time); /* * Doesn't need to call this. child_exit() calls it. * send_frontend_exits(); */ child_exit(2); } continue; } … /* * Ok, negotiaton with frontend has been done. Let's go to the * next step. Connect to backend if there's no existing * connection which can be reused by this frontend. * Authentication is also done in this step. */ … /* * if there's no connection associated with user and database, * we need to connect to the backend and send the startup packet. */                            
        /* look for existing connection */ found = 0; backend = pool_get_cp(sp->user, sp->database, sp->major, 1); … /* Mark this connection pool is conncted from frontend */ pool_coninfo_set_frontend_connected(pool_get_process_context()->proc_id, pool_pool_index()); /* query process loop */                            
        for (;;) { POOL_STATUS status; status = pool_process_query(frontend, backend, 0); sp = MASTER_CONNECTION(backend)->sp; switch (status) { … } if (status != POOL_CONTINUE) break; } … } child_exit(0); } /* * Main module for query processing * reset_request: if non 0, call reset_backend to execute reset queries */ POOL_STATUS pool_process_query(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend, int reset_request) { … for (;;) { … /* * If we are prcessing query, process it. */                            
        if (pool_is_query_in_progress()) { status = ProcessBackendResponse(frontend, backend, &state, &num_fields); if (status != POOL_CONTINUE) return status; } /* * If frontend and all backends do not have any pending data in * the receiving data cache, then issue select(2) to wait for new * data arrival */                            
        else if (is_cache_empty(frontend, backend)) { bool cont = true; ① status = read_packets_and_process(frontend, backend, reset_request, &state, &num_fields, &cont); if (status != POOL_CONTINUE) return status; else if (!c/o*n Dt)etected admin shutdown */                        
                return status; } else { … } … } return POOL_CONTINUE; } /* * Read packet from either frontend or backend and process it. */                                    
static POOL_STATUS read_packets_and_process(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend, int reset_request, int *state, short *num_fields, bool *cont) { … if (!reset_request) { if (FD_ISSET(frontend->fd, &exceptmask)) return POOL_END; else if (FD_ISSET(frontend->fd, &readmask)) { ② status = ProcessFrontendResponse(frontend, backend); if (status != POOL_CONTINUE) return status; } } … return POOL_CONTINUE; } POOL_STATUS ProcessFrontendResponse(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend) { … switch (fkind) { … case 'X': /* Terminate */ free(contents); return POOL_END; case 'Q': /* Query */ allow_close_transaction = 1; ③ status = SimpleQuery(frontend, backend, len, contents); break; … default: pool_error("ProcessFrontendResponse: unknown message type %c(%02x)", fkind, fkind); status = POOL_ERROR; } free(contents); if (status != POOL_CONTINUE) status = POOL_ERROR; return status; } /* * Process Query('Q') message * Query messages include an SQL string. */ POOL_STATUS SimpleQuery(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend, int len, char *contents) { … /* log query to log file if necessary */                                
    if (pool_config->log_statement) { pool_log("statement: %s", contents); } else { pool_debug("statement2: %s", contents); } … if (parse_tree_list != NIL) { … /* * Decide where to send query */ ④ pool_where_to_send(query_context, query_context->original_query, query_context->parse_tree); … } … /* switch memory context */ pool_memory_context_switch_to(old_context); return POOL_CONTINUE; } /* * Decide where to send queries(thus expecting response) */                                    
void pool_where_to_send(POOL_QUERY_CONTEXT *query_context, char *query, Node *node) { … /* * In raw mode, we send only to master node. Simple enough. */                                
    if (RAW_MODE) { pool_set_node_to_be_sent(query_context, REAL_MASTER_NODE_ID); } else if (MASTER_SLAVE && query_context->is_multi_statement) { … } else if (MASTER_SLAVE) { POOL_DEST dest; POOL_MEMORY_POOL *old_context; old_context = pool_memory_context_switch_to(query_context->memory_context); ⑤ dest = send_to_where(node, query); pool_memory_context_switch_to(old_context); pool_debug("send_to_where: %d query: %s", dest, query); /* Should be sent to primary only? */                            
        if (dest == POOL_PRIMARY) { pool_set_node_to_be_sent(query_context, PRIMARY_NODE_ID); } /* Should be sent to both primary and standby? */                            
        else if (dest == POOL_BOTH) { pool_setall_node_to_be_sent(query_context); } /* * Ok, we might be able to load balance the SELECT query. */                            
        else { … } } else if (REPLICATION || PARALLEL_MODE) { … } else { pool_error("pool_where_to_send: unknown mode"); return; } … return; } /* * From syntactically analysis decide the statement to be sent to the * primary, the standby or either or both in master/slave+HR/SR mode. */                                    
static POOL_DEST send_to_where(Node *node, char *query) { if (bsearch(&nodeTag(node), nodemap, sizeof(nodemap)/sizeof(nodemap[0]), sizeof(NodeTag), compare) != NULL) { /* * SELECT INTO * SELECT FOR SHARE or UPDATE */                            
        if (IsA(node, SelectStmt)) { /* SELECT INTO or SELECT FOR SHARE or UPDATE ? */                        
            if (pool_has_insertinto_or_locking_clause(node)) return POOL_PRIMARY; return POOL_EITHER; } … /* * Transaction commands */                            
        else if (IsA(node, TransactionStmt)) { /* * Check "BEGIN READ WRITE" "START TRANSACTION READ WRITE" */                        
            if (is_start_transaction_query(node)) { /* But actually, we send BEGIN to standby if it's BEGIN READ WRITE or START TRANSACTION READ WRITE */                    
                if (is_read_write((TransactionStmt *)node)) return POOL_BOTH; /* Other TRANSACTION start commands are sent to both primary and standby */                
                else                    
                    return POOL_BOTH; } /* SAVEPOINT related commands are sent to both primary and standby */                        
            else if (is_savepoint_query(node)) return POOL_BOTH; /* * 2PC commands */                        
            else if (is_2pc_transaction_query(node)) return POOL_PRIMARY; else                        
            /* COMMIT etc. */                        
                return POOL_BOTH; } … /* * EXECUTE */                            
        else if (IsA(node, ExecuteStmt)) { /* This is temporary decision. where_to_send will inherit * same destination AS PREPARE. */                        
            return POOL_PRIMARY; } … /* * Other statements are sent to primary */                            
        return POOL_PRIMARY; } /* * All unknown statements are sent to primary */                                
    return POOL_PRIMARY; } 

分析如下：

　send_to_where函数中，处在Master/Slave模式的时候，数据的增、删、改指令只向PrimaryDB发送。
　begin/commit这样的事务有关的指令，则既向Master送信，也向Slave送信。

再看源代码概要B:

　通过上述的分析，从pool_process_query→send_to_where　的调用关系，
　commit则既向Master送信，也向Slave送信，但是！
　由于子进程与Slave之间的网络通信被中断，pool_read发生错误，那么此子进程就exit消亡了。
　而此时，已经向PrimaryDB发送了的commit指令，已经成功，是无法取消的。

/* * child main loop */                        
void do_child(int unix_fd, int inet_fd) { … for (;;) { … /* query process loop */                
        for (;;) { POOL_STATUS status; status = pool_process_query(frontend, backend, 0); … switch (status) { … /* error occured. discard backend connection pool and disconnect connection to the frontend */        
                case POOL_ERROR: pool_log("do_child: exits with status 1 due to error"); child_exit(1); break; … default: break; } if (status != POOL_CONTINUE) break; } … } child_exit(0); } /* * Do house keeping works when pgpool child process exits */                        
void child_exit(int code) { … /* let backend know now we are exiting */ send_frontend_exits(); exit(code); } /* * send frontend exiting messages to all connections. this is called * in any case when child process exits, for example failover, child * life time expires or child max connections expires. */                        
static void send_frontend_exits(void) { … for (i=0;i<pool_config->max_pool;i++, p++) { ///ここで、マスタDB関連コネクションへ、exit信号は発送されません 
        if (!MASTER_CONNECTION(p)) continue; if (!MASTER_CONNECTION(p)->sp) continue; if (MASTER_CONNECTION(p)->sp->user == NULL) continue; pool_send_frontend_exits(p); } POOL_SETMASK(&oldmask); } /* * send "terminate"(X) message to all backends, indicating that * backend should prepare to close connection to frontend (actually * pgpool). Note that caller must be protecedt from a signal * interruption while calling this function. Otherwise the number of * valid backends might be changed by failover/failback. */                        
void pool_send_frontend_exits(POOL_CONNECTION_POOL *backend) { … for (i=0;i<NUM_BACKENDS;i++) { … if (VALID_BACKEND(i) && CONNECTION_SLOT(backend, i)) { … pool_set_nonblock(CONNECTION(backend, i)->fd); pool_flush_it(CONNECTION(backend, i)); pool_unset_nonblock(CONNECTION(backend, i)->fd); } } } /* * flush write buffer */                        
int pool_flush_it(POOL_CONNECTION *cp) { … for (;;) { … if (sts > 0) { … } else if (errno == EAGAIN || errno == EINTR) { continue; } else { /* If this is the backend stream, report error. Otherwise * just report debug message. */            
            if (cp->isbackend) pool_error("pool_flush_it: write failed to backend (%d). reason: %s offset: %d wlen: %d", cp->db_node_id, strerror(errno), offset, wlen); else pool_debug("pool_flush_it: write failed to frontend. reason: %s offset: %d wlen: %d", strerror(errno), offset, wlen); cp->wbufpo = 0; return -1; } } … return 0; } /* * Main module for query processing * reset_request: if non 0, call reset_backend to execute reset queries */ POOL_STATUS pool_process_query(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend, int reset_request) { … for (;;) { … /* * If we are prcessing query, process it. */                
        if (pool_is_query_in_progress()) { status = ProcessBackendResponse(frontend, backend, &state, &num_fields); if (status != POOL_CONTINUE) return status; } … } return POOL_CONTINUE; } POOL_STATUS ProcessBackendResponse(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend, int *state, short *num_fields) { … status = read_kind_from_backend(frontend, backend, &kind); if (status != POOL_CONTINUE) return status; … } /* * read_kind_from_backend: read kind from backends. * the "frontend" parameter is used to send "kind mismatch" error message to the frontend. * the out parameter "decided_kind" is the packet kind decided by this function. * this function uses "decide by majority" method if kinds from all backends do not agree. */ POOL_STATUS read_kind_from_backend(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend, char *decided_kind) { … for (i=0;i<NUM_BACKENDS;i++) { … if (VALID_BACKEND(i)) { … do { char *p, *value; int len; if (pool_read(CONNECTION(backend, i), &kind, 1) < 0) { pool_error("read_kind_from_backend: failed to read kind from %d th backend", i); return POOL_ERROR; } … } while (kind == 'S'); … } else kind_list[i] = 0; } … return POOL_CONTINUE; } /* * read len bytes from cp * returns 0 on success otherwise -1. */                        
int pool_read(POOL_CONNECTION *cp, void *buf, int len) { … while (len > 0) { … if (cp->ssl_active > 0) { readlen = pool_ssl_read(cp, readbuf, READBUFSZ); } else { readlen = read(cp->fd, readbuf, READBUFSZ); } … if (readlen == -1) { … pool_error("pool_read: read failed (%s)", strerror(errno)); if (cp->isbackend) { /* if fail_over_on_backend_erro is true, then trigger failover */        
                if (pool_config->fail_over_on_backend_error) { notice_backend_error(cp->db_node_id); child_exit(1); } else        
                    return -1; } else { return -1; } } else if (readlen == 0) { if (cp->isbackend) { pool_error("pool_read: EOF encountered with backend"); return -1; } else { /* * if backend offers authentication method, frontend could close connection */        
                return -1; } } … } return 0; } 

回到上一级页面： PostgreSQL集群方案相关索引页     回到顶级页面：PostgreSQL索引页

磨砺技术珠矶，践行数据之道，追求卓越价值