FastDFS源码阅读笔记(三)

 然后在是跟踪服务器tacker部分:

  (五).fast_queue_queue.c

static struct fast_task_queue g_free_queue;   static struct fast_task_info *g_mpool = NULL;   int task_queue_init(struct fast_task_queue *pQueue) {     int result;     if ((result=init_pthread_lock(&(pQueue->lock))) != 0)     {         logError("file: "__FILE__", line: %d, " /             "init_pthread_lock fail, errno: %d, error info: %s", /             __LINE__, result, strerror(result));         return result;     }     pQueue->head = NULL;     pQueue->tail = NULL;       return 0; }   //空闲队列初始化 int free_queue_init(const int max_connections, const int min_buff_size, /                     const int max_buff_size, const int arg_size) {     struct fast_task_info *pTask;     char *p;     char *pCharEnd;     int block_size;     int alloc_size;     int64_t total_size;     int result;     if ((result=init_pthread_lock(&(g_free_queue.lock))) != 0)     {         logError("file: "__FILE__", line: %d, " /             "init_pthread_lock fail, errno: %d, error info: %s", /             __LINE__, result, strerror(result));         return result;     }     //设置块大小: 每一块的大小=任务体结构大小+附加的信息     block_size = sizeof(struct fast_task_info) + arg_size; //arg_size=sizeof(TrackerClientInfo) 两个指针的大小     //总的分配大小     alloc_size = block_size * max_connections;     //在tracker_service_init()函数里: max_buff_size=min_buff_size=8*1024=8K     if (max_buff_size > min_buff_size)     {         total_size = alloc_size;         g_free_queue.malloc_whole_block = false;     }     else     {         struct rlimit rlimit_data;         rlim_t max_data_size;         if (getrlimit(RLIMIT_DATA, &rlimit_data) < 0)         {             logError("file: "__FILE__", line: %d, " /                 "call getrlimit fail, " /                 "errno: %d, error info: %s", /                 __LINE__, errno, strerror(errno));             return errno != 0 ? errno : EPERM;         }         //设定上限M         if (rlimit_data.rlim_cur == RLIM_INFINITY)         {             max_data_size = 512 * 1024 * 1024;         }         else         {             max_data_size = rlimit_data.rlim_cur;             if (max_data_size > 512 * 1024 * 1024)             {                 max_data_size = 512 * 1024 * 1024;             }         }         //总的尺寸: (sizeof(struct fast_task_info) + arg_size+min_buff_size)*max_connections         total_size = alloc_size+(int64_t)min_buff_size*max_connections; //每个包的大小*连接数+附加信息的大小         if (total_size <= max_data_size)         {             //如果整个尺寸<设定的最大数据尺寸,那么分配整个块malloc_whole_block=true             g_free_queue.malloc_whole_block = true;             //块大小=sizeof(struct fast_task_info) + arg_size+min_buff_size             block_size += min_buff_size;         }         else         {             //否则的话,并不整个进行分配,并不对数据包进行分配大小,数据包的缓冲需要在重新申请内存             g_free_queue.malloc_whole_block = false;             total_size = alloc_size;  //不包括数据包的大小         }     }     //设置内存池     g_mpool = (struct fast_task_info *)malloc(total_size);     if (g_mpool == NULL)     {         logError("file: "__FILE__", line: %d, " /             "malloc "INT64_PRINTF_FORMAT" bytes fail, " /             "errno: %d, error info: %s", /             __LINE__, total_size, errno, strerror(errno));         return errno != 0 ? errno : ENOMEM;     }     memset(g_mpool, 0, total_size);     //重新将g_mpool转为字节指针     pCharEnd = ((char *)g_mpool) + total_size;     //每一步的偏移是块的大小,block_size大小包括两种情况,包含有数据包以及不含有数据包的情况     for (p=(char *)g_mpool; p<pCharEnd; p += block_size)     {         pTask = (struct fast_task_info *)p;         pTask->size = min_buff_size;  //设置包的大小         pTask->arg = p + sizeof(struct fast_task_info);  //这里参数显然指向TrackerClientInfo         //如果申请内存小于限制malloc_whole_block=true         if (g_free_queue.malloc_whole_block)         {             //如果没有分配数据包的内存,那么直接使用已经申请的内存             //data: buffer for write or recv             pTask->data = pTask->arg + arg_size;          }         else         {             //否则的话,单独申请数据包的内存             pTask->data = malloc(pTask->size);             if (pTask->data == NULL)             {                 free_queue_destroy();                 logError("file: "__FILE__", line: %d, " /                     "malloc %d bytes fail, " /                     "errno: %d, error info: %s", /                     __LINE__, pTask->size, /                     errno, strerror(errno));                 return errno != 0 ? errno : ENOMEM;             }         }     }     //设置尾指针,构成任务空闲队列     g_free_queue.tail = (struct fast_task_info *)(pCharEnd - block_size);     for (p=(char *)g_mpool; p<(char *)g_free_queue.tail; p += block_size)     {         pTask = (struct fast_task_info *)p;         pTask->next = (struct fast_task_info *)(p + block_size);     }     g_free_queue.max_connections = max_connections;     g_free_queue.min_buff_size = min_buff_size;     g_free_queue.max_buff_size = max_buff_size;     g_free_queue.arg_size = arg_size;     g_free_queue.head = g_mpool;     g_free_queue.tail->next = NULL;     return 0; }   /*清除任务队列*/ void free_queue_destroy() {     if (g_mpool == NULL)     {         return;     }     //如果接收的数据包的内存是自己申请的     if (!g_free_queue.malloc_whole_block)     {         char *p;         char *pCharEnd;         int block_size;         struct fast_task_info *pTask;         /*按block_size作为偏移量进行释放*/         block_size = sizeof(struct fast_task_info) + /             g_free_queue.arg_size;         pCharEnd = ((char *)g_mpool) + block_size * /             g_free_queue.max_connections;         for (p=(char *)g_mpool; p<pCharEnd; p += block_size)         {             pTask = (struct fast_task_info *)p;             if (pTask->data != NULL)             {                 free(pTask->data);                 pTask->data = NULL;             }         }     }     free(g_mpool);     g_mpool = NULL;     pthread_mutex_destroy(&(g_free_queue.lock)); }   struct fast_task_info *free_queue_pop() {     return task_queue_pop(&g_free_queue);; }   /*将任务返回到空闲队列中*/ int free_queue_push(struct fast_task_info *pTask) {     char *new_buff;     int result;     *(pTask->client_ip) = '/0';     pTask->length = 0;     pTask->offset = 0;     pTask->req_count = 0;     //对数据包的申请内存的处理     if (pTask->size > g_free_queue.min_buff_size) //need thrink     {         new_buff = (char *)malloc(g_free_queue.min_buff_size);         if (new_buff == NULL)         {             logWarning("file: "__FILE__", line: %d, " /                 "malloc %d bytes fail, " /                 "errno: %d, error info: %s", /                 __LINE__, g_free_queue.min_buff_size, /                 errno, strerror(errno));         }         else         {             //将该任务申请的内存重新设置为最小缓冲尺寸             free(pTask->data);             pTask->size = g_free_queue.min_buff_size;             pTask->data = new_buff;         }     }       if ((result=pthread_mutex_lock(&g_free_queue.lock)) != 0)     {         logError("file: "__FILE__", line: %d, " /             "call pthread_mutex_lock fail, " /             "errno: %d, error info: %s", /             __LINE__, result, strerror(result));     }     //最后将使用完成的任务移到队列的头部     pTask->next = g_free_queue.head;     g_free_queue.head = pTask;     if (g_free_queue.tail == NULL)     {         g_free_queue.tail = pTask;     }       if ((result=pthread_mutex_unlock(&g_free_queue.lock)) != 0)     {         logError("file: "__FILE__", line: %d, " /             "call pthread_mutex_unlock fail, " /             "errno: %d, error info: %s", /             __LINE__, result, strerror(result));     }       return result; }   int free_queue_count() {     return task_queue_count(&g_free_queue); }   int task_queue_push(struct fast_task_queue *pQueue,struct fast_task_info *pTask) {     int result;       if ((result=pthread_mutex_lock(&(pQueue->lock))) != 0)     {         logError("file: "__FILE__", line: %d, " /             "call pthread_mutex_lock fail, " /             "errno: %d, error info: %s", /             __LINE__, result, strerror(result));         return result;     }     //push到队列的尾部     pTask->next = NULL;     if (pQueue->tail == NULL)     {         pQueue->head = pTask;     }     else     {         pQueue->tail->next = pTask;     }     pQueue->tail = pTask;     if ((result=pthread_mutex_unlock(&(pQueue->lock))) != 0)     {         logError("file: "__FILE__", line: %d, " /             "call pthread_mutex_unlock fail, " /             "errno: %d, error info: %s", /             __LINE__, result, strerror(result));     }       return 0; }   //取一个空闲任务,因为是在线程里面进行的处理,所以需要加锁的操作 //并且是弹出了这个任务 struct fast_task_info *task_queue_pop(struct fast_task_queue *pQueue) {     struct fast_task_info *pTask;     int result;     if ((result=pthread_mutex_lock(&(pQueue->lock))) != 0)     {         logError("file: "__FILE__", line: %d, " /             "call pthread_mutex_lock fail, " /             "errno: %d, error info: %s", /             __LINE__, result, strerror(result));         return NULL;     }     pTask = pQueue->head;     if (pTask != NULL)     {         pQueue->head = pTask->next;         if (pQueue->head == NULL)         {             pQueue->tail = NULL;         }     }     if ((result=pthread_mutex_unlock(&(pQueue->lock))) != 0)     {         logError("file: "__FILE__", line: %d, " /             "call pthread_mutex_unlock fail, " /             "errno: %d, error info: %s", /             __LINE__, result, strerror(result));     }     return pTask; }

 

 (六).tracker_nio.c

//任务完成清空操作 //跟客户端建立的socket读写操作,都会取一个任务指针,在发生异常情况下,在回收这个任务指针 //所以建立的socket连接是长连接 void task_finish_clean_up(struct fast_task_info *pTask) {     TrackerClientInfo *pClientInfo;     pClientInfo = (TrackerClientInfo *)pTask->arg;     if (pClientInfo->pGroup != NULL)     {         if (pClientInfo->pStorage != NULL)         {             tracker_mem_offline_store_server(pClientInfo->pGroup,pClientInfo->pStorage);             pClientInfo->pStorage = NULL;         }         pClientInfo->pGroup = NULL;     }     free_queue_push(pTask); }   //服务端接收到读的信息:回调函数,socket的处理过程是先创建工作线程,在线程里面注册事件处理函数,同事处理过程 //也是在线程里面进行处理,因此在线程里面的操作需要加锁处理 //然后服务端在接收客户端连接,进行socket的读写操作 void recv_notify_read(int sock, short event, void *arg) {     int bytes;     int incomesock;     int result;     struct tracker_thread_data *pThreadData;     struct fast_task_info *pTask;     char szClientIp[IP_ADDRESS_SIZE];     in_addr_t client_addr;     while (1)     {         if ((bytes=read(sock, &incomesock, sizeof(incomesock))) < 0)         {             if (!(errno == EAGAIN || errno == EWOULDBLOCK))             {                 logError("file: "__FILE__", line: %d, " /                     "call read failed, " /                     "errno: %d, error info: %s", /                     __LINE__, errno, strerror(errno));             }               break;         }         else if (bytes == 0)         {             break;         }         if (incomesock < 0)         {             struct timeval tv;                         tv.tv_sec = 1;                         tv.tv_usec = 0;             pThreadData = g_thread_data + (-1 * incomesock - 1) % /                     g_work_threads;             event_base_loopexit(pThreadData->ev_base, &tv);             return;         }         //获得客户端的ip地址         client_addr = getPeerIpaddr(incomesock,szClientIp, IP_ADDRESS_SIZE);         if (g_allow_ip_count >= 0)         {             if (bsearch(&client_addr, g_allow_ip_addrs, /                     g_allow_ip_count, sizeof(in_addr_t), /                     cmp_by_ip_addr_t) == NULL)             {                 logError("file: "__FILE__", line: %d, " /                     "ip addr %s is not allowed to access", /                     __LINE__, szClientIp);                   close(incomesock);                 continue;             }         }         if (tcpsetnonblockopt(incomesock) != 0)         {             close(incomesock);             continue;         }         //从队列头部取出一个空闲任务         pTask = free_queue_pop();         if (pTask == NULL)         {             logError("file: "__FILE__", line: %d, " /                 "malloc task buff failed", /                 __LINE__);             close(incomesock);             continue;         }         //分配的处理线程采用取余数的方法,获得线程数据指针         pThreadData = g_thread_data + incomesock % g_work_threads;          strcpy(pTask->client_ip, szClientIp);         //注册客户端socket读事件函数         event_set(&pTask->ev_read, incomesock, EV_READ, client_sock_read, pTask);         if (event_base_set(pThreadData->ev_base, &pTask->ev_read) != 0)         {             task_finish_clean_up(pTask);             close(incomesock);               logError("file: "__FILE__", line: %d, " /                 "event_base_set fail.", __LINE__);             continue;         }         //注册客户端socket写事件函数         event_set(&pTask->ev_write, incomesock, EV_WRITE, client_sock_write, pTask);         if ((result=event_base_set(pThreadData->ev_base, /                 &pTask->ev_write)) != 0)         {             task_finish_clean_up(pTask);             close(incomesock);               logError("file: "__FILE__", line: %d, " /                     "event_base_set fail.", __LINE__);             continue;         }         if (event_add(&pTask->ev_read, &g_network_tv) != 0)         {             task_finish_clean_up(pTask);             close(incomesock);             logError("file: "__FILE__", line: %d, " /                 "event_add fail.", __LINE__);             continue;         }     } }   int send_add_event(struct fast_task_info *pTask) {     pTask->offset = 0;       /* direct send */     client_sock_write(pTask->ev_write.ev_fd, EV_WRITE, pTask);       return 0; }   //跟踪服务器度客户端操作 static void client_sock_read(int sock, short event, void *arg) {     int bytes;     int recv_bytes;     struct fast_task_info *pTask;     pTask = (struct fast_task_info *)arg;  //pTask作为回调函数的参数     if (event == EV_TIMEOUT)     {         if (pTask->offset == 0 && pTask->req_count > 0)         {             if (event_add(&pTask->ev_read, &g_network_tv) != 0)             {                 close(pTask->ev_read.ev_fd);                 task_finish_clean_up(pTask);                   logError("file: "__FILE__", line: %d, " /                         "event_add fail.", __LINE__);             }         }         else         {             logError("file: "__FILE__", line: %d, " /                 "client ip: %s, recv timeout, " /                 "recv offset: %d, expect length: %d", /                 __LINE__, pTask->client_ip, /                 pTask->offset, pTask->length);               close(pTask->ev_read.ev_fd);             task_finish_clean_up(pTask);         }           return;     }     while (1)     {         //开始读的时候数据长度是,pTask->offset=0,这个时候接收包头         if (pTask->length == 0) //recv header         {             recv_bytes = sizeof(TrackerHeader) - pTask->offset;         }         else         {             //尝试一次性接收剩下的包             recv_bytes = pTask->length - pTask->offset;          }         //然后又尝试一次性在收完接下的包         bytes = recv(sock, pTask->data + pTask->offset, recv_bytes, 0);         if (bytes < 0)         {             if (errno == EAGAIN || errno == EWOULDBLOCK)             {                 if(event_add(&pTask->ev_read, &g_network_tv)!=0)                 {                     close(pTask->ev_read.ev_fd);                     task_finish_clean_up(pTask);                       logError("file: "__FILE__", line: %d, "/                         "event_add fail.", __LINE__);                 }             }             else             {                 logError("file: "__FILE__", line: %d, " /                     "client ip: %s, recv failed, " /                     "errno: %d, error info: %s", /                     __LINE__, pTask->client_ip, /                     errno, strerror(errno));                   close(pTask->ev_read.ev_fd);                 task_finish_clean_up(pTask);             }               return;         }         else if (bytes == 0)         {             logDebug("file: "__FILE__", line: %d, " /                 "client ip: %s, recv failed, " /                 "connection disconnected.", /                 __LINE__, pTask->client_ip);               close(pTask->ev_read.ev_fd);             task_finish_clean_up(pTask);             return;         }         if (pTask->length == 0) //header         {             //接收包头时pTask->offset=0,bytes=包头字节大小             if (pTask->offset + bytes < sizeof(TrackerHeader))             {                 if (event_add(&pTask->ev_read, &g_network_tv)!=0)                 {                     close(pTask->ev_read.ev_fd);                     task_finish_clean_up(pTask);                       logError("file: "__FILE__", line: %d, "/                         "event_add fail.", __LINE__);                 }                 pTask->offset += bytes;                 return;             }             //在这里通过第一次读包头,得到数据包的长度             pTask->length = buff2long(((TrackerHeader *) pTask->data)->pkg_len);             if (pTask->length < 0)             {                 logError("file: "__FILE__", line: %d, " /                     "client ip: %s, pkg length: %d < 0", /                     __LINE__, pTask->client_ip, /                     pTask->length);                   close(pTask->ev_read.ev_fd);                 task_finish_clean_up(pTask);                 return;             }             //整个数据包的长度=包头+包体             pTask->length += sizeof(TrackerHeader);             //数据包长度不能超过K?             if (pTask->length > TRACKER_MAX_PACKAGE_SIZE)             {                 logError("file: "__FILE__", line: %d, " /                     "client ip: %s, pkg length: %d > " /                     "max pkg size: %d", __LINE__, /                     pTask->client_ip, pTask->length, /                     TRACKER_MAX_PACKAGE_SIZE);                   close(pTask->ev_read.ev_fd);                 task_finish_clean_up(pTask);                 return;             }         }         pTask->offset += bytes;         if (pTask->offset >= pTask->length) //接收请求完成         {             pTask->req_count++;             tracker_deal_task(pTask); //跟踪服务器处理任务             return;         }     }       return; }   //跟踪服务器写socket信息到客户端 static void client_sock_write(int sock, short event, void *arg) {     int bytes;     int result;     struct fast_task_info *pTask;     pTask = (struct fast_task_info *)arg;     if (event == EV_TIMEOUT)     {         logError("file: "__FILE__", line: %d, " /             "send timeout", __LINE__);           close(pTask->ev_write.ev_fd);         task_finish_clean_up(pTask);           return;     }     while (1)     {         bytes = send(sock, pTask->data + pTask->offset, /                 pTask->length - pTask->offset,  0);         //printf("%08X sended %d bytes/n", (int)pTask, bytes);         if (bytes < 0)         {             if (errno == EAGAIN || errno == EWOULDBLOCK)             {                 if (event_add(&pTask->ev_write, &g_network_tv) != 0)                 {                     close(pTask->ev_write.ev_fd);                     task_finish_clean_up(pTask);                       logError("file: "__FILE__", line: %d, " /                         "event_add fail.", __LINE__);                 }             }             else             {                 logError("file: "__FILE__", line: %d, " /                     "client ip: %s, recv failed, " /                     "errno: %d, error info: %s", /                     __LINE__, pTask->client_ip, /                     errno, strerror(errno));                   close(pTask->ev_write.ev_fd);                 task_finish_clean_up(pTask);             }               return;         }         else if (bytes == 0)         {             logWarning("file: "__FILE__", line: %d, " /                 "send failed, connection disconnected.", /                 __LINE__);               close(pTask->ev_write.ev_fd);             task_finish_clean_up(pTask);             return;         }         pTask->offset += bytes;         //如果发送完毕,重新设置pTask->offset=0,pTask->length=0         if (pTask->offset >= pTask->length)         {             pTask->offset = 0;             pTask->length  = 0;             if ((result=event_add(&pTask->ev_read, /                         &g_network_tv)) != 0)             {                 close(pTask->ev_read.ev_fd);                 task_finish_clean_up(pTask);                 logError("file: "__FILE__", line: %d, "/                     "event_add fail.", __LINE__);                 return;             }             return;         }     } }

 (七).tracker_mem.c

//加载组信息:data_path=/ms/var/FastDFS/data static int tracker_load_groups(const char *data_path) { #define STORAGE_DATA_GROUP_FIELDS   4       FILE *fp;     char szLine[256];     char group_name[FDFS_GROUP_NAME_MAX_LEN + 1];     char *fields[STORAGE_DATA_GROUP_FIELDS];     int result;     int col_count;     TrackerClientInfo clientInfo; //用途?     bool bInserted;     //打开storage_groups.dat文件,存储分组信息     if ((fp=fopen(STORAGE_GROUPS_LIST_FILENAME, "r")) == NULL)     {         logError("file: "__FILE__", line: %d, " /             "open file /"%s/%s/" fail, " /             "errno: %d, error info: %s", /             __LINE__, data_path, STORAGE_GROUPS_LIST_FILENAME, /             errno, strerror(errno));         return errno != 0 ? errno : ENOENT;     }       result = 0;     while (fgets(szLine, sizeof(szLine), fp) != NULL)     {         if (*szLine == '/0')         {             continue;         }         //使用,进行拆分         col_count = splitEx(szLine, STORAGE_DATA_FIELD_SEPERATOR, /             fields, STORAGE_DATA_GROUP_FIELDS);         if (col_count != STORAGE_DATA_GROUP_FIELDS && /             col_count != STORAGE_DATA_GROUP_FIELDS - 2)         {             logError("file: "__FILE__", line: %d, " /                 "the format of the file /"%s/%s/" is invalid", /                 __LINE__, data_path, /                 STORAGE_GROUPS_LIST_FILENAME);             result = errno != 0 ? errno : EINVAL;             break;         }           memset(&clientInfo, 0, sizeof(TrackerClientInfo));         //组名是第一个字段名         snprintf(group_name, sizeof(group_name),/             "%s", trim(fields[0]));         //设置clientInfo对像属性值         if ((result=tracker_mem_add_group(&clientInfo, /             group_name, false, &bInserted)) != 0)         {             break;         }         //在保存的数据文件中一个组只能出现一次,否则报错         if (!bInserted)         {             logError("file: "__FILE__", line: %d, " /                 "in the file /"%s/%s/", " /                 "group /"%s/" is duplicate", /                 __LINE__, data_path, /                 STORAGE_GROUPS_LIST_FILENAME, /                 group_name);             result = errno != 0 ? errno : EEXIST;             break;         }         //该组的数据服务器端口号:这里为什么需要这个值?         clientInfo.pGroup->storage_port = atoi(trim(fields[1]));         if (col_count == STORAGE_DATA_GROUP_FIELDS - 2)         {  //version < V1.12             clientInfo.pGroup->store_path_count = 0;             clientInfo.pGroup->subdir_count_per_path = 0;         }         else         {             //store_path_count:取的是第三个字段?             clientInfo.pGroup->store_path_count = /                 atoi(trim(fields[2]));             clientInfo.pGroup->subdir_count_per_path = /                 atoi(trim(fields[3]));         }     }       fclose(fp);     return result; }   //定位数据服务器之间的同步信息 static int tracker_locate_storage_sync_server(FDFSStorageSync *pStorageSyncs, /                                               const int nStorageSyncCount, const bool bLoadFromFile) {     FDFSGroupInfo *pGroup;     FDFSStorageDetail *pStorage;     FDFSStorageSync *pSyncServer;     FDFSStorageSync *pSyncEnd;       pSyncEnd = pStorageSyncs + nStorageSyncCount;     for (pSyncServer=pStorageSyncs; pSyncServer<pSyncEnd; pSyncServer++)     {         //首先得到组         pGroup = tracker_mem_get_group(pSyncServer->group_name);         if (pGroup == NULL)         {             continue;         }         //在根据ip地址得到存储服务器信息         pStorage=tracker_mem_get_storage(pGroup, pSyncServer->ip_addr);         if (pStorage == NULL)         {             continue;         }         //主要功能是完成设置源数据同步服务器的指针         pStorage->psync_src_server = tracker_mem_get_storage(pGroup, /             pSyncServer->sync_src_ip_addr);         if (pStorage->psync_src_server == NULL)         {             char buff[MAX_PATH_SIZE+64];             if (bLoadFromFile)             {                 snprintf(buff, sizeof(buff), /                     "in the file /"%s/data/%s/", ", /                     g_fdfs_base_path, /                     STORAGE_SERVERS_LIST_FILENAME);             }             else             {                 buff[0] = '/0';             }               logError("file: "__FILE__", line: %d, " /                 "%sgroup_name: %s, storage server /"%s:%d/" " /                 "does not exist", /                 __LINE__, buff, pSyncServer->group_name, /                 pSyncServer->sync_src_ip_addr, /                 pGroup->storage_port);               return ENOENT;         }     }       return 0; }   static int tracker_load_storages(const char *data_path) { #define STORAGE_DATA_SERVER_FIELDS  20       FILE *fp;     char szLine[256];     char group_name[FDFS_GROUP_NAME_MAX_LEN + 1];     char *fields[STORAGE_DATA_SERVER_FIELDS];     char ip_addr[IP_ADDRESS_SIZE];     char *psync_src_ip_addr;     FDFSStorageSync *pStorageSyncs;     int nStorageSyncSize;     int nStorageSyncCount;     int cols;     int result;     TrackerClientInfo clientInfo;  //这个变量的用途?     bool bInserted;     //打开storage_servers.dat文件     if ((fp=fopen(STORAGE_SERVERS_LIST_FILENAME, "r")) == NULL)     {         logError("file: "__FILE__", line: %d, " /             "open file /"%s/%s/" fail, " /             "errno: %d, error info: %s", /             __LINE__, data_path, STORAGE_SERVERS_LIST_FILENAME, /             errno, strerror(errno));         return errno != 0 ? errno : ENOENT;     }       nStorageSyncSize = 0;     nStorageSyncCount = 0;     pStorageSyncs = NULL;     result = 0;     while (fgets(szLine, sizeof(szLine), fp) != NULL)     {         if (*szLine == '/0')         {             continue;         }           cols = splitEx(szLine, STORAGE_DATA_FIELD_SEPERATOR, /             fields, STORAGE_DATA_SERVER_FIELDS);         if (cols != STORAGE_DATA_SERVER_FIELDS && /             cols != STORAGE_DATA_SERVER_FIELDS - 2 && /             cols != STORAGE_DATA_SERVER_FIELDS - 3)         {             logError("file: "__FILE__", line: %d, " /                 "the format of the file /"%s/%s/" is invalid" /                 ", colums: %d != expect colums: %d or %d or %d",/                 __LINE__, data_path, /                 STORAGE_SERVERS_LIST_FILENAME, /                 cols, STORAGE_DATA_SERVER_FIELDS, /                 STORAGE_DATA_SERVER_FIELDS - 2, /                 STORAGE_DATA_SERVER_FIELDS - 3);             result = EINVAL;             break;         }         //打开数据存储服务器列表,读取所属组名,ip地址         memset(&clientInfo, 0, sizeof(TrackerClientInfo));         snprintf(group_name, sizeof(group_name), "%s", trim(fields[0]));         snprintf(ip_addr, sizeof(ip_addr), "%s", trim(fields[1]));         //首先通过组名进行查找,得到数据服务器所在组的指针         if ((clientInfo.pGroup=tracker_mem_get_group(group_name)) == NULL)         {             logError("file: "__FILE__", line: %d, " /                 "in the file /"%s/%s/", " /                 "group /"%s/" is not found", /                 __LINE__, data_path, /                 STORAGE_SERVERS_LIST_FILENAME, /                 group_name);             result = errno != 0 ? errno : ENOENT;             break;         }         //加入到存储服务器信息,clientInfo.pGroup 就是该组的信息         if ((result=tracker_mem_add_storage(&clientInfo, ip_addr, /             false, &bInserted)) != 0)         {             break;         }         //还是需要判断一下插入是否成功         if (!bInserted)         {             logError("file: "__FILE__", line: %d, " /                 "in the file /"%s/%s/", " /                 "storage /"%s/" is duplicate", /                 __LINE__, data_path, /                 STORAGE_SERVERS_LIST_FILENAME, ip_addr);             result = errno != 0 ? errno : EEXIST;             break;         }         //状态         clientInfo.pStorage->status = atoi(trim_left(fields[2]));         if (!((clientInfo.pStorage->status == /             FDFS_STORAGE_STATUS_WAIT_SYNC) || /             (clientInfo.pStorage->status == /             FDFS_STORAGE_STATUS_SYNCING) || /             (clientInfo.pStorage->status == /             FDFS_STORAGE_STATUS_INIT)))         {             clientInfo.pStorage->status = /                 FDFS_STORAGE_STATUS_OFFLINE;         }         //向该数据服务器storage server同步已有数据文件的源服务器         psync_src_ip_addr = trim(fields[3]);         //将数据服务器加入到所在组后得到了该数据服务器的指针: pClientInfo->pStorage = pStorageServer;         //同步已有数据文件的截至时间（UNIX时间戳）         clientInfo.pStorage->sync_until_timestamp = atoi( /             trim_left(fields[4]));         //上传文件次数         clientInfo.pStorage->stat.total_upload_count = strtoll( /             trim_left(fields[5]), NULL, 10);         //成功上传文件次数         clientInfo.pStorage->stat.success_upload_count = strtoll( /             trim_left(fields[6]), NULL, 10);         //更改meta data次数         clientInfo.pStorage->stat.total_set_meta_count = strtoll( /             trim_left(fields[7]), NULL, 10);         //成功更改meta data次数         clientInfo.pStorage->stat.success_set_meta_count = strtoll( /             trim_left(fields[8]), NULL, 10);         //删除文件次数         clientInfo.pStorage->stat.total_delete_count = strtoll( /             trim_left(fields[9]), NULL, 10);         //成功删除文件次数         clientInfo.pStorage->stat.success_delete_count = strtoll( /             trim_left(fields[10]), NULL, 10);         //下载文件次数         clientInfo.pStorage->stat.total_download_count = strtoll( /             trim_left(fields[11]), NULL, 10);         //成功下载文件次数         clientInfo.pStorage->stat.success_download_count = strtoll( /             trim_left(fields[12]), NULL, 10);         //获取meta data次数         clientInfo.pStorage->stat.total_get_meta_count = strtoll( /             trim_left(fields[13]), NULL, 10);         //成功获取meta data次数         clientInfo.pStorage->stat.success_get_meta_count = strtoll( /             trim_left(fields[14]), NULL, 10);         //最近一次源头更新时间（更新操作来自客户端）         clientInfo.pStorage->stat.last_source_update = atoi( /             trim_left(fields[15]));         //最近一次同步更新时间（更新操作来自其他storage server的同步）         clientInfo.pStorage->stat.last_sync_update = atoi( /             trim_left(fields[16]));         if (cols > STORAGE_DATA_SERVER_FIELDS - 3)         {             clientInfo.pStorage->changelog_offset = strtoll( /                 trim_left(fields[17]), NULL, 10);             if (clientInfo.pStorage->changelog_offset < 0)             {                 clientInfo.pStorage->changelog_offset = 0;             }             if (clientInfo.pStorage->changelog_offset > /                 g_changelog_fsize)             {                 clientInfo.pStorage->changelog_offset = /                     g_changelog_fsize;             }                 if (cols > STORAGE_DATA_SERVER_FIELDS - 2)             {                 clientInfo.pStorage->storage_port = /                     atoi(trim_left(fields[18]));                 clientInfo.pStorage->storage_http_port = /                     atoi(trim_left(fields[19]));             }             else             {                 clientInfo.pStorage->storage_port = 0;                 clientInfo.pStorage->storage_http_port = 0;             }         }         else         {             clientInfo.pStorage->changelog_offset = 0;             clientInfo.pStorage->storage_port = 0;             clientInfo.pStorage->storage_http_port = 0;         }           if (*psync_src_ip_addr == '/0')         {             continue;         }         //第一次循环时nStorageSyncSize=nStorageSyncCount=0         if (nStorageSyncSize <= nStorageSyncCount)         {             nStorageSyncSize += 8;             //在次进行了分配             pStorageSyncs = (FDFSStorageSync *)realloc( /                 pStorageSyncs, /                 sizeof(FDFSStorageSync) * nStorageSyncSize);             if (pStorageSyncs == NULL)             {                 result = errno != 0 ? errno : ENOMEM;                 logError("file: "__FILE__", line: %d, " /                     "realloc %d bytes fail", __LINE__, /                     (int)sizeof(FDFSStorageSync) * /                     nStorageSyncSize);                 break;             }         }         //设置同步:从数据服务器====>数据服务器的信息         strcpy(pStorageSyncs[nStorageSyncCount].group_name, /             clientInfo.pGroup->group_name);         strcpy(pStorageSyncs[nStorageSyncCount].ip_addr, /             clientInfo.pStorage->ip_addr);         snprintf(pStorageSyncs[nStorageSyncCount].sync_src_ip_addr, /             IP_ADDRESS_SIZE, "%s", psync_src_ip_addr);  //向该storage server同步已有数据文件的源服务器         nStorageSyncCount++;     }     fclose(fp);     if (pStorageSyncs == NULL)     {         return result;     }       if (result != 0)     {         free(pStorageSyncs);         return result;     }     result = tracker_locate_storage_sync_server(pStorageSyncs, /         nStorageSyncCount, true);     free(pStorageSyncs);     return result; } int tracker_mem_init() {     int result;     FDFSGroupInfo **ppGroup;     FDFSGroupInfo **ppGroupEnd;     /*初始化线程锁,文件锁*/     if ((result=init_pthread_lock(&mem_thread_lock)) != 0)     {         return result;     }       if ((result=init_pthread_lock(&mem_file_lock)) != 0)     {         return result;     }       if ((result=tracker_open_changlog_file()) != 0)     {         return result;     }       g_groups.alloc_size = TRACKER_MEM_ALLOC_ONCE;  //值是     g_groups.count = 0;  //初始时组的个数设置为     g_groups.current_write_group = 0;  //长传文件时当前组的索引设置为     g_groups.pStoreGroup = NULL;       //上传文件时的组的指针设置为空     //初始设置分配组的个数(默认值是)     g_groups.groups = (FDFSGroupInfo **)malloc( /         sizeof(FDFSGroupInfo *) * g_groups.alloc_size);     if (g_groups.groups == NULL)     {         logCrit("file: "__FILE__", line: %d, " /             "malloc %d bytes fail, program exit!", __LINE__, /             (int)sizeof(FDFSGroupInfo *) * g_groups.alloc_size);         return errno != 0 ? errno : ENOMEM;     }       memset(g_groups.groups, 0, /         sizeof(FDFSGroupInfo *) * g_groups.alloc_size);       ppGroupEnd = g_groups.groups + g_groups.alloc_size; //首指针+分配个数     for (ppGroup=g_groups.groups; ppGroup<ppGroupEnd; ppGroup++)     {         //在依次进行分配         *ppGroup = (FDFSGroupInfo *)malloc(sizeof(FDFSGroupInfo));         if (*ppGroup == NULL)         {             logCrit("file: "__FILE__", line: %d, " /                 "malloc %d bytes fail, program exit!", /                 __LINE__, (int)sizeof(FDFSGroupInfo));             return errno != 0 ? errno : ENOMEM;         }         //清空指向的地址内容         memset(*ppGroup, 0, sizeof(FDFSGroupInfo));     }     //通过组名进行排序     g_groups.sorted_groups = (FDFSGroupInfo **) /         malloc(sizeof(FDFSGroupInfo *) * g_groups.alloc_size);     if (g_groups.sorted_groups == NULL)     {         free(g_groups.groups);         g_groups.groups = NULL;           logCrit("file: "__FILE__", line: %d, " /             "malloc %d bytes fail, program exit!", __LINE__, /             (int)sizeof(FDFSGroupInfo *) * g_groups.alloc_size);         return errno != 0 ? errno : ENOMEM;     }       memset(g_groups.sorted_groups, 0, /         sizeof(FDFSGroupInfo *) * g_groups.alloc_size);     //启动时,从数据文件中加载数据     if ((result=tracker_load_data()) != 0)     {         return result;     }       return 0; }   static int tracker_mem_init_group(FDFSGroupInfo *pGroup) {     FDFSStorageDetail **ppServer;     FDFSStorageDetail **ppServerEnd;     int err_no;     //初始化组信息:同样就行了初始设置,比如说一个组初始设置包含有多少个数据服务器     pGroup->alloc_size = TRACKER_MEM_ALLOC_ONCE;     pGroup->count = 0; //初始设置数据服务器个数为     //初始化活动的数据服务器指针数组     pGroup->all_servers = (FDFSStorageDetail **) /         malloc(sizeof(FDFSStorageDetail *) * pGroup->alloc_size);     if (pGroup->all_servers == NULL)     {         logError("file: "__FILE__", line: %d, " /             "malloc %d bytes fail", __LINE__, /             (int)sizeof(FDFSStorageDetail *) * pGroup->alloc_size);         return errno != 0 ? errno : ENOMEM;     }       memset(pGroup->all_servers, 0, /         sizeof(FDFSStorageDetail *) * pGroup->alloc_size);     ppServerEnd = pGroup->all_servers + pGroup->alloc_size;     for (ppServer=pGroup->all_servers; ppServer<ppServerEnd; ppServer++)     {         *ppServer = (FDFSStorageDetail *)malloc( /             sizeof(FDFSStorageDetail));         if (*ppServer == NULL)         {             tracker_mem_free_group(pGroup);               logError("file: "__FILE__", line: %d, " /                 "malloc %d bytes fail", __LINE__, /                 (int)sizeof(FDFSStorageDetail));             return errno != 0 ? errno : ENOMEM;         }         memset(*ppServer, 0, sizeof(FDFSStorageDetail));     }     //同样创建数据服务器依据ip地址进行的排序     pGroup->sorted_servers = (FDFSStorageDetail **) /         malloc(sizeof(FDFSStorageDetail *) * pGroup->alloc_size);     if (pGroup->sorted_servers == NULL)     {         tracker_mem_free_group(pGroup);           logError("file: "__FILE__", line: %d, " /             "malloc %d bytes fail", __LINE__, /             (int)sizeof(FDFSStorageDetail *) * pGroup->alloc_size);         return errno != 0 ? errno : ENOMEM;     }     memset(pGroup->sorted_servers, 0, /         sizeof(FDFSStorageDetail *) * pGroup->alloc_size);     //初始设置活动的数据服务器指针数组     pGroup->active_servers = (FDFSStorageDetail **) /         malloc(sizeof(FDFSStorageDetail *) * pGroup->alloc_size);     if (pGroup->active_servers == NULL)     {         tracker_mem_free_group(pGroup);           logError("file: "__FILE__", line: %d, " /             "malloc %d bytes fail", __LINE__, /             (int)sizeof(FDFSStorageDetail *) * pGroup->alloc_size);         return errno != 0 ? errno : ENOMEM;     }     memset(pGroup->active_servers, 0, /         sizeof(FDFSStorageDetail *) * pGroup->alloc_size);   #ifdef WITH_HTTPD     if (g_http_check_interval <= 0)     {         pGroup->http_servers = pGroup->active_servers;     }     else     {         pGroup->http_servers = (FDFSStorageDetail **) /             malloc(sizeof(FDFSStorageDetail *)*pGroup->alloc_size);         if (pGroup->http_servers == NULL)         {             tracker_mem_free_group(pGroup);               logError("file: "__FILE__", line: %d, " /                 "malloc %d bytes fail", __LINE__, /                 (int)sizeof(FDFSStorageDetail *) * /                 pGroup->alloc_size);             return errno != 0 ? errno : ENOMEM;         }         memset(pGroup->http_servers, 0, /             sizeof(FDFSStorageDetail *) * pGroup->alloc_size);         g_http_servers_dirty = true;     } #endif     //初始化最后同步时间     pGroup->last_sync_timestamps = tracker_malloc_last_sync_timestamps( /         pGroup->alloc_size, &err_no);     return err_no; }   static void tracker_mem_insert_into_sorted_groups( /                                                   FDFSGroupInfo *pTargetGroup) {     FDFSGroupInfo **ppGroup;     FDFSGroupInfo **ppEnd;     //初始时g_groups.sorted_groups已经分配了两个指针,而g_groups.count=0     //所以ppEnd=g_groups.sorted_groups,因此循环做不了     //实际上就是将指针排序进去,通过倒叙的方法     ppEnd = g_groups.sorted_groups + g_groups.count;      for (ppGroup=ppEnd; ppGroup > g_groups.sorted_groups; ppGroup--)     {         if (strcmp(pTargetGroup->group_name, /             (*(ppGroup-1))->group_name) > 0)         {             //通过比较组名来确定排序             *ppGroup = pTargetGroup;             return;         }         else         {             *ppGroup = *(ppGroup-1);         }     }     //初始时因为ppGroup指向了g_groups.sorted_groups指针的头部,这里该头部指针指向新生成的pTargetGroup指针     *ppGroup = pTargetGroup; }   int tracker_mem_add_group(TrackerClientInfo *pClientInfo, /                           const char *group_name, const bool bNeedSleep, bool *bInserted) {     FDFSGroupInfo *pGroup;     int result;       if ((result=pthread_mutex_lock(&mem_thread_lock)) != 0)     {         logError("file: "__FILE__", line: %d, " /             "call pthread_mutex_lock fail, " /             "errno: %d, error info: %s", /             __LINE__, result, strerror(result));         return result;     }       do     {           result = 0;         *bInserted = false;         //首先通过组名进行查找:从g_groups中的组排序双重指针进行查找         pGroup = tracker_mem_get_group(group_name);         //如果能够查找得到,退出循环         if (pGroup != NULL)         {             break;         }         //如果大于设定的值,需要重新进行分配         if (g_groups.count >= g_groups.alloc_size)         {             result = tracker_mem_realloc_groups(bNeedSleep);             if (result != 0)             {                 break;             }         }         //指向尾指针:初始时g_groups.count=0,g_groups.groups 分配了两个         //如果是第一次的话,就相当于是取了第一个指针,在这里对它进行初始化赋值         pGroup = *(g_groups.groups + g_groups.count);         result = tracker_mem_init_group(pGroup);         if (result != 0)         {             break;         }         strcpy(pGroup->group_name, group_name);         //插入到依据组名排序的双重指针:就相当于是指针数组         tracker_mem_insert_into_sorted_groups(pGroup);         g_groups.count++;  //组的个数+1           if ((g_groups.store_lookup == /             FDFS_STORE_LOOKUP_SPEC_GROUP) && /             (g_groups.pStoreGroup == NULL) && /             (strcmp(g_groups.store_group, /             pGroup->group_name) == 0))  //也比较了上传文件时的组的名称         {             g_groups.pStoreGroup = pGroup;  //指向了上传文件时的组         }         *bInserted = true;     } while (0);       if (pthread_mutex_unlock(&mem_thread_lock) != 0)     {         logError("file: "__FILE__", line: %d, "   /             "call pthread_mutex_unlock fail", /             __LINE__);     }       if (result != 0)     {         return result;     }     //进行赋值     pClientInfo->pGroup = pGroup;     return 0; }   //删除存储服务器 int tracker_mem_delete_storage(FDFSGroupInfo *pGroup, const char *ip_addr) {     FDFSStorageDetail *pStorageServer;     FDFSStorageDetail **ppServer;     FDFSStorageDetail **ppEnd;       pStorageServer = tracker_mem_get_storage(pGroup, ip_addr);     if (pStorageServer == NULL || pStorageServer->status == /         FDFS_STORAGE_STATUS_IP_CHANGED)     {         return ENOENT;     }       if (pStorageServer->status == FDFS_STORAGE_STATUS_ONLINE || /         pStorageServer->status == FDFS_STORAGE_STATUS_ACTIVE)     {         return EBUSY;  //如果是忙的话,直接返回     }       if (pStorageServer->status == FDFS_STORAGE_STATUS_DELETED)     {         return EALREADY;     }       ppEnd = pGroup->all_servers + pGroup->count;     for (ppServer=pGroup->all_servers; ppServer<ppEnd; ppServer++)     {         if ((*ppServer)->psync_src_server != NULL && /             strcmp((*ppServer)->psync_src_server->ip_addr, ip_addr) == 0)         {             (*ppServer)->psync_src_server = NULL;  //在设置删除的存储服务器同步的指针为空         }     }     //删除该数据存储服务器     tracker_mem_clear_storage_fields(pStorageServer);     //可见删除操作是在改变数据服务器的状态     pStorageServer->status = FDFS_STORAGE_STATUS_DELETED;     pGroup->chg_count++;  //当前组改变的个数加     //写日志     tracker_write_to_changelog(pGroup, pStorageServer, NULL);     return 0; }   //删除存储服务器 int tracker_mem_delete_storage(FDFSGroupInfo *pGroup, const char *ip_addr) {     FDFSStorageDetail *pStorageServer;     FDFSStorageDetail **ppServer;     FDFSStorageDetail **ppEnd;       pStorageServer = tracker_mem_get_storage(pGroup, ip_addr);     if (pStorageServer == NULL || pStorageServer->status == /         FDFS_STORAGE_STATUS_IP_CHANGED)     {         return ENOENT;     }       if (pStorageServer->status == FDFS_STORAGE_STATUS_ONLINE || /         pStorageServer->status == FDFS_STORAGE_STATUS_ACTIVE)     {         return EBUSY;  //如果是忙的话,直接返回     }       if (pStorageServer->status == FDFS_STORAGE_STATUS_DELETED)     {         return EALREADY;     }       ppEnd = pGroup->all_servers + pGroup->count;     for (ppServer=pGroup->all_servers; ppServer<ppEnd; ppServer++)     {         if ((*ppServer)->psync_src_server != NULL && /             strcmp((*ppServer)->psync_src_server->ip_addr, ip_addr) == 0)         {             (*ppServer)->psync_src_server = NULL;  //在设置删除的存储服务器同步的指针为空         }     }     //删除该数据存储服务器     tracker_mem_clear_storage_fields(pStorageServer);     //可见删除操作是在改变数据服务器的状态     pStorageServer->status = FDFS_STORAGE_STATUS_DELETED;     pGroup->chg_count++;  //当前组改变的个数加     //写日志     tracker_write_to_changelog(pGroup, pStorageServer, NULL);     return 0; }