linux 下实现简单的异步多线程任务分发

linux 下实现简单的异步多线程任务分发

异步多任务并行处理在linux场景下有很多种实现方式。
当前主流有几种方案：多进程方案，1 master+N workers 方式，类似apache等等；多线程方案，或维护一个线程池。
本例子实现一个简单的单进程多线程异步分发任务模型。本实现有如下特点：
* 维护线程简单
* 直接可以传送函数，函数的参数
* 线程之间使用消息队列通信，实时性高，简单
* 可以实现简单的与主线程分离的任务
* swoole中，nodejs中的异步任务，需要在特定的io或事件场合，才可以异步，并不是随时可以异步，本例子实现随时任何情形都可以异步
受限于消息队列等因素，本例子还有未实现功能：
* 线程间没有锁机制，共享数据将不安全
* linux消息队列每个元素最大只能65535字节
* 暂时不能实现协程调度，工作线程和主线程是异步的，但是1个工作线程同时只能接受到1个任务，必须执行完才能接下一个任务。如果总共三个工作线程，每一个都阻塞，那就不能继续执行新的任务。
后续更新：
* 使用getcontext,setcontext,swapcontext来实现协程调度
* 实现有锁调用和无锁调用两种方式

code.c:

/*
 * 实现简单消息队列分发任务异步执行
 * c99 thread.c -o thread -l pthread -D _XOPEN_SOURCE
 * */
#define _XOPEN_SOURCE 1
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 

#define LOGF(STR, ...) logfmt(STR, __FILE__,__LINE__,__func__,##__VA_ARGS__)

int msgqid = 0;             // 用于主线程和worker线程通信的消息队列id，由主线程申请
int msgtype = 9812;         // 消息标识符，类型
int IPC_KEY = 100011;       // 申请消息的IPC_KEY 本应该用ftok获得

/* 消息队列消息结构 */
/* 以下消息队列中保存的指针，只在当前运行进程内有效，其他进程或者重启进程将失效 */
typedef struct message {
    long msg_type;              /* 消息标识符 */
    void* (*func)(void*);       /* 调用函数指针 在不同进程和进程重启后将失效 切记 */
    void* param;                /* 此函数的参数 */
    void** retval;              /* 此函数的返回值接收地址 发送方提供 发送方先申请好空间 */
    void* (*callback)(void* retval, void*origin_param);   /* 完成后的callback函数 参数依次是 返回值，上次调用的函数的参数 */
} message;

/* 线程任务单元 */
typedef struct threadtask {
    pthread_t tid;
    int is_working;
    int working_times;
} threadtask;
/* 小线程池 */
typedef struct taskpool {
     threadtask* pool;
     int len;
} taskpool;

taskpool* mainpool;

void logfmt(const char* str, const char* file, const int line, const char* func, ...);

/*protected 线程的main函数 task consumer */
void* taskconsumer(void*ptr) {
    threadtask* myth = (threadtask*)ptr;
    struct message msgq;
    int i = 0;
    for (i=0; ;i++) {
        msgrcv(msgqid, &msgq, sizeof(message)-sizeof(long), msgtype, 0);
        myth->is_working = 1;
        myth->working_times++;
        LOGF("threadtask(%ld) got a task: loopid=%d taskfunc=%p param=%d\n", myth->tid, i, msgq.func, (msgq.param==NULL?0:*(int*)msgq.param));

        if (msgq.func == NULL) {
            LOGF("msgq.func is NULL, ignore this msg\n");
            continue;
        }

        void*ret = msgq.func((void*)msgq.param);
        if (msgq.retval != NULL) {
            *(msgq.retval) = ret;
        }
        myth->is_working = 0;
        if (msgq.callback != NULL) {
            LOGF("this task has callback loopid=%d\n", i);
            msgq.callback((void*)ret, msgq.param);
        }
    }
}

/*public user task*/
void* task1(void* ptr) {
    LOGF("task1 start, void*=%p v=%d\n", ptr, *(int*)ptr);
    sleep(1);
    LOGF("task1 over\n");
    return NULL;
}

void* task2(void* ptr) {
    LOGF("task2 start, void*=%x v=%d\n", ptr, *(int*)ptr);
    sleep(10);
    LOGF("task2 over\n");
}

void* callback1(void* ret, void* param) {
    LOGF("i am callback: ret=%p param=%p\n", ret, param);
}

int recycletaskpool(taskpool* thepool);

/*for main 分配线程池 init workers, waiting msgqueue for task*/
taskpool* inittaskpool(int worker_num) {
    taskpool* mainpool = (taskpool*)malloc(sizeof(taskpool));
    memset(mainpool, 0, sizeof(*mainpool));
    mainpool->pool = (threadtask*)malloc(sizeof(threadtask)*worker_num);
    memset(mainpool->pool, 0, sizeof(*mainpool->pool));
    mainpool->len = worker_num;

    int has_error = 0;
    int i = 0;
    for (i=0; ipool[i].tid);
        int ret = pthread_create(tid_ptr, NULL, (void *)taskconsumer, (void*)&(mainpool->pool[i]));
        if (ret != 0) {
            perror("create thread failed:");
            has_error = 1;
            break;
        }

        pthread_detach(*tid_ptr);
    }
    if (has_error) {
        LOGF("create pthread error, go to recyclepool\n");
        recycletaskpool(mainpool);
    }
    return mainpool;
}
/*for main 回收线程池资源*/
int recycletaskpool(taskpool* thepool) {
    if (thepool != NULL) {
        if (thepool->pool != NULL) {
            free(thepool->pool);
        }
        free(thepool);
    }
    return 0;
}
/*public 异步启动一个任务*/
int async_do(void*(*func)(void*), void* param, void** ret, void* (*callback)(void*,void*)) {
    struct message msgq;
    msgq.msg_type = msgtype;
    msgq.func = func;
    msgq.param = param;
    msgq.retval = ret;
    msgq.callback = callback;
    // send msgq to queue
    int sendret = msgsnd(msgqid, &msgq, sizeof(message)-sizeof(long), 0);
    return sendret;
}

int main() {

    // 初始化队列
    msgqid = msgget(IPC_KEY, IPC_CREAT | 0666);

    // 初始化4个工作线程
    int worker_num = 4;
    mainpool = inittaskpool(worker_num);

    char s[100] = {};
    int sendret = 0;
    void* (*taskfunc) (void*) = NULL;
    void* (*thecallback) (void*,void*) = NULL;
    int dorecycle = 0;

    int i = 0;
    for (i=0; i<16; i++) {
        LOGF("input which task run: i=%d\n", i);
        scanf("%s", s);
        switch (s[0]) {
            case '1':
                taskfunc = task1;
                thecallback = NULL;
                break;
            case '2':
                taskfunc = task2;
                thecallback = NULL;
                break;
            case '3':case '4':case '5':case '6':case '7':case '8':
                taskfunc = task2;
                thecallback = callback1;
                break;
            case 'q':
                dorecycle = 1; // 退出
                break;
            default:
                continue;
                break;

        }
        if (dorecycle) {
            break;
        }
        sendret = async_do(taskfunc, &s[0], NULL, thecallback);
        //LOGF("you put v=%d when i=%d ret=%d\n", v, i, sendret);
    }

    // do recycle
    int rmmsgret = msgctl(msgqid, IPC_RMID, NULL);
    LOGF("msgqid %d have removed: ret=%d\n", msgqid, rmmsgret);
    recycletaskpool(mainpool);
    LOGF("mainpool has been recycled\n");
}
// log
void logfmt(const char* str, const char* file, const int line, const char* func, ...) {
    time_t now = time(NULL);
    struct tm* tn = localtime(&now);
    static char prestr[256] = {};
    // 先预备好前缀
    sprintf(prestr, "[%04d-%02d-%02d %02d:%02d:%02d] [%s:%d %s()] %s\x00", tn->tm_year+1900,tn->tm_mon+1,tn->tm_mday,tn->tm_hour,tn->tm_min,tn->tm_sec, file, line, func, str);
    va_list ap;
    va_start(ap, func);
    // 不能使用printf 要使用vprintf
    vprintf(prestr, ap);
    va_end(ap);
}

运行结果：

[root@iZ25gcs79rvZ thread]# ./thread 
[2017-07-13 16:54:44] [thread.c:164 main()] input which task run: i=0
1
[2017-07-13 16:54:46] [thread.c:164 main()] input which task run: i=1
[2017-07-13 16:54:46] [thread.c:57 taskconsumer()] threadtask(139760144594688) got a task: loopid=0 taskfunc=0x400bc9 param=49
[2017-07-13 16:54:46] [thread.c:78 task1()] task1 start, void*=0x7fff4f3fb0b0 v=49
2
[2017-07-13 16:54:46] [thread.c:164 main()] input which task run: i=2
[2017-07-13 16:54:46] [thread.c:57 taskconsumer()] threadtask(139760152987392) got a task: loopid=0 taskfunc=0x400c32 param=50
[2017-07-13 16:54:46] [thread.c:85 task2()] task2 start, void*=4f3fb0b0 v=50
2
[2017-07-13 16:54:47] [thread.c:164 main()] input which task run: i=3
[2017-07-13 16:54:47] [thread.c:57 taskconsumer()] threadtask(139760161380096) got a task: loopid=0 taskfunc=0x400c32 param=50
[2017-07-13 16:54:47] [thread.c:85 task2()] task2 start, void*=4f3fb0b0 v=50
2[2017-07-13 16:54:47] [thread.c:80 task1()] task1 over

[2017-07-13 16:54:47] [thread.c:164 main()] input which task run: i=4
[2017-07-13 16:54:47] [thread.c:57 taskconsumer()] threadtask(139760169772800) got a task: loopid=0 taskfunc=0x400c32 param=50
[2017-07-13 16:54:47] [thread.c:85 task2()] task2 start, void*=4f3fb0b0 v=50
2
[2017-07-13 16:54:48] [thread.c:164 main()] input which task run: i=5
[2017-07-13 16:54:48] [thread.c:57 taskconsumer()] threadtask(139760144594688) got a task: loopid=1 taskfunc=0x400c32 param=50
[2017-07-13 16:54:48] [thread.c:85 task2()] task2 start, void*=4f3fb0b0 v=50
2
[2017-07-13 16:54:48] [thread.c:164 main()] input which task run: i=6

[2017-07-13 16:54:56] [thread.c:87 task2()] task2 over
[2017-07-13 16:54:56] [thread.c:57 taskconsumer()] threadtask(139760152987392) got a task: loopid=1 taskfunc=0x400c32 param=50
[2017-07-13 16:54:56] [thread.c:85 task2()] task2 start, void*=4f3fb0b0 v=50
[2017-07-13 16:54:57] [thread.c:87 task2()] task2 over
[2017-07-13 16:54:57] [thread.c:87 task2()] task2 over
[2017-07-13 16:54:58] [thread.c:87 task2()] task2 over
q
[2017-07-13 16:55:04] [thread.c:196 main()] msgqid 917504 have removed: ret=0
[2017-07-13 16:55:04] [thread.c:198 main()] mainpool has been recycled