C语言 librdkafka 简单分析生产、消费的实例

生产

#include 
#include 
#include 
#include 

static volatile sig_atomic_t run = 1;

/**
 * @brief Signal termination of program
 */
static void stop (int sig) {
    run = 0;
    fclose(stdin); /* abort fgets() */
}

static void dr_msg_cb (rd_kafka_t *rk,
                       const rd_kafka_message_t *rkmessage, void *opaque) {
    if (rkmessage->err)
        fprintf(stderr, "%% Message delivery failed: %s\n",
                rd_kafka_err2str(rkmessage->err));
    else
        fprintf(stderr,
                "%% Message delivered (%zd bytes, "
                "partition %"PRId32")\n",
                rkmessage->len, rkmessage->partition);
    /* The rkmessage is destroyed automatically by librdkafka */
}



int main (int argc, char **argv) {
    rd_kafka_t *rk;         /* kafka客户端对象 */
    rd_kafka_conf_t *conf;  /* 临时配置对象 */
    char errstr[512];       /* API错误报告缓冲区 */
    char buf[512];          /* 消息值临时缓冲区 */
    const char *brokers;    /* broker代理列表 */
    const char *topic;      /* 主题 */
    
    //brokers = argv[1];
    //topic   = argv[2];
    brokers = "192.168.205.128";
    topic   = "test_topic";
    //初始化过程
    conf = rd_kafka_conf_new();//创建rd_kafka_conf_t对象
    //设置一个配置属性,conf必须是以前用rd_kafka_conf_new()创建的。
    //这里就是对broker去进行配置
    if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers,
                          errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
       fprintf(stderr, "%s\n", errstr);
       return 1;
    }
    //设置回调函数dr_msg_cb，rd_kafka_produce()接收的每条消息都会调用一次该回调函数
    rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb);
    //创建一个新的Kafka句柄,即创建producer实例
    rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr));
    if (!rk) {
            fprintf(stderr,
                    "%% Failed to create new producer: %s\n", errstr);
            return 1;
    }
    
    /* 信号捕捉 */
    signal(SIGINT, stop);
    
    fprintf(stderr,"%% Press Ctrl-C or Ctrl-D to exit\n");
    
    while (run && fgets(buf, sizeof(buf), stdin)) {
        size_t len = strlen(buf);
        rd_kafka_resp_err_t err;
    
        if (buf[len-1] == '\n') /* Remove newline */
                buf[--len] = '\0';
    
        if (len == 0) {
                /* Empty line: only serve delivery reports */
                rd_kafka_poll(rk, 0/*non-blocking */);//轮询提供的kafka事件句柄。0表示非阻塞
                //continue;
        }
    
    retry:
        err = rd_kafka_producev(//异步调用将消息发送到指定的topic
                    /* Producer handle */
                    rk,
                    /* Topic name */
                    RD_KAFKA_V_TOPIC(topic),
                    /* Make a copy of the payload. */
                    RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY),
                    /* Message value and length */
                    RD_KAFKA_V_VALUE(buf, len),
                    /* Per-Message opaque, provided in
                     * delivery report callback as
                     * msg_opaque. */
                    RD_KAFKA_V_OPAQUE(NULL),
                    /* End sentinel */
                    RD_KAFKA_V_END);
    
        if (err) {
                    //消息发送失败
                    fprintf(stderr,
                            "%% Failed to produce to topic %s: %s\n",
                            topic, rd_kafka_err2str(err));
    
                    if (err == RD_KAFKA_RESP_ERR__QUEUE_FULL) {
                            rd_kafka_poll(rk, 1000/*block for max 1000ms*/);
                            goto retry;
                    }
        } else {
               fprintf(stderr, "%% Enqueued message (%zd bytes) "
                       "for topic %s\n",
                       len, topic);
        }
    
    
        rd_kafka_poll(rk, 0/*non-blocking*/);
    }
    
    
    /* 等待最后消息处理完毕:传递成功或失败。
     * rd_kafka_flush() 是 rd_kafka_poll() 的抽象化,以便等待消息传输完毕,*/
    fprintf(stderr, "%% Flushing final messages..\n");
    rd_kafka_flush(rk, 10*1000 /* wait for max 10 seconds ,第二个参数是等待时间*/);
    //rd_kafka_flush:等待所有未完成的请求,这个函数将调用rd_kafka_poll()，从而触发回调
    //这通常应该在销毁生产者实例之前完成，以确保在终止之前完成所有队列和正在运行的生产请求。
    
    /* 如果到这里输出队列不是空的，则会出现向集群生成消息的问题 */
    if (rd_kafka_outq_len(rk) > 0)
        fprintf(stderr, "%% %d message(s) were not delivered\n",
                rd_kafka_outq_len(rk));
    
    /* 退出前销毁生产实例 */
    rd_kafka_destroy(rk);
    
    return 0;
}

消费

#include 
#include 
#include 
#include 
#include 

static volatile sig_atomic_t run = 1;

/**
 * @brief Signal termination of program
 */
static void stop (int sig) {
        run = 0;
}

/**
 * @returns 1 if all bytes are printable, else 0.
 */
static int is_printable (const char *buf, size_t size) {
        size_t i;

        for (i = 0 ; i < size ; i++)
                if (!isprint((int)buf[i]))
                        return 0;

        return 1;
}


int main (int argc, char **argv) {
        rd_kafka_t *rk;          
        rd_kafka_conf_t *conf;   
        rd_kafka_resp_err_t err; 
        char errstr[512];        
        const char *brokers;     
        const char *groupid;     /* Consumer group id */
        char **topics;           /* 订阅的主题列表 */
        int topic_cnt;           /* 订阅的主题数量 */
        rd_kafka_topic_partition_list_t *subscription; 
        int i;


        if (argc < 4) {
                fprintf(stderr,
                        "%% Usage: "
                        "%s    ..\n",
                        argv[0]);
                return 1;
        }

        brokers   = argv[1];
        groupid   = argv[2];
        topics    = &argv[3];
        topic_cnt = argc - 3;


        conf = rd_kafka_conf_new();


        if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers,
                              errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
                fprintf(stderr, "%s\n", errstr);
                rd_kafka_conf_destroy(conf);
                return 1;
        }

        /* 设置用户组id.
         * 共享相同组id的所有使用者将加入相同的组
         * 订阅的主题分区将根据分区调配策略去进行分配*/
        if (rd_kafka_conf_set(conf, "group.id", groupid,
                              errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
                fprintf(stderr, "%s\n", errstr);
                rd_kafka_conf_destroy(conf);
                return 1;
        }

        /* 如果一个分区之前没有提交偏移量，那么自动设置偏移量。
         * reset策略将用于决定在分区的哪个位置开始获取消息。
         * 如果之前没有提交偏移量，将该值设置为early，使用者将读取分区中的所有消息 */
        if (rd_kafka_conf_set(conf, "auto.offset.reset", "earliest",
                              errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
                fprintf(stderr, "%s\n", errstr);
                rd_kafka_conf_destroy(conf);
                return 1;
        }

        /* 创建消费者实例
         * 获取conf对象的所有权
         */
        //实例化一个顶级对象rd_kafka_t作为基础容器，提供全局配置和共享状态
        rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr));
        if (!rk) {
                fprintf(stderr,
                        "%% Failed to create new consumer: %s\n", errstr);
                return 1;
        }

        conf = NULL; /* 配置对象现在由rd_kafka_t实例拥有并释放 */


        /* 重定向 rd_kafka_poll()队列到consumer_poll()队列
         * 将所有消息从每个分区队列重定向到主队列，
         * 以便在一次调用中使用所有分配的分区中的消息*/
        rd_kafka_poll_set_consumer(rk);


        /* 创建一个新的列表/向量 主题和分区容器 
         * 使用rd_kafka_topic_partition_list_destroy()释放列表和列表本身使用的所有资源。
        */
        subscription = rd_kafka_topic_partition_list_new(topic_cnt);
        for (i = 0 ; i < topic_cnt ; i++)
                rd_kafka_topic_partition_list_add(subscription,
                                                  topics[i],
                                                  /* the partition is ignored
                                                   * by subscribe() */
                                                  RD_KAFKA_PARTITION_UA);

        /* Subscribe to the list of topics */
        err = rd_kafka_subscribe(rk, subscription);
        if (err) {
                fprintf(stderr,
                        "%% Failed to subscribe to %d topics: %s\n",
                        subscription->cnt, rd_kafka_err2str(err));
                rd_kafka_topic_partition_list_destroy(subscription);
                rd_kafka_destroy(rk);
                return 1;
        }

        fprintf(stderr,
                "%% Subscribed to %d topic(s), "
                "waiting for rebalance and messages...\n",
                subscription->cnt);
        //释放列表和列表本身使用的所有资源。
        rd_kafka_topic_partition_list_destroy(subscription);


        /* Signal handler for clean shutdown */
        signal(SIGINT, stop);

        /* Subscribing to topics will trigger a group rebalance
         * which may take some time to finish, but there is no need
         * for the application to handle this idle period in a special way
         * since a rebalance may happen at any time.
         * Start polling for messages. */

        while (run) {
                rd_kafka_message_t *rkm;

                rkm = rd_kafka_consumer_poll(rk, 100);
                if (!rkm)
                        continue; /* 超时:100ms
                                   */

                /* consumer_poll()将返回一个适当的消息或错误(rkm->err)。*/
                if (rkm->err) {//打印错误
                        fprintf(stderr,
                                "%% Consumer error: %s\n",
                                rd_kafka_message_errstr(rkm));
                        rd_kafka_message_destroy(rkm);
                        continue;
                }

                /* 打印message. */
                printf("Message on %s [%"PRId32"] at offset %"PRId64":\n",
                       rd_kafka_topic_name(rkm->rkt), rkm->partition,
                       rkm->offset);

                /* Print the message key. */
                if (rkm->key && is_printable(rkm->key, rkm->key_len))
                        printf(" Key: %.*s\n",
                               (int)rkm->key_len, (const char *)rkm->key);
                else if (rkm->key)
                        printf(" Key: (%d bytes)\n", (int)rkm->key_len);

                /* Print the message value/payload. */
                if (rkm->payload && is_printable(rkm->payload, rkm->len))
                        printf(" Value: %.*s\n",
                               (int)rkm->len, (const char *)rkm->payload);
                else if (rkm->key)
                        printf(" Value: (%d bytes)\n", (int)rkm->len);

                rd_kafka_message_destroy(rkm);
        }


        /* 关闭用户:提交最终偏移量并离开组 */
        fprintf(stderr, "%% Closing consumer\n");
        rd_kafka_consumer_close(rk);


        /* 销毁consumer对象 */
        rd_kafka_destroy(rk);

        return 0;
}

编译加上： -lrdkafka -lz -lpthread -lrt