kafka客户端类
我的kafka客户端测试类
- 客户端
- 头文件"kafka_control.h"
- 源文件"kafka_control.cpp"
- 例子：
一个项目中实际用到的跨平台的kafka客户端类，vs2013和sentos7实际测试可用
客户端
封装几个简单的函数，给自己留备份，仅供参考。
是针对kafka集群的客户端，而且根据id，做哈希计算后，写入对应的kafka分区，本例子中kafka分区默认为5，复制粘贴的同学自己修改代码。
头文件"kafka_control.h"
#ifndef KAFKA_CONTROL_H
#define KAFKA_CONTROL_H
#ifdef _WIN32
#include ".\librdkafka\include\rdkafka.h"
#else
#include "/home/xby/librdkafka-master/src/rdkafka.h"
#endif
#include 源文件"kafka_control.cpp"
// An highlighted block
#ifdef _WIN64
#pragma comment(lib, ".\\librdkafka\\dll\\x64\\librdkafka.lib")
#else
#pragma comment(lib, ".\\librdkafka\\dll\\x86\\librdkafka.lib")
#endif
#pragma comment(lib,"wsock32.lib")
#include "kafka_control.h"
void kafka_control::dr_msg_cb(rd_kafka_t *rk, const rd_kafka_message_t *rkmessage, void *opaque)
{
if (rkmessage->err)
fprintf(stderr, "%% Message delivery failed: %s\n",
rd_kafka_err2str(rkmessage->err));
else
{
/*fprintf(stderr,
"%% Message delivered (%zd bytes, "
"partition %"PRId32")\n",
rkmessage->len, rkmessage->partition);*/
/* rkmessage被librdkafka自动销毁*/
//printf("send kafka successed ,partition is %d,data len is %d. \r\n", rkmessage->partition, rkmessage->len);
}
}
int kafka_control::InitBrokers(const char *topic, const char *pcBrokers)
{
if (topic == NULL || pcBrokers == NULL){
printf("config info have an error :topic name is '%s' ,brokers is '%s'", topic, pcBrokers);
return -1;
}
#ifdef _WIN32
WORD wVerisonRequested;
WSADATA wsaData;
int err;
wVerisonRequested = MAKEWORD(1, 1);
err = WSAStartup(wVerisonRequested, &wsaData);
if (err != 0){
return -1;
}
#endif
//rd_kafka_t *rk; /*Producer instance handle*/
//rd_kafka_topic_t *rkt; /*topic对象*/
//rd_kafka_conf_t *conf; /*临时配置对象*/
char errstr[512];
char brokers[512];
//const char *topic;
//CMS数据接入测试环境 TOPIC
//原始数据： --partitions 5 --topic cmsdata_test
// 指标数据： --partitions 1 --topic cmsmetrics_test
// 预警数据： --partitions 1 --topic cmsalarm_test
// CMS数据接入生产环境 TOPIC
// 原始数据： --partitions 5 --topic cmsdata
// 指标数据： --partitions 1 --topic cmsmetrics
// 预警数据： --partitions 1 --topic cmsalarm
//192.168.190.152:9092，192.168.190.152 : 9092，192.168.190.152 : 9092
"192.168.190.152:9092,192.168.190.153:9092,192.168.190.154:9092";
//brokers = "192.168.190.152:9092,192.168.190.153:9092,192.168.190.154:9092";
strcpy(brokers,pcBrokers);
//brokers = "hadoop2:9092,hadoop3:9092,hadoop4:9092";
// topic = "scada_test";
//const char *topic = "cmsalarm_test";//cmsdata_test //cmsmetrics_test
/* 创建一个kafka配置占位 */
conf = rd_kafka_conf_new();
/*创建broker集群*/
if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr,
sizeof(errstr)) != RD_KAFKA_CONF_OK){
fprintf(stderr, "%s\n", errstr);
return 1;
}
/*设置发送报告回调函数，rd_kafka_produce()接收的每条消息都会调用一次该回调函数
*应用程序需要定期调用rd_kafka_poll()来服务排队的发送报告回调函数*/
rd_kafka_conf_set_dr_msg_cb(conf, this->dr_msg_cb);
/*创建producer实例
rd_kafka_new()获取conf对象的所有权,应用程序在此调用之后不得再次引用它*/
rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr));
if (!rk){
fprintf(stderr, " Failed to create new producer:%s\r\n", errstr);
return 1;
}
/*实例化一个或多个topics(`rd_kafka_topic_t`)来提供生产或消费，topic
对象保存topic特定的配置，并在内部填充所有可用分区和leader brokers，*/
rkt = rd_kafka_topic_new(rk, topic, NULL);
if (!rkt)
{
fprintf(stderr, "%% Failed to create topic object: %s\n",
rd_kafka_err2str(rd_kafka_last_error()));
rd_kafka_destroy(rk);
return 1;
}
return 0;
}
int kafka_control::SendAlarmData(const char *pcData)
{
this->SendData(NULL, pcData, 0);
return 0;
}
int kafka_control::SendCharacterData(const char *pcData)
{
this->SendData(NULL, pcData, 0);
return 0;
}
int kafka_control::SendWaveFormData(const char* pcWtgHadoopId, const char *pcData)
{
this->SendData(pcWtgHadoopId, pcData, 5);
return 0;
}
int kafka_control::SendWtgScadaData(const char* pcWtgHadoopId, const char *pcData)
{
this->SendData(pcWtgHadoopId, pcData, 5);
return 0;
}
int kafka_control::SendData(const char* pcWtgHadoopId, const char *pcData, int nPartition)
{
const char* pcTopic = rd_kafka_topic_name(this->rkt);
int nPartition1 = 0;
if (pcWtgHadoopId == NULL){
nPartition1 = 0;
}
else{
if (nPartition <= 0){
printf("kafka nPartition have error\r\n");
}
size_t nHashValue = this->my_Hash(pcWtgHadoopId);
nPartition1 = nHashValue % nPartition;
}
int len = (int)strlen(pcData);
if (-1 == rd_kafka_produce(
/* Topic object */
rkt,
/*使用内置的分区来选择分区*/
nPartition1,
//1,
/*生成payload的副本*/
RD_KAFKA_MSG_F_COPY,
/*消息体和长度*/
(void*)pcData, len,
/*可选键及其长度*/
NULL, 0,
NULL))
{
fprintf(stderr,
"%% Failed to produce to topic %s: %s\n",
rd_kafka_topic_name(rkt),
rd_kafka_err2str(rd_kafka_last_error()));
if (rd_kafka_last_error() == RD_KAFKA_RESP_ERR__QUEUE_FULL){
/*如果内部队列满，等待消息传输完成并retry,
内部队列表示要发送的消息和已发送或失败的消息，
内部队列受限于queue.buffering.max.messages配置项*/
rd_kafka_poll(rk, 1000);
}
}
else{
//fprintf(stderr, "%% Enqueued message (%zd bytes) for topic %s\n",
//len, rd_kafka_topic_name(rkt));
}
/*producer应用程序应不断地通过以频繁的间隔调用rd_kafka_poll()来为
传送报告队列提供服务。在没有生成消息以确定先前生成的消息已发送了其
发送报告回调函数(和其他注册过的回调函数)期间，要确保rd_kafka_poll()
仍然被调用*/
rd_kafka_poll(rk, 0);
return 0;
}
size_t kafka_control::my_Hash(const char* pcPara)
{
/*std::hash h1;
size_t nR = h1(pcPara);
return nR;*/
int i, l;
unsigned long ret = 0;
unsigned short *s;
if (pcPara == NULL) return(0);
l = (strlen(pcPara) + 1) / 2;
s = (unsigned short *)pcPara;
for (i = 0; i<l; i++)
ret ^= (s[i] << (i & 0x0f));
return(ret);
}
例子：
// An highlighted block
//kafka
kafka_control* pkafka_class = new kafka_control();
printf("");
#ifdef CONVERTER
pkafka_class->InitBrokers("cvtdata","192.168.190.152:9092,192.168.190.153:9092,192.168.190.154:9092");
pkafka_class->SendWtgScadaData("HadoopId", "data");
部分内容参考了下面文章：
他的更简练，我的符合自己的需求
https://blog.csdn.net/lijinqi1987/article/details/76582067