1、下载、解压
kafka官网下载0.10.x.x版本的kafka
tar -xvf 下载文件 -C /usr/local //-C 用于指定解压目录，不指定则为当前文件夹
2、配制
官方文档
2.1 zookeeper配制
为简单起见，此处使用kafka内置的zookeeper，其配制文件为config/zookeeper.peoperties.
其主要的配制参数为，port，默认为2181
文件可以不改动，若须精细化配制，参见官档。
2.2 kafka配制
kafka 0.10.x.x版本中，config/server.peoperties仅有三项为必须配制的。
- broker.id
唯一标识一个kakfa实例 - log.dirs
用于存储kafka数据 - zookeeper.connect
配制zookeeper时最好使用zookeeper所在主机的ip地址，若使用主机名，则使用kafka客户端的主机，必须在hosts文件是配置ip和hostname的映射。否则，应用程序会出现Exception
KafKa error java.nio.channels.UnresolvedAddressException
listeners=PLAINTEXT://ip:9093 //此项要配制成ip +端口号，否则同样会出现UnresolvedAddressException
其它选项，默认即可
单主机多Broker实例
则参见 官方文档"Step 6: Setting up a multi-broker cluster"
仅需要增加listeners，配制项，并保证每个配制文件中broker.id，log.dir 唯一即可。多主机单Broker实例
各个主机中的配制文件，除broker.id之外内容可相同。
config/server-1.properties:
broker.id=1
listeners=PLAINTEXT://:9093
log.dir=/tmp/kafka-logs-1
zookeeper.connect 同上
config/server-2.properties:
broker.id=2
listeners=PLAINTEXT://:9094
log.dir=/tmp/kafka-logs-2
zookeeper.connect 同上
3、 启动
- 启动zookeeper
bin/zookeeper-server-start.sh -daemon config/zookeeper.peoperties.
- 启动 kafka
bin/kafka-server-start.sh -daemon config/server.peoperties.
若配制了多个broker实例，则
bin/kafka-server-start.sh -daemon config/server-1.properties
bin/kafka-server-start.sh -daemon config/server-2.properties
4、测试
- 创建topic
bin/kafka-topics.sh --create --zookeeper your_host:2181 --replication-factor 1 --partitions 1 --topic topicname
- 查看是否创建成功
bin/kafka-topics.sh --list --zookeeper your_host:2181 topicname
- 开启console-producer
bin/kafka-console-producer.sh --broker-list your_host:9092 --topic topicname
- 开启 console-consumer
bin/kafka-console-consumer.sh --zookeeper your_host:2181 --topic topicname --from-beginning
在console-producer 控制台 输入内容， console-consumer可以接收，则表示配制成功。若遇到如下异常：
java.nio.BufferUnderflowException
或
org.apache.kafka.common.protocol.types.SchemaException: Error reading field 'topics':
则说明，kafka 集群中存在不同的版本，kafka集群中每个broker是对等的，只要是注册到同一zookeeper下的kafka就会协同。(因为是console模式所以可以排除client与server版本不同的可能)。
5、自定义 Producer与Consumer
此处使用新配制
5.1 Producer
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import java.util.Properties;
/**
* Created by cc on 17-1-6.
*/
public class KafkaConnector {
KafkaProducer producer;
public KafkaConnector(String brokerList) {
Properties props = new Properties();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList);
props.put(ProducerConfig.ACKS_CONFIG, "all");
props.put(ProducerConfig.CLIENT_ID_CONFIG, "producer");
props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432);
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
producer = new KafkaProducer(props);
}
/**
* Returns the actual number of sent records
*/
public void sendRecord(String topic, String key, String value) {
try{
ProducerRecord record = new ProducerRecord(topic, key, value);
producer.send(record);
} catch (Exception e) {
e.printStackTrace();
}
}
public void close() {
producer.close();
}
public static void main(String[] args) {
String brokerlist = "192.168.4.49:9092";
String topic = "topicname";
KafkaConnector connector = new KafkaConnector(brokerlist);
for(int i = 0; i < 200; i++) {
connector.sendRecord(topic, i+"", i+"");
}
connector.close();
}
}
5.2 Consumer
import java.util.*;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.TopicPartition;
import org.apache.log4j.BasicConfigurator;
/**
* Created by cc on 17-1-6.
*/
public class Consumer {
public KafkaConsumer consumer;
public Consumer(String brokerList) {
BasicConfigurator.configure(); //防止出现log4j: WARN no appender警告，也可配制解决。
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList);
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest"); //"earliest" else "latest"
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
props.put(ConsumerConfig.GROUP_ID_CONFIG, "group");
consumer = new KafkaConsumer(props);
}
public void process(List topic) {
consumer.subscribe(topic);
try{
while (true) {
ConsumerRecords records = consumer.poll(Long.MAX_VALUE);
for(TopicPartition partition: records.partitions()) {
List> partitionRecords = records.records(partition);
for(ConsumerRecord record: partitionRecords) {
System.out.println(record.value());
}
}
consumer.commitSync();;
}
}catch (Exception e) {
e.printStackTrace();
} finally {
consumer.close();
}
}
public static void main(String[] args) {
List topic = new ArrayList();
topic.add("topicname");
String brokerlist = "192.168.4.49:9092";
new Consumer(brokerlist).process(topic);
}
}
5.3 问题解决
出现“log4j:WARN No appenders could be found for logger (org.apache.kafka.clients.producer.ProducerConfig).”的解决办法：
- 编程解决
BasicConfigurator.configure(); //增加此即可解决
- 配制解决
classpath下配制log4j.properties