Kafka常用命令整理
kinit 环境登录
kinit -kt /etc/security/keytabs/kafka.service.keytab kafka/[email protected];
基本操作
./bin/kafka-console-producer.sh --broker-list 10.201.168.136:6667 --topic test1 --producer.config config/producer.properties --producer-property security.protocol=SASL_PLAINTEXT
bin/kafka-console-consumer.sh --bootstrap-server hzadg-mammut-platform2.server.163.org:6667 --topic test1 --from-beginning --consumer.config=config/consumer.properties --new-consumer --property security.protocol=SASL_PLAINTEXTkafka在zk中使用的znode:
[zk: hzadg-mammut-platform2.server.163.org:2181(CONNECTED) 13] ls /
[hbase-secure, zk-test, zookeeper]
[zk: hzadg-mammut-platform2.server.163.org:2181(CONNECTED) 14] ls /
[hbase-secure, zk-test, isr_change_notification, kafka-acl, admin, zookeeper, kafka-acl-changes, consumers, config, controller, brokers, controller_epoch]producer
sudo su kafka -l -c 'sh /usr/ndp/current/kafka-broker/config/kafka-env.sh && /usr/ndp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list hzadg-mammut-platform2.server.163.org:6667 --topic test1 --producer.config /usr/ndp/current/kafka-broker/config/producer.properties --producer-property security.protocol=SASL_PLAINTEXT'
./bin/kafka-console-producer.sh --broker-list hzadg-mammut-platform2.server.163.org:6667 --topic test --producer.config config/producer.properties --producer-property security.protocol=SASL_PLAINTEXT
./bin/kafka-console-producer.sh --broker-list localhost:6667 --topic ATLAS_ENTITIES
--security-protocol PLAINTEXTSASL
./bin/kafka-console-producer.sh --broker-list 127.0.0.1:6667 --producer.config config/producer.properties --producer-property security.protocol=SASL_PLAINTEXT --topic test2
./bin/kafka-console-producer.sh --broker-list hzadg-mammut-platform2.server.163.org:6667 --producer.config config/producer.properties --producer-property security.protocol=SASL_PLAINTEXT --topic test2consumer
bin/kafka-console-consumer.sh --zookeeper hzadg-mammut-platform2.server.163.org:2181 --from-beginning --consumer.config=config/consumer.properties --property security.protocol=SASL_PLAINTEXT --topic test3
bin/kafka-console-consumer.sh --from-beginning --consumer.config=config/consumer.properties --new-consumer --bootstrap-server=hzadg-mammut-platform2.server.163.org:6667 --property security.protocol=SASL_PLAINTEXT --topic test3
bin/kafka-console-consumer.sh --from-beginning --consumer.config config/consumer.properties --new-consumer --bootstrap-server hzadg-mammut-platform2.server.163.org:6667 --property security.protocol=SASL_PLAINTEXT --zookeeper hzadg-mammut-platform2.server.163.org:2181 --topic test3
./bin/kafka-console-producer.sh --broker-list hzadg-mammut-platform2.server.163.org:6667 --producer.config config/producer.properties --producer-property security.protocol=SASL_PLAINTEXT --topic test3
hzadg-mammut-platform2.server.163.org:2181,hzadg-mammut-platform3.server.163.org:2181,hzadg-mammut-platform4.server.163.org:2181
bin/kafka-console-consumer.sh --from-beginning --consumer.config=config/consumer.properties --new-consumer --bootstrap-server=hzadg-mammut-platform2.server.163.org:6667 --property security.protocol=SASL_PLAINTEXT --topic test3
# bin/kafka-console-consumer.sh --from-beginning --consumer.config=config/consumer.properties --new-consumer --bootstrap-server=hzadg-mammut-platform2.server.163.org:6667,hzadg-mammut-platform2.server.163.org:6668,hzadg-mammut-platform2.server.163.org:6669 --topic test2创建topic
./bin/kafka-topics.sh --zookeeper hzadg-mammut-platform2.server.163.org:2181 --topic ATLAS_ENTITIES --create --partitions 1 --replication-factor 1
./bin/kafka-topics.sh --zookeeper hzadg-mammut-platform2.server.163.org:2181 --topic test8 --create --partitions 1 --replication-factor 1list topic
bin/kafka-topics.sh –zookeeper hzadg-mammut-platform2.server.163.org:2181 –list
topic状态
./bin/kafka-topics.sh –describe –zookeeper hzadg-mammut-platform2.server.163.org:2181 –topic test6
删除topic
bin/kafka-topics.sh –zookeeper hzadg-mammut-platform2.server.163.org:2181 –delete –topic ATLAS_HOOK
授权list
bin/kafka-acls.sh --authorizer-properties zookeeper.connect=hzadg-mammut-platform2.server.163.org:2181 --group atlas --list --topic ATLAS_HOOK
./bin/kafka-acls.sh --authorizer-properties zookeeper.connect=hzadg-mammut-platform2.server.163.org:2181 --add --topic test8 --allow-principal User:* --producer
./bin/kafka-acls.sh --authorizer-properties zookeeper.connect=hzadg-mammut-platform2.server.163.org:2181 --add --topic test7 --allow-principal User:* --consumer --group test-consumer-group启动kafka
source /usr/ndp/current/kafka-broker/config/kafka-env.sh ; /usr/ndp/current/kafka-broker/bin/kafka.sh start
bin/kafka-console-producer.sh --zookeeper hzadg-mammut-platform2.server.163.org:2181 --topic ATLAS_HOOK --property "security-protocol=PLAINTEXTSASL"
bin/kafka-console-producer.sh --broker-list hzadg-mammut-platform1.server.163.org:6667 --topic ATLAS_HOOK
bin/kafka-console-consumer.sh --zookeeper hzadg-mammut-platform2.server.163.org:2181 --topic ATLAS_HOOK --from-beginning --property "security-protocol=SASL_PLAINTEXT"
bin/kafka-console-consumer.sh --zookeeper hzadg-mammut-platform2.server.163.org:2181 --topic ATLAS_HOOK --security-protocol PLAINTEXTSASL --from-beginning
advertised.host.name其他命令参考：
rmr /brokers/topics/__consumer_offsets
./bin/kafka-topics.sh --zookeeper hzadg-mammut-platform2.server.163.org:2181 --topic test8 --create --partitions 1 --replication-factor 1
./bin/kafka-acls.sh --authorizer-properties zookeeper.connect=hzadg-mammut-platform2.server.163.org:2181 --add --topic test8 --allow-principal User:* --producer --group test-consumer-group
./bin/kafka-acls.sh --authorizer-properties zookeeper.connect=hzadg-mammut-platform2.server.163.org:2181 --add --topic ATLAS_HOOK --allow-principal User:* --consumer --group test-consumer-group
./bin/kafka-topics.sh --zookeeper hzadg-mammut-platform2.server.163.org:2181 --topic test4 --create --partitions 1 --replication-factor 1
./bin/kafka-console-producer.sh --broker-list hzadg-mammut-platform1.server.163.org:6667 --producer.config config/producer.properties --producer-property security.protocol=SASL_PLAINTEXT --topic ATLAS_HOOK
bin/kafka-console-consumer.sh --from-beginning --consumer.config config/consumer.properties --new-consumer --bootstrap-server hzadg-mammut-platform2.server.163.org:6667 --property security.protocol=SASL_PLAINTEXT --topic ATLAS_HOOK
参考：
* http://www.infoq.com/cn/articles/kafka-analysis-part-7
* http://blog.csdn.net/u012373815/article/details/53648757
* https://docs.confluent.io/3.1.1/streams/architecture.html#streams-architecture
* https://gist.github.com/timothyrenner/a99c86b2d6ed2c22c8703e8c7760af3a
* https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=51809888
* https://cwiki.apache.org/confluence/display/KAFKA/Kafka+Improvement+Proposals
* confluent Authorization and ACLs: https://docs.confluent.io/current/kafka/authorization.html
* https://cwiki.apache.org/confluence/display/KAFKA/KIP-11+-+Authorization+Interface
* IBM Kafka ACLs in Practice: https://developer.ibm.com/opentech/2017/05/31/kafka-acls-in-practice/
* Kafka ACL: http://orchome.com/185
* kafka 官网： https://kafka.apache.org/documentation/