emqx+Kafka安装

emqx+kafka插件（未完待续。。。）

一.emqx安装

1. 下载erlang

   apt-get install git autoconf
   wget http://erlang.org/download/otp_src_21.3.tar.gz
   

2. 编译安装elang
   安装依赖并编译安装

   apt-get install ncurses-dev libssl1.0-dev
   cd otp
   ./otp_build autoconf
   ./configure --prefix=/usr/local/erlang
   make
   make install
   

3. 将erlang添加到环境变量中

   export PATH=$PATH:/usr/local/erlang/bin
   

4. 下载emqx

   git clone https://github.com/emqx/emqx-rel.git
   

5. 编译

   cd emqx-relx && make
   

6. 启动

   cd _rel/emqx 
   ./bin/emqx console
   

7. 测试

二.zookeeper+kafka安装

1. 下载jdk

   wget https://download.oracle.com/otn-pub/java/jdk/8u201-   b09/42970487e3af4f5aa5bca3f542482c60/jdk-8u201-linux-x64.tar.gz?AuthParam=1555165093_9eb04dda41b22e7c9ad39839915e3c02
   

2. 解压

   tar -zxvf jdk-8u201-linux-x64.tar.gz
   

3. 添加环境变量
   vi /etc/profile

   JAVA_HOME=/usr/local/jdk1.8.0_201
   CLASSPATH=$JAVA_HOME/lib/
   PATH=$PATH:$JAVA_HOME/bin
   

4. 更新环境变量设置

   source /etc/profile
   

5. 下载zookeeper

   wget https://mirrors.tuna.tsinghua.edu.cn/apache/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz
   

6. 配置zookeeper
   修改配置文件

   cp zoo_sample.cfg zoo.cfg
   

7. 在zoo.cfg中添加

   dataDir=/var/kafka/zookeeper
   

8. 下载Kafka

   wget https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/2.2.0/kafka_2.12-2.2.0.tgz
   

9. 配置kafka
   修改配置文件 vi config/server.properties

   listeners=PLAINTEXT://127.0.0.1:9092
   log.dirs=/var/kafka/kafka-logs
   zookeeper.connect=localhost:2181
   

10. 修改启动文件
    修改启动文件，vi kafka-server-start.sh

    if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
         export KAFKA_HEAP_OPTS="-Xmx256M -Xms128M"
    fi  
    

11. 启动zookeeper和kafka

    ./bin/zkServer.sh start
    ./bin/kafka-server-start.sh config/server.properties       
    

12. 创建主题

    ./bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test-topic
    

13. 启动生产者

    bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test-topic
    

14. 启动消费者

    bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test-topic --from-beginning
    

15. 测试，生产者发送消息，消费者收到消息，表示成功。
16. 后台启动

    nohup ./kafka-server-start.sh ./config/server.properties > /dev/null 2>&1 &
    

三.emqx配置插件

1. 复制一份emqx插件模板

   cd emqx/deps
   cp -r emqx_plugin_template  emqx_plugin_kafka
   

2. 将emqx_plugin_kafka文件下的所有template改为kafka
   进入kafka插件，执行make clean,并将文件下文件名和内容的template改为kafka。

   mv etc/emqx_plugin_template.config  etc/emqx_plugin_kafka.conf
   

3. 修改编译文件

   BUILD_DEPS = emqx cuttlefish ekaf
   dep_ekaf = git https://github.com/helpshift/ekaf master
   

4. 在emqx-rel下的Makefile文件中增加插件
5. 在relx.config中增加

   kafkamocker,
   ekaf,
   {emqx_plugin_kafka,load},
   

6. 修改emqx_plugin_kafka.erl，

   -module(emqx_plugin_kafka).
   -include_lib("emqx/include/emqx.hrl").
   -define(TOPIC, <<"test-topic">>).
     
   load(Env) ->
       ekaf_init([Env]),
   
   %%---------------------------message publish start--------------------------%%
   on_message_publish(Message = #message{topic = <<"$SYS/", _/binary>>}, _Env) ->
   {ok, Message};
   
   on_message_publish(Message, _Env) ->
    	io:format("Publish ~s~n", [emqx_message:format(Message)]),
   
    	Topic=Message#message.topic,
   	Payload=Message#message.payload,
    	Qos=Message#message.qos,
    	Timestamp=Message#message.timestamp,
   
    	Json = jsx:encode([
        		{type,<<"published">>},
        		{topic,Topic},
        		{payload,Payload},
        		{qos,Qos},
        		{cluster_node,node()}
        		%%,{ts,emqx_time:now_to_secs(Timestamp)}
   	 ]),
   
   	ekaf:produce_sync(?TOPIC,Json),
   
    	{ok, Message}.
   
   %%---------------------message publish stop----------------------%%
   
   
   %%--------------------------ekaf_init start---------------------%%
   ekaf_init(_Env) ->
       %%这里参照了https://github.com/helpshift/ekaf
       %%仅实现了固定的broker和topic后期还需完善配置文件
    	%% Get parameters
   	%%{ok, Broker} = application:get_env(emqx_plugin_kafka, broker),
   	%%{ok, Topic} = application:get_env(emqx_plugin_kafka, topic),
   	%%Broker = "127.0.0.1:9092",
    	%%Topic = <<"test-topic">>,
    	%% start
    	application:load(ekaf),
   
   	application:set_env(ekaf, ekaf_bootstrap_broker, {"127.0.0.1", 9092}),
   	%%application:set_env(ekaf, ekaf_bootstrap_topics, <<"test-topic">>),
   
   	{ok, _} = application:ensure_all_started(ekaf),
   
    	ekaf:produce_sync(?TOPIC, <<"emqx_plugin_kafka is started!">>).
   
   
    	%%io:format("Init ekaf with ~p~n", [Broker]).
   
   %%------------------------------ekaf_init stop-----------------------------------%%