Flume采集端口数据kafka消费

Flume采集端口数据kafka消费

1.flume单独搭建

#cd /root/flume
#vi conf/necat.conf

a1.sources = r1
a1.sinks = k1
a1.channels = c1

# sources
a1.sources.r1.type = netcat
a1.sources.r1.bind = master
a1.sources.r1.port = 44444

# sinks
a1.sinks.k1.type = logger

# channels
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# merge
a1.sources.r1.channels = c1
a1.sinks.k1.channels = c1

# 启动
cd /root/flume
nc -lk ip 44444(开端口)
/root/flume/bin/flume-ng agent -n a1 -c /root/flume/conf -f /root/flume/conf/netcat.conf -Dflume.root.logger=INFO,console

2.Flume采集端口数据kafka消费

# 采端口
cd /root/flume
vi conf/nc.conf

a1.sources = r1
a1.sinks = k1
a1.channels = c1

# sources
a1.sources.r1.type =netcat
a1.sources.r1.bind = master
a1.sources.r1.port = 7777

# sinks
a1.sinks.k1.type=org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.topic=test
a1.sinks.k1.kafka.bootstrap.servers=master:9092
a1.sinks.k1.kafka.producer.acks=1

# channels
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100

# merge
a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1

#先开kafka消费
cd /root/kafka
/root/kafka/bin/kafka-console-consumer --bootstrap-server master:9092 --topic test
#再开flume
cd /root/flume
/root/flume/bin/flume-ng agent -n a1 -c /root/flume/conf -f /root/flume/conf/nc.conf -Dflume.root.logger=INFO,console
#再开nc连接端口
nc master 7777
#nc master 7777
输入数据，kafka消费可以看到