flume集群

flume-ng agent --conf /root/flume-1.4.0/conf --conf-file  /root/flume-1.4.0/conf/a.conf --name a1 -Dflume.root.logger=INFO,console

curl -X get -d '[{  "headers" :{"namenode" :  "namenode.example.com","datanode" :  "random_datanode.example.com"},"body" :  "really_random_body"}]' http://vm006:44444

节点1

a2.sources = r1
a2.sinks = k1
a2.channels = c1


# Describe/configure the source
a2.sources.r1.type = avro
a2.sources.r1.bind = 0.0.0.0
a2.sources.r1.port = 44444
a2.sources.r1.handler = org.apache.flume.source.http.JSONHandler


# Describe the sink
a2.sinks.k1.type = file_roll
a2.sinks.k1.sink.directory = /data/flume/


# Use a channel which buffers events in memory
a2.channels.c1.type = memory
a2.channels.c1.capacity = 1000
a2.channels.c1.transactionCapacity = 100


# Bind the source and sink to the channel
a2.sources.r1.channels = c1
a2.sinks.k1.channel = c1

节点2

a2.sources = r1
a2.sinks = k1
a2.channels = c1


# Describe/configure the source
a2.sources.r1.type = avro
a2.sources.r1.bind = 0.0.0.0
a2.sources.r1.port = 44444
a2.sources.r1.handler = org.apache.flume.source.http.JSONHandler


# Describe the sink
a2.sinks.k1.type = file_roll
a2.sinks.k1.sink.directory = /data/flume


# Use a channel which buffers events in memory
a2.channels.c1.type = memory
a2.channels.c1.capacity = 1000
a2.channels.c1.transactionCapacity = 100


# Bind the source and sink to the channel
a2.sources.r1.channels = c1
a2.sinks.k1.channel = c1

负载均衡节点

a2.sources = r1
a2.sinks = k1
a2.channels = c1


# Describe/configure the source
a2.sources.r1.type = avro
a2.sources.r1.bind = 0.0.0.0
a2.sources.r1.port = 44444
a2.sources.r1.handler = org.apache.flume.source.http.JSONHandler


# Describe the sink
a2.sinks.k1.type = file_roll
a2.sinks.k1.sink.directory = /data/flume


# Use a channel which buffers events in memory
a2.channels.c1.type = memory
a2.channels.c1.capacity = 1000
a2.channels.c1.transactionCapacity = 100


# Bind the source and sink to the channel
a2.sources.r1.channels = c1
a2.sinks.k1.channel = c1

客户端机器暂时使用flume-ng的agent发送，配置如下：

 # Define a memory channel called c1 on a1
 a1.channels.c1.type = file
 a1.channels.c1.checkpointDir = /tmp/flume/checkpoint
 a1.channels.c1.dataDirs = /tmp/flume/data


 a1.sources.r1.channels = c1
 a1.sources.r1.type = exec
 a1.sources.r1.command = cat /tmp/linux.log

 a1.sinks.k1.type = avro
 a1.sinks.k1.channel = c1
 a1.sinks.k1.hostname = host1
 a1.sinks.k1.port = 41415

 a1.channels = c1
 a1.sources = r1
 a1.sinks = k1