flume-ng avro方式传输数据配置 flume-ng多节点实例

 

tail-to-avro

agent1.sources = source1
agent1.sinks = sink1
agent1.channels = channel1

# Describe/configure spooldir source1
#agent1.sources.source1.type = spooldir
#agent1.sources.source1.spoolDir = /var/log/apache/flumeSpool1
#agent1.sources.source1.fileHeader = true

# Describe/configure tail -F source1
agent1.sources.source1.type = exec
agent1.sources.source1.command = tail -n +0 -F /tmp/log.log
agent1.sources.source1.channels = channel1

# Describe/configure nc source1
#agent1.sources.source1.type = netcat
#agent1.sources.source1.bind = localhost
#agent1.sources.source1.port = 44444

#configure host for source
agent1.sources.source1.interceptors = i1
agent1.sources.source1.interceptors.i1.type = host
agent1.sources.source1.interceptors.i1.hostHeader = hostname

# Describe logger  sink1
#agent1.sinks.sink1.type = logger

# Describe avro  sink1
agent1.sinks.sink1.type = avro
agent1.sinks.sink1.hostname = 172.16.10.175
agent1.sinks.sink1.port = 4545

# Use a channel which buffers events in memory
agent1.channels.channel1.type = memory
agent1.channels.channel1.keep-alive = 120
agent1.channels.channel1.capacity = 500000
agent1.channels.channel1.transactionCapacity = 600

# Bind the source and sink to the channel
agent1.sources.source1.channels = channel1
agent1.sinks.sink1.channel = channel1

 


avro-to-rollfile

agent1.sources = source1
agent1.sinks = sink1
agent1.channels = channel1

# Describe/configure avro source

agent1.sources.source1.type = avro
agent1.sources.source1.bind =172.16.10.175
agent1.sources.source1.port = 4545


# Describe logger sink1
#agent1.sinks.sink1.type = logger

# Describe file sink1
agent1.sinks.sink1.type = file_roll
agent1.sinks.sink1.sink.directory = /var/log/flume

# Use a channel which buffers events in memory
agent1.channels.channel1.type = memory
agent1.channels.channel1.keep-alive = 120
agent1.channels.channel1.capacity = 500000
agent1.channels.channel1.transactionCapacity = 600

# Bind the source and sink to the channel
agent1.sources.source1.channels = channel1
agent1.sinks.sink1.channel = channel1

 

启动：

 

./flume-ng agent -c /usr/local/flumeng/apache-flume-1.5.0-bin/conf/ -f /usr/local/flumeng/apache-flume-1.5.0-bin/conf/flume-single.properties -n agent1 -Dflume.root.logger=INFO,console

 

 

 

 

亲自操作如下：

 

 

 source配置（接收）：

 

# The configuration file needs to define the sources, 
# the channels and the sinks.
# Sources, channels and sinks are defined per agent, 
# in this case called 'a
#agent section  
producer.sources = s 
producer.channels = c
producer.sinks = r

#producer.sources.s.type = seq
producer.sources.s.channels = c
#producer.sources.s.type = exec
#producer.sources.s.command=tail -n +0 -F /usr/local/nginx/nginxlog/access.log
producer.sources.s.deletePolicy=never
producer.sources.s.type = avro
producer.sources.s.bind = localhost
producer.sources.s.port = 4545
# Each sink's type must be defined(给谁了)
#producer.sinks.r.type = avro
#producer.sinks.r.hostname = 10.1.1.100
#producer.sinks.r.port = 20000
producer.sinks.r.type = org.xx.clickstream.sink.kafka.KafkaSink
producer.sinks.r.zk.connect = 127.0.0.1:2181
producer.sinks.r.metadata.broker.list=127.0.0.1:9092
producer.sinks.r.partitioner.class=org.xx.clickstream.partition.TypePartitioner
producer.sinks.r.serializer.class=kafka.serializer.StringEncoder
producer.sinks.r.request.required.acks=1
producer.sinks.r.max.message.size=1000000
producer.sinks.r.producer.type=sync
producer.sinks.r.custom.encoding=UTF-8
#Specify the channel the sink should use
producer.sinks.r.channel = c
# Each channel's type is defined.
producer.channels.c.type = memory  
producer.channels.c.capacity = 1000000
producer.channels.c.transactionCapacity = 1000000

#producer.channels.c.type=file
#producer.channels.c.checkpointDir=/usr/local/flumeng/checkpointdir/tcpdir/example_agent
#producer.channels.c.dataDirs=/usr/local/flumeng/datadirs/tddirs/example_agen

 

sink配置（发送）：

 

# The configuration file needs to define the sources, 
# the channels and the sinks.
# Sources, channels and sinks are defined per agent, 
# in this case called 'a
#agent section  
producer.sources = s 
producer.channels = c
producer.sinks = r

#producer.sources.s.type = seq
producer.sources.s.channels = c
producer.sources.s.type = exec
producer.sources.s.command=tail -n +0 -F /usr/local/nginx/nginxlog/access.log
producer.sources.s.deletePolicy=never
#producer.sources.s.type = avro
#producer.sources.s.bind = localhost
#producer.sources.s.port = 10000
# Each sink's type must be defined(给谁了)
producer.sinks.r.type = avro
producer.sinks.r.hostname = localhost
producer.sinks.r.port = 4545
#producer.sinks.r.type = org.xx.clickstream.sink.kafka.KafkaSink
#producer.sinks.r.zk.connect = 127.0.0.1:2181
#producer.sinks.r.metadata.broker.list=127.0.0.1:9092
#producer.sinks.r.partitioner.class=org.xx.clickstream.partition.TypePartitioner
#producer.sinks.r.serializer.class=kafka.serializer.StringEncoder
#producer.sinks.r.request.required.acks=1
producer.sinks.r.max.message.size=1000000
producer.sinks.r.producer.type=sync
producer.sinks.r.custom.encoding=UTF-8
#Specify the channel the sink should use
producer.sinks.r.channel = c
# Each channel's type is defined.
producer.channels.c.type = memory  
producer.channels.c.capacity = 1000000
producer.channels.c.transactionCapacity = 1000000

#producer.channels.c.type=file
#producer.channels.c.checkpointDir=/usr/local/flumeng/checkpointdir/tcpdir/example_agent
#producer.channels.c.dataDirs=/usr/local/flumeng/datadirs/tddirs/example_agen

 

启动顺序，先启动source接收，再启动sink

 

#先启动接收source，准备好接收
#./flume-ng agent -c /usr/local/flumeng/apache-flume-1.5.2-bin/conf/ -f /usr/local/flumeng/apache-flume-1.5.2-bin/conf/flume-avrosource.properties -n producer -Dflume.root.logger=INFO,console

#再启动发送sink，发送
#./flume-ng agent -c /usr/local/flumeng/apache-flume-1.5.2-bin/conf/ -f /usr/local/flumeng/apache-flume-1.5.2-bin/conf/flume-avrosink.properties -n producer -Dflume.root.logger=INFO,console