flume-ng配置文件详解（一）

1 Avro案例

[root@localhost conf]# vi avro.conf
agent1.sources = r1
agent1.sinks = k1
agent1.channels = c1
 
agent1.sources.r1.type = avro
agent1.sources.r1.channels = c1
agent1.sources.r1.bind = 192.168.100.200
agent1.sources.r1.port = 4141
 
agent1.sinks.k1.type = logger
 
agent1.channels.c1.type = memory
agent1.channels.c1.capacity = 1000
agent1.channels.c1.transactionCapacity = 100
 
agent1.sources.r1.channels = c1
agent1.sinks.k1.channel = c1
[root@localhost apache-flume-1.7.0-bin]# bin/flume-ng agent --conf conf --conf-file conf/avro.conf --name agent1 -Dfilume.root.loggger=INFO,console
[root@localhost ~]# echo "hello world" > /usr/local/log.00
[root@localhost apache-flume-1.7.0-bin]# bin/flume-ng avro-client -c . -H 192.168.100.200 -p 4141 -F /usr/local/log.00

在对应的日记文件查看
[root@localhost apache-flume-1.7.0-bin]# vi logs/flume.log
21 Nov 2016 00:22:11,191 INFO  [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.sink.LoggerSink.process:95)  - Event: { headers:{} body: 68 65 6C 6C 6F 20 77 6F 72 6C 64                hello world }

2 Spool案例

[root@localhost local]# mkdir /usr/local/logs
[root@localhost conf]# vi spool.conf
agent1.sources = r1
agent1.sinks = k1
agent1.channels = c1

agent1.sources.r1.type = spooldir
agent1.sources.r1.channels = c1
agent1.sources.r1.spoolDir = /usr/local/logs
agent1.sources.r1.fileHeader = true

agent1.sinks.k1.type = logger

agent1.channels.c1.type = memory
agent1.channels.c1.capacity = 1000
agent1.channels.c1.transactionCapacity = 100

agent1.sources.r1.channels = c1
agent1.sinks.k1.channel = c1
[root@localhost apache-flume-1.7.0-bin]# bin/flume-ng agent --conf conf --conf-file conf/spool.conf --name agent1 -Dfilume.root.loggger=INFO,console
[root@localhost local]# echo "spool" > /usr/local/logs/spool.log

在对应的日记文件查看
[root@localhost apache-flume-1.7.0-bin]# vi logs/flume.log
21 Nov 2016 00:30:13,716 INFO  [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.sink.LoggerSink.process:95)  -

Event: { headers:{file=/usr/local/logs/spool.log} body: 73 70 6F 6F 6C                                  spool }

3 Exec案例

[root@localhost apache-flume-1.7.0-bin]# vi conf/exec.conf
agent1.sources = r1
agent1.sinks = k1
agent1.channels = c1

agent1.sources.r1.type = exec
agent1.sources.r1.channels = c1
agent1.sources.r1.command = tail -F /usr/local/logs/tail.txt

agent1.sinks.k1.type = logger

agent1.channels.c1.type = memory
agent1.channels.c1.capacity = 1000
agent1.channels.c1.transactionCapacity = 100

agent1.sources.r1.channels = c1
agent1.sinks.k1.channel = c1
[root@localhost apache-flume-1.7.0-bin]# bin/flume-ng agent --conf conf --conf-file conf/exec.conf --name agent1 -Dfilume.root.loggger=INFO,console
生成足够多的文件内容
[root@localhost logs]# for i in {1..100};do echo "exec tail$i" >> /usr/local/logs/tail.txt;echo $i;sleep 0.1;done
[root@localhost apache-flume-1.7.0-bin]# vi logs/flume.log
21 Nov 2016 00:37:40,669 INFO  [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.sink.LoggerSink.process:95)  
- Event: { headers:{} body: 65 78 65 63 20 74 61 69 6C 31                   exec tail1 }