flume安装配置实战

一、 读取网络端口的日志 

flume-ng agent --name a1 --conf $FLUME_HOME/conf --conf-file $FLUME_HOME/conf/example.conf -Dflume.root.logger=Info,console

二、 读取本地日志文件的更改

 

flume-ng agent --name a1 --conf $FLUME_HOME/conf --conf-file $FLUME_HOME/conf/exec-memory-logger.conf -Dflume.root.logger=INFO,console

 

/**
*向文件中写入东西
*/
echo ahahahhahha >> flumeLog.log

三、

 

（第一台avro-memory-logger.conf）

avro-memory-logger.sources = avro-source
avro-memory-logger.sinks = logger-sink
avro-memory-logger.channels = memory-channel

avro-memory-logger.sources.avro-source.type = avro
avro-memory-logger.sources.avro-source.bind = 192.168.163.128
avro-memory-logger.sources.avro-source.port = 44444

avro-memory-logger.sinks.logger-sink.type = logger

avro-memory-logger.channels.memory-channel.type = memory

avro-memory-logger.sources.avro-source.channels=memory-channel
avro-memory-logger.sinks.logger-sink.channel=memory-channel

（第二台：exec-memory-avro.conf） 该文件启动产生异常

exec-memory-avro.sources=exec-source
exec-memory-avro.sinks = avro-sink
exec-memory-avro.channels = memory-channel

exec-memory-avro.sources.exec-source.type=exec
exec-memory-avro.sources.exec-source.command=tail -F /home/peng/peng/data/flumeLog.log
exec-memory-avro.sources.exec-source.shell = /bin/sh -c

exec-memory-avro.sinks.avro-sink.type=avro
exec-memory-avro.sinks.avro-sink.hostname = 192.168.163.128
exec-memory-avro.sinks.avro-sink.port = 44444
exec-memory-avro.channels.memory-channel.type = memory
exec-memory-avro.sources.exec.source.channels=memory-channel
exec-memory-avro.sinks.avro-sink.channel=memory-channel

3.1 启动顺序也很重要

/**
* 先启动这个
*/
flume-ng agent --name avro-memory-logger --conf $FLUME_HOME/conf --conf-file $FLUME_HOME/conf/avro-memory-logger.conf -Dflume.root.logger=INFO,console

/**
* 第二个启动这个
*/

flume-ng agent --name exec-memory-avro --conf $FLUME_HOME/conf --conf-file $FLUME_HOME/conf/exec-memory-avro.conf -Dflume.root.logger=INFO,console