flume配置文件详解

a1.sources = r1
a1.sinks = k1
a1.channels = c1

##注意：不能往监控目中重复丢同名文件
a1.sources.r1.type = spooldir (cdh) /exec(cdh) /taildir(apache1.7版本之后
,监控文件和目录,断点续查)
a1.sources.r1.spoolDir = /root/logs2 //监控的文件夹
a1.sources.r1.fileHeader = true

a1.sinks.k1.type = hdfs
a1.sinks.k1.channel = c1
a1.sinks.k1.hdfs.path = /flume/events/%y-%m-%d/%H%M/ //上传hdfs的路径
a1.sinks.k1.hdfs.filePrefix = events- //加个前缀
a1.sinks.k1.hdfs.round = true //是符开启时间上的舍弃
a1.sinks.k1.hdfs.roundValue = 10 //10分钟数据滚动,10分钟创建一个文件夹
a1.sinks.k1.hdfs.roundUnit = minute
a1.sinks.k1.hdfs.rollInterval = 0 //以间隔时间滚动
a1.sinks.k1.hdfs.rollSize = 134217728 //以文件大小滚动 byte
a1.sinks.k1.hdfs.rollCount = 0 //以event的个数滚动
a1.sinks.k1.hdfs.idleTimeout= 0 //超时时间设置
a1.sinks.k1.hdfs.batchSize = 1 //1批次写几个
a1.sinks.k1.hdfs.useLocalTimeStamp = true //开启本地时间戳功能
#生成的文件类型，默认是Sequencefile，可用DataStream，则为普通文本
a1.sinks.k1.hdfs.fileType = DataStream

a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

#event是agent的名称，event中定义了一个叫r1的source，如果有多个，使用空格间隔
event.sources = r1
event.sinks = k1
event.channels = c1

#组名名.属性名=属性值
#定义source
event.sources.r1.type=spooldir
event.sources.r1.spoolDir=/opt/mydata/events/
event.sources.r1.interceptors=i1
event.sources.r1.interceptors.i1.type = regex_filter
event.sources.r1.interceptors.i1.regex=event_id.*
event.sources.r1.interceptors.i1.excludeEvents=true
event.sources.r1.deserializer.maxLineLength=204800

#定义chanel
event.channels.c1.type=memory

#定义sink
event.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
event.sinks.k1.kafka.topic=events
event.sinks.k1.kafka.bootstrap.servers=192.168.80.181:9092

#连接组件 同一个source可以对接多个channel，一个sink只能从一个channel拿数据！
event.sources.r1.channels=c1
event.sinks.k1.channel=c1