flume安装,配置与使用

flume-****-cdh.tar.gz
解压
配置文件
flume-env.sh
JAVA_HOME=$JAVA_HOME

监听本地机器44444端口
配置文件
telnet的配置文件 flume-telnet.conf
# Name the components on this agent
telnet.sources = r1
telnet.sinks = k1
telnet.channels = c1

# Describe/configure the source
telnet.sources.r1.type = netcat
telnet.sources.r1.bind = localhost
telnet.sources.r1.port = 44444

# Describe the sink
telnet.sinks.k1.type = logger

# Use a channel which buffers events in memory
telnet.channels.c1.type = memory
telnet.channels.c1.capacity = 1000
telnet.channels.c1.transactionCapacity = 100

# Bind the source and sink to the channel
telnet.sources.r1.channels = c1
telnet.sinks.k1.channel = c1

启动flume
bin/flume-ng agent \
--conf conf/ \
--name telnet \
--conf-file /conf/flume-telnet.conf \
-Dflume.root.logger==INFO.console

监控hive滚动日志并上传到HDFS目录中
# Name the components on this agent
HDFS.sources = r2
HDFS.sinks = k2
HDFS.channels = c2

# Describe/configure the source
HDFS.sources.r2.type = exec
HDFS.sources.r2.command = tail -f /opt/modules/cdh/hive-0.13.1-cdh5.3.6/logs/hive.log
HDFS.sources.r2.shell = /bin/bash -c

# Describe the sink
HDFS.sinks.k2.type = hdfs
HDFS.sinks.k2.hdfs.path = hdfs://192.168.122.20:8020/flume/%Y%m%d/%H
#上传文件的前缀
HDFS.sinks.k2.hdfs.filePrefix = events-hive-
#是否按照时间滚动文件夹
HDFS.sinks.k2.hdfs.round = true
#多少时间单位创建一个新的文件夹
HDFS.sinks.k2.hdfs.roundValue = 1
#重新定义时间单位
HDFS.sinks.k2.hdfs.roundUnit = hour
#是否使用本地时间戳
HDFS.sinks.k2.hdfs.useLocalTimeStamp = true
#积攒多少个Event才flush到HDFS一次
HDFS.sinks.k2.hdfs.batchSize = 1000
#设置文件类型，可支持压缩
HDFS.sinks.k2.hdfs.fileType = DataStream
#多久生成一个新的文件
HDFS.sinks.k2.hdfs.rollInterval = 600
#设置每个文件的滚动大小
HDFS.sinks.k2.hdfs.rollSize = 134217700
#文件的滚动与Event数量无关
HDFS.sinks.k2.hdfs.rollCount = 0
#最小冗余数
HDFS.sinks.k2.hdfs.minBlockReplicas = 1

# Use a channel which buffers events in memory
HDFS.channels.c2.type = memory
HDFS.channels.c2.capacity = 1000
HDFS.channels.c2.transactionCapacity = 100

# Bind the source and sink to the channel
HDFS.sources.r2.channels = c2
HDFS.sinks.k2.channel = c2

开启hive.log-->HDFS的服务
bin/flume-ng agent \
-n HDFS \
-c /conf \
-f /conf/flume-HDFS.conf \
-Dflume.root.logger==INFO.console

监控指定目录下的文件,并上传到HDFS中
创建配置文件flume-dir.conf
# Name the components on this agent
DIR.sources = r3
DIR.sinks = k3
DIR.channels = c3

# Describe/configure the source
DIR.sources.r3.type = spooldir
DIR.sources.r3.spoolDir = /opt/modules/uploadJeremy/
a3.sources.r3.fileHeader = true
#忽略所有以.tmp结尾的文件，不上传
a3.sources.r3.ignorePattern = ([^ ]*\.tmp)

# Describe the sink
DIR.sinks.k3.type = hdfs
DIR.sinks.k3.hdfs.path = hdfs://192.168.110.101:8020/flume/upload/%Y%m%d/%H
#上传文件的前缀
DIR.sinks.k3.hdfs.filePrefix = upload-
#是否按照时间滚动文件夹
DIR.sinks.k3.hdfs.round = true
#多少时间单位创建一个新的文件夹
DIR.sinks.k3.hdfs.roundValue = 1
#重新定义时间单位
DIR.sinks.k3.hdfs.roundUnit = hour
#是否使用本地时间戳
DIR.sinks.k3.hdfs.useLocalTimeStamp = true
#积攒多少个upload才flush到DIR一次
DIR.sinks.k3.hdfs.batchSize = 1000
#设置文件类型，可支持压缩
DIR.sinks.k3.hdfs.fileType = DataStream
#多久生成一个新的文件
DIR.sinks.k3.hdfs.rollInterval = 600
#设置每个文件的滚动大小
DIR.sinks.k3.hdfs.rollSize = 134217700
#文件的滚动与Event数量无关
DIR.sinks.k3.hdfs.rollCount = 0
#最小冗余数
DIR.sinks.k3.hdfs.minBlockReplicas = 1

# Use a channel which buffers events in memory
DIR.channels.c3.type = memory
DIR.channels.c3.capacity = 1000
DIR.channels.c3.transactionCapacity = 100

# Bind the source and sink to the channel
DIR.sources.r3.channels = c3
DIR.sinks.k3.channel = c3

bin/flume-ng agent -n DIR -c conf/ -f conf/flume-dir.conf -Dflume.root.logger==INFO.console

向/opt/modules/uploadJeremy的目录下传输文件进行测试