Flume的安装和实际使用
flume的安装和使用
wget “http://mirrors.cnnic.cn/apache/flume/1.6.0/apache-flume-1.6.0-bin.tar.gz”
官方下载包
创建文件夹并解压
cd /opt/module
mkdir flume
tar -zxvf apache-flume-1.8.0-bin.tar.gz -C /opt/module/flume
配置并使其生效
vi /etc/profile
export FLUME_HOME=/opt/module/flume/apache-flume-1.8.0-bin
export FLUME_CONF_DIR= $FLUME_HOME/conf
export PATH=$FLUME_HOME/bin:$PATH
source /etc/profile
配置flume-env.sh的JAVA_HOME
cd /opt/module/flume/apache-flume-1.8.0-bin
cd conf
cp flume-env.sh.template flume-env.sh
vi flume-env.sh
在其中写入
export JAVA_HOME=/opt/module/jdk1.8.0_121
查看flume版本
cd /opt/module/flume/apache-flume-1.8.0-bin
./bin/flume-ng version
此时会出现加载不出来主类的问题
解决：
cd /opt/module/hbase/conf
vi hbase-env.sh
将export HBASE_CLASSPATH=/opt/module/hbase/conf这句话注释
此时再执行
cd /opt/module/flume/apache-flume-1.8.0-bin
./bin/flume-ng version
flume的应用实例
使用Flumen接受来自AvroSoure的信息
cd /opt/module/flume/apache-flume-1.8.0-bin/conf
vi avro.conf
a1.sources= r1
a1.sinks= k1a1.channels= c1
#Describe/configure the source
a1.sources.r1.type=avro
a1.sources.r1.channels= c1
a1.sources.r1.bind= 0.0.0.0
a1.sources.r1.port= 4141
#Describe the sink
a1.sinks.k1.type=logger
Use achannel which buffers events in memory
a1.channels.c1.type=memory
a1.channels.c1.capacity= 1000
a1.channels.c1.transactionCapacity= 100
Bindthe source and sink to the channel
a1.sources.r1.channels= c1
a1.sinks.k1.channel= c1
接下来继续执行
cd /opt/module/flume/apache-flume-1.8.0-bin
./bin/flume-ng agent -c . -f /opt/module/flume/apache-flume-1.8.0-bin/conf/avro.conf -n a1 -Dflume.root.logger=INFO,consol
出现以下结果
**此时打开另一个终端
**
cd /opt/module/flume/apache-flume-1.8.0-bin/
touch log.00
sh -c ‘echo“hello world” > /opt/module/flume/apache-flume-1.8.0-bin/log.00’
执行命令
./bin/flume-ng avro-client --conf conf -H localhost -p 4141 -F /opt/module/flume/apache-flume-1.8.0-bin/log.00
此时之前那个终端出现以下
使用Flumen接受来自Soure的信息
创建文件
cd /opt/module/flume/apache-flume-1.8.0-bin/
touch example.conf
写入
#Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1
#Describe/configure the sourcea1.sources.r1.type = netcat
a1.sources.r1.bind = localhost
a1.sources.r1.port = 44444
#Describe the sink
a1.sinks.k1.type = logger
#Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
#Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
执行下面命令
./bin/flume-ng agent --conf ./conf --conf-file ./example.conf --name a1 -Dflume.root.logger=INFO,console
显示如下
打开另一个终端
执行
yum install telnet
telnet localhost 44444
出现以下结果
原终端显示如下
exec1.conf接收本地文件
vi /opt/module/logstest/1.log
随便输入点内容
cd /opt/module/flume/apache-flume-1.8.0-bin
cd conf
vi exec1.conf
输入以下内容
#Name the components on this agent
a1.sources=r1
a1.sinks=k1
a1.channels=c1
#For each one of the source, the type is defined
a1.sources.r1.type=exec
a1.sources.r1.command=tail -F /opt/module/logstest/1.log
#whereis bash
a1.sources.r1.shell=/usr/bin/bash -c
#Each sink’s type must be defined
a1.sinks.k1.type=hdfs
a1.sinks.k1.hdfs.path=hdfs://bigdata128:9000/flume/%y%m%d/%H
a1.sinks.k1.hdfs.filePrefix=logs-
a1.sinks.k1.hdfs.round=true
a1.sinks.k1.hdfs.roundValue=1
a1.sinks.k1.hdfs.roundUnit=minute
a1.sinks.k1.hdfs.useLocalTimeStamp=true
a1.sinks.k1.hdfs.batchSize=100
a1.sinks.k1.hdfs.fileType=DataStream
a1.sinks.k1.hdfs.rollInterval=30
a1.sinks.k1.hdfs.rollSize=134217700
a1.sinks.k1.hdfs.rollCount=0
a1.sinks.k1.hdfs.minBlockReplicas=1
#Specify the channel should use
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100
#Bind thr source and sink to the channel
a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1
执行以下语句
/opt/module/flume/apache-flume-1.8.0-bin/bin/flume-ng agent --conf ./conf --conf-file exec1.conf --name a1 -Dflume.root.logger=INFO,console
运行结果如下
HDFS查看
Flume接收本地文件夹
cd /opt/module/flume/apache-flume-1.8.0-bin
cd conf
vi spooldir1.conf
输入以下内容
#agent名， source、channel、sink的名称
a1.sources = r1
a1.channels = c1
a1.sinks = k1
#具体定义source
a1.sources.r1.type = spooldir
a1.sources.r1.spoolDir = /opt/module/flume/apache-flume-1.8.0-bin/logs
#具体定义channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
#具体定义sink
a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.path = hdfs://bigdata128:9000/flume/%Y%m%d
a1.sinks.k1.hdfs.filePrefix = events-
a1.sinks.k1.hdfs.fileType = DataStream
a1.sinks.k1.hdfs.useLocalTimeStamp = true
#不按照条数生成文件
a1.sinks.k1.hdfs.rollCount = 0
#HDFS上的文件达到128M时生成一个文件
a1.sinks.k1.hdfs.rollSize = 134217700
#HDFS上的文件达到60秒生成一个文件
a1.sinks.k1.hdfs.rollInterval = 30
#组装source、channel、sink
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
输入以下命令
/opt/module/flume/apache-flume-1.8.0-bin/bin/flume-ng agent --conf ./conf --conf-file spooldir1.conf --name a1 -Dflume.root.logger=INFO,console
输出结果如下
HDFS显示如下
