Flume监听上传Hive日志文件到HDFS-02

拷贝Hadoop相关jar到Flume的lib目录下
                    share/hadoop/common/lib/hadoop-auth-2.5.0-cdh5.3.6.jar
                    share/hadoop/common/lib/commons-configuration-1.6.jar
                    share/hadoop/mapreduce1/lib/hadoop-hdfs-2.5.0-cdh5.3.6.jar
                    share/hadoop/common/hadoop-common-2.5.0-cdh5.3.6.jar

拷贝一份配置文件出来

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.


# The configuration file needs to define the sources, 
# the channels and the sinks.
# Sources, channels and sinks are defined per agent, 
# in this case called 'agent'

# example.conf: A single-node Flume configuration

# Name the components on this agent
#把agent起个名叫a2,sources叫r2,sinks叫k2.hdfs,channels叫c2
a2.sources = r2
a2.sinks = k2
a2.channels = c2

#监听数据来源为本地的4444端口
# Describe/configure the source
a2.sources.r2.type = exec
a2.sources.r2.command = tail -f /opt/modules/hive-0.13.1-cdh5.3.6/logs/hive.log

#/bin/bash -c     echo `date` 可以写成  echo /bin/bash -c date 就是它后面跟一个可执行脚本,来执行上面的监控日志的命令
a2.sources.r2.shell = /bin/bash -c

#logger是流传输过来解码后的数据
# Describe the sink
a2.sinks.k2.type = hdfs
a2.sinks.k2.hdfs.path = hdfs://192.168.1.20:8020/flume/%Y%m%d/%H
#上传文件的前缀
a2.sinks.k2.hdfs.filePrefix = events-hive-
#是否按照时间滚动文件夹
a2.sinks.k2.hdfs.round = true
#多长时间创建一个新的文件夹  1小时
a2.sinks.k2.hdfs.roundVlaue = 1
#重新定义时间单位
a2.sinks.k2.hdfs.roundUnit = hour
#是否使用本地时间戳
a2.sinks.k2.hdfs.useLocalTimeStamp = true
#积攒多个event的时候才flush到hdfs上
a2.sinks.k2.hdfs.batchSize = 1000
#设置文件类型,可支持压缩
a2.sinks.k2.hdfs.fileType = DataStream
#多久生成一个新文件 10分钟
a2.sinks.k2.hdfs.rollInterval = 600
#设置文件块多大.即使没到600秒,但是达到这个大小了也生成新文件
a2.sinks.k2.hdfs.rollSize = 134217700
#文件的滚动与event数量无关
a2.sinks.k2.hdfs.rollCount = 0
#最小冗余数
a2.sinks.k2.hdfs.minBlockReplicas = 1



#channels阶段以内存的形式保存数据  event数量100
# Use a channel which buffers events in memory
a2.channels.c2.type = memory
a2.channels.c2.capacity = 1000
a2.channels.c2.transactionCapacity = 100
 
#把source和sink和channel对接   source可以对接多个channels  sinks只能对接一个channel
# Bind the source and sink to the channel
a2.sources.r2.channels = c2
a2.sinks.k2.channel = c2

执行监控配置  $ bin/flume-ng agent --conf conf/ --name a2 --conf-file conf/flume-hdfs.conf 

然后在另一边进行对hive操作

就生成文件了,tmp是正在滚动的文件,当到达设置的条件10分钟或者128兆的时候,就会产生新的文件出来.