[root@master ~]# flume-ng help
Usage: /usr/flume/apache-flume-1.9.0-bin/bin/flume-ng [options]...
commands:
help display this help text
agent run a Flume agent
avro-client run an avro Flume client
version show Flume version info
global options:
--conf,-c use configs in directory
--classpath,-C append to the classpath
--dryrun,-d do not actually start Flume, just print the command
--plugins-path colon-separated list of plugins.d directories. See the
plugins.d section in the user guide for more details.
Default: $FLUME_HOME/plugins.d
-Dproperty=value sets a Java system property value
-Xproperty=value sets a Java -X option
agent options:
--name,-n the name of this agent (required)
--conf-file,-f specify a config file (required if -z missing)
--zkConnString,-z specify the ZooKeeper connection to use (required if -f missing)
--zkBasePath,-p specify the base path in ZooKeeper for agent configs
--no-reload-conf do not reload config file if changed
--help,-h display help text
avro-client options:
--rpcProps,-P RPC client properties file with server connection params
--host,-H hostname to which events will be sent
--port,-p port of the avro source
--dirname directory to stream to avro source
--filename,-F text file to stream to avro source (default: std input)
--headerFile,-R File containing event headers as key/value pairs on each new line
--help,-h display help text
Either --rpcProps or both --host and --port must be specified.
Note that if directory is specified, then it is always included first
in the classpath.
[root@master Flume]# vi log-example.conf
# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# Describe/configure the source
a1.sources.r1.type = exec
a1.sources.r1.command = tail -F /var/log/secure
a1.sources.r1.channels = c1
# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
# Describe the sink
a1.sinks.k1.type = hdfs
a1.sinks.k1.channel = c1
a1.sinks.k1.hdfs.path =hdfs://master:9000/1daoyun/file/flume
a1.sinks.k1.hdfs.filePrefix = xiandian-sec
a1.sinks.k1.hdfs.round = true
a1.sinks.k1.hdfs.roundValue = 10
a1.sinks.k1.hdfs.roundUnit = minute
a1.sinks.k1.hdfs.path =hdfs://master:9000/1daoyun/file/flume中的HDFS目录要提前创建。
[root@master Flume]# flume-ng agent --conf . -f /root/tiku/Flume/log-example.conf -n a1 -Dflume.root.logger=INFO,console
[root@master ~]# hadoop fs -ls /1daoyun/file/flume
-rw-r--r-- 2 root supergroup 1237 2020-04-02 15:15 /1daoyun/file/flume/xiandian-sec.1585811701465
[root@master Flume]# vi hdfs-example.conf
# Name the components on this agent
master.sources = webmagic
master.sinks = k1
master.channels = c1
# Describe/configure the source
master.sources.webmagic.type = spooldir
master.sources.webmagic.fileHeader = true
master.sources.webmagic.fileHeaderKey = fileName
master.sources.webmagic.fileSuffix = .COMPLETED
master.sources.webmagic.deletePolicy = never
master.sources.webmagic.spoolDir = /opt/xiandian/
master.sources.webmagic.ignorePattern = ^$
master.sources.webmagic.consumeOrder = oldest
master.sources.webmagic.deserializer =org.apache.flume.sink.solr.morphline.BlobDeserializer$Builder
master.sources.webmagic.batchsize = 5
master.sources.webmagic.channels = c1
# Use a channel which buffers events in memory
master.channels.c1.type = memory
# Describe the sink
master.sinks.k1.type = hdfs
master.sinks.k1.channel = c1
master.sinks.k1.hdfs.path =hdfs://master:9000/data/flume/%{dicName}
master.sinks.k1.hdfs.filePrefix = %{fileName}
master.sinks.k1.hdfs.fileType = DataStream
master.sources.webmagic.spoolDir = /opt/xiandian/
master.sinks.k1.hdfs.path =hdfs://master:9000/data/flume/%{dicName}
在自己的节点上查看是否有这两个路径。没有则创建,前一个是file系统路径,后一个是HDFS系统路径。
[root@master Flume]# flume-ng agent --conf . -f /root/tiku/Flume/hdfs-example.conf -n master -Dflume.root.logger=INFO,console
这里的 -n 后面接的是master.sources = webmagic语句的最前面的元素。
做个小测试:
[root@master ~]# cd /opt/xiandian/
[root@master xiandian]# vi abc
aaa
bbb
ccc
[root@master xiandian]# hadoop fs -put /opt/xiandian/abc.COMPLETED /data/flume
我们再到服务端查看一下:
看一下之前服务开启的时间和做测试之后服务器的时间。有变化。
在此感谢先电云提供的题库。
感谢Apache开源技术服务支持
感谢抛物线、mn525520、菜鸟一枚2019三位博主的相关博客。