package com.xxxx.flume.sink;
import org.apache.flume.*;
import org.apache.flume.conf.Configurable;
import org.apache.flume.sink.AbstractSink;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class MySink extends AbstractSink implements Configurable {
private Logger logger = LoggerFactory.getLogger(MySink.class);
private String prefix;
private String subfix;
//读取conf中 配置的参数
public void configure(Context context) {
this.prefix = context.getString("prefix");
this.subfix = context.getString("subfix");
}
public Status process() throws EventDeliveryException {
Status status = null;
Channel channel = getChannel();
Transaction transaction = channel.getTransaction();
transaction.begin();
try {
Event event = channel.take();
if (event != null) {
// ==== 添加业务逻辑开始 ====
String body = new String(event.getBody());
logger.warn("prefix:" + this.prefix + ",\nbody:" + body + ",\nsubfix:" + this.subfix);
String out_log_body = "prefix:" + this.prefix + ",\tbody:" + body + ",\tsubfix:" + this.subfix;
logger.warn(out_log_body);
logger.info(out_log_body);
// ==== 业务逻辑结束 ====
}
transaction.commit();
status = Status.READY;
} catch (ChannelException e) {
e.printStackTrace();
transaction.rollback();
status = Status.BACKOFF;
} finally {
transaction.close();
}
return status;
}
}
#name
a1.sources = s1
a1.channels = c1
a1.sinks = k1
# Source
a1.sources.s1.type = netcat
a1.sources.s1.bind = localhost
a1.sources.s1.port = 44449
# channels [c1]
a1.channels.c1.type = memory
a1.channels.c1.capacity = 100
a1.channels.c1.overflowCapacity = 1000
# sinks [k1 k2]
a1.sinks.k1.type = com.xxxx.flume.sink.MySink
a1.sinks.k1.prefix = xxxx__
a1.sinks.k1.subfix = __yyyy
# Bind
a1.sources.s1.channels = c1
a1.sinks.k1.channel = c1
启动flume
bin/flume-ng agent -c conf -f job/sink_test/flume1.conf --name a1 -Dflume.root.logger=INFO,console
结果如下:
# 输入
% telnet localhost 44449
...
0000000000
OK
# 输出 一个是字符串拼接有问题,一个是logger.info()输出有问题
...
2019-11-18 16:26:54,557 (SinkRunner-PollingRunner-DefaultSinkProcessor) [WARN - com.xxxx.flume.sink.MySink.process(MySink.java:42)] prefix:xxxx__,
,ody:0000000000
subfix:__yyyy
2019-11-18 16:26:54,559 (SinkRunner-PollingRunner-DefaultSinkProcessor) [WARN - com.xxxx.flume.sink.MySink.process(MySink.java:45)],prefix:subfix:__yyyy0000000000
2019-11-18 16:26:54,559 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - com.xxxx.flume.sink.MySink.process(MySink.java:46)],prefix:subfix:__yyyy0000000000
...
最后在命令行输入数据的时候发现乱码
admin@Tmp flume_interceptor % telnet localhost 44449
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
asd
OK
o<^Lo<^Lo<^Lo<^Lo<^Lo<^Lo<^L
OK
c^@^Qc^@^Qc^@^Q
OK
于是把source的类型更换为exec,在conf文件中使用
# Source
a1.sources.s1.type = exec
a1.sources.s1.command = tail -F /Users/admin/log/test.log
a1.sources.s1.shell= /bin/sh -c
重启flume的agent,在命令行写入数据到监控到文件,问题解决
输入:
echo 321321321 >> /Users/admin/log/test.log
输出总于正常了:
2019-11-18 19:11:59,872 (SinkRunner-PollingRunner-DefaultSinkProcessor) [WARN - com.babyfs.flume.sink.MySink.process(MySink.java:42)] prefix:xxxx__,
body:321321321,
subfix:__yyyy
2019-11-18 19:11:59,872 (SinkRunner-PollingRunner-DefaultSinkProcessor) [WARN - com.babyfs.flume.sink.MySink.process(MySink.java:45)] prefix:xxxx__, body:321321321, subfix:__yyyy
2019-11-18 19:11:59,873 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - com.babyfs.flume.sink.MySink.process(MySink.java:46)] prefix:xxxx__, body:321321321, subfix:__yyyy
虽然 只是学习测试用,但是不调通还是不舒服,现在舒服了。