Flume-ng生产环境实践（四）实现log格式化interceptor
续上篇，由于filesink中需要使用/data/log/%{dayStr}/log-%{hourStr}%{minStr}-这样文件格式的，为了使file-sink能使用%{dayStr}这样的标签，需要在数据传输过程中，给event的header中添加对应的键值对。在flume-ng中提供了很方便的方式：
Interceptor
以下为实现的interceptor，首先使用正则表达式匹配nginx日志，如何匹配成功，则获取匹配到的数据，并且对url中的参数进行处理，最后所有日志信息都被存储在Map中。根据配置文件中需要输出的键找到对应的值，按照顺序输出为csv格式的行。
原始日志格式：
|
112.245.239.72 - - [29/Dec/2012:15:00:00 +0800] "GET /p.gif?a=1&b=2 HTTP/1.1" 200 0 "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4
.0; 4399Box.1357; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; AskTbPTV2/5.9.1.14019; 4399Box.1357)" |
最终结果：
|
1,2
|
配置信息为：
| agent.sources = source agent.channels = channel agent.sinks = sink agent.sources.source.type = exec #agent.sources.source.command = tail -n +0 -F /data/tmp/accesspvpb_2012-11-18.log agent.sources.source.command = cat /opt/nginx/logs/vvaccess_log_pipe agent.sources.source.interceptors = logformat agent.sources.source.interceptors.logformat.type = org.apache.flume.interceptor.LogFormatInterceptor$Builder agent.sources.source.interceptors.logformat.confpath = /usr/programs/flume/conf/logformat_vv.properties agent.sources.source.interceptors.logformat.dynamicprop = true agent.sources.source.interceptors.logformat.hostname = vv111 agent.sources.source.interceptors.logformat.prop.monitor.rollInterval = 100000 # The channel can be defined as follows. agent.sources.source.channels = channel agent.sinks.sink.type = avro agent.sinks.sink.hostname = 192.168.0.100 agent.sinks.sink.port = 44444 agent.sinks.sink.channel = channel # Each channel's type is defined. agent.channels.channel.type = file agent.channels.channel.checkpointDir = /data/tmpc/checkpoint agent.channels.channel.dataDirs = /data/tmpc/data agent.channels.channel.transactionCapacity = 15000 |
/usr/programs/flume/conf/logformat_vv.properties文件内容为：
| keys=a,b regexp=([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})\\s-\\s-\\s\\[([^]]+)\\]\\s\"GET\\s/p.gif\\?(.+)\\s.*\"\\s[0-9]+\\s[0-9]+\\s\"(.+)\" |
interceptor的代码：
|
package
org.apache.flume.interceptor;
import
static
org.apache.flume.interceptor.LogFormatInterceptor.Constants.
CONF_PATH
;
import
static
org.apache.flume.interceptor.LogFormatInterceptor.Constants.
DYNAMICPROP
;
import
static
org.apache.flume.interceptor.LogFormatInterceptor.Constants.
DYNAMICPROP_DFLT
;
import
static
org.apache.flume.interceptor.LogFormatInterceptor.Constants.
HOSTNAME
;
import
static
org.apache.flume.interceptor.LogFormatInterceptor.Constants.
HOSTNAME_DFLT
;
import
static
org.apache.flume.interceptor.LogFormatInterceptor.Constants.
PROPMONITORINTERVAL
;
import
static
org.apache.flume.interceptor.LogFormatInterceptor.Constants.
PROPMONITORINTERVAL_DFLT
;
import
java.io.File;
import
java.io.FileInputStream;
import
java.io.FileNotFoundException;
import
java.io.IOException;
import
java.text.ParseException;
import
java.text.SimpleDateFormat;
import
java.util.Date;
import
java.util.HashMap;
import
java.util.LinkedList;
import
java.util.List;
import
java.util.Map;
import
java.util.Properties;
import
org.apache.flume.Context;
import
org.apache.flume.Event;
import
org.apache.flume.event.EventBuilder;
import
org.apache.oro.text.regex.MalformedPatternException;
import
org.apache.oro.text.regex.MatchResult;
import
org.apache.oro.text.regex.Pattern;
import
org.apache.oro.text.regex.PatternCompiler;
import
org.apache.oro.text.regex.PatternMatcher;
import
org.apache.oro.text.regex.Perl5Compiler;
import
org.apache.oro.text.regex.Perl5Matcher;
import
org.slf4j.Logger;
import
org.slf4j.LoggerFactory;
public
class
LogFormatInterceptor
implements
Interceptor {
private
static
final
Logger
logger
= LoggerFactory
. getLogger(LogFormatInterceptor.
class
);
private
String
conf_path
=
null
;
private
boolean
dynamicProp
=
false
;
private
String
hostname
=
null
;
private
long
propLastModify
= 0;
private
long
propMonitorInterval
;
private
String
regexp
=
null
;
private
List<String>
keys
=
null
;
private
Pattern
pattern
=
null
;
private
PatternCompiler
compiler
=
null
;
private
PatternMatcher
matcher
=
null
;
private
SimpleDateFormat
sdf
=
null
;
private
SimpleDateFormat
sd
=
null
;
private
SimpleDateFormat
sh
=
null
;
private
SimpleDateFormat
sm
=
null
;
private
SimpleDateFormat
sdfAll
=
null
;
private
long
eventCount
= 0l;
public
LogFormatInterceptor(String conf_path,
boolean
dynamicProp,
String hostname,
long
propMonitorInterval) {
this
.
conf_path
= conf_path;
this
.
dynamicProp
= dynamicProp;
this
.
hostname
= hostname;
this
.
propMonitorInterval
= propMonitorInterval;
}
@Override
public
void
close () {
}
@Override
public
void
initialize () {
try
{
// 读取配置文件，初始化正在表达式和输出的key列表
File file =
new
File(
conf_path
);
propLastModify
= file.lastModified();
Properties props =
new
Properties();
FileInputStream fis;
fis =
new
FileInputStream(file);
props.load(fis);
regexp
= props.getProperty(
"regexp"
);
String strKey = props.getProperty(
"keys"
);
if
(strKey !=
null
) {
String[] strkeys = strKey.split(
","
);
keys
=
new
LinkedList<String>();
for
(String key : strkeys) {
keys
.add(key);
}
}
if
(
keys
==
null
) {
logger
.error(
"====================keys is null===================="
);
}
else
{
logger
.info(
"keys="
+
keys
);
}
if
(
regexp
==
null
) {
logger
.error(
"====================regexp is null===================="
);
}
else
{
logger
.info(
"regexp="
+
regexp
);
}
// 初始化正在表达式以及时间格式化类
compiler
=
new
Perl5Compiler();
pattern
=
compiler
.compile(
regexp
);
matcher
=
new
Perl5Matcher();
sdf
=
new
SimpleDateFormat(
"dd/MMM/yyyy:HH:mm:ss Z"
,
java.util.Locale.
US
);
sd
=
new
SimpleDateFormat(
"yyyyMMdd"
);
sh
=
new
SimpleDateFormat(
"HH"
);
sm
=
new
SimpleDateFormat(
"mm"
);
sdfAll
=
new
SimpleDateFormat(
"yyyyMMddHHmmss"
);
}
catch
(MalformedPatternException e) {
logger
.error(
"Could not complile pattern!"
, e);
}
catch
(FileNotFoundException e) {
logger
.error(
"conf file is not found!"
, e);
}
catch
(IOException e) {
logger
.error(
"conf file can not be read!"
, e);
}
}
@Override
public
Event intercept(Event event) {
++
eventCount
;
try
{
if
(
dynamicProp
&&
eventCount
>
propMonitorInterval
) {
File file =
new
File(
conf_path
);
if
(file.lastModified() >
propLastModify
) {
propLastModify
= file.lastModified();
Properties props =
new
Properties();
FileInputStream fis;
fis =
new
FileInputStream(file);
props.load(fis);
String strKey = props.getProperty(
"keys"
);
if
(strKey !=
null
) {
String[] strkeys = strKey.split(
","
);
List<String> keystmp =
new
LinkedList<String>();
for
(String key : strkeys) {
keystmp.add(key);
}
if
(keystmp.size() >
keys
.size()) {
keys
= keystmp;
logger
.info(
"dynamicProp status updated = "
+
keys
);
}
else
{
logger
.error(
"dynamicProp status new keys size less than old,so status update fail = "
+
keys
);
}
}
else
{
logger
.error(
"dynamicProp status get keys fail ,so status update fail = "
+
keys
);
}
}
}
Map<String, String> headers = event.getHeaders();
headers.put(
"host"
,
hostname
);
String body =
new
String(event.getBody());
if
(
pattern
!=
null
) {
StringBuffer stringBuffer =
new
StringBuffer();
Date date =
null
;
Map<String, String> index =
new
HashMap<String, String>();
if
(
matcher
.contains(body,
pattern
)) {
index.put(
"host"
,
hostname
);
MatchResult result =
matcher
.getMatch();
index.put(
"ip"
, result.group(1));
try
{
date =
sdf
.parse(result.group(2));
index.put(
"loc_time"
,
sdfAll
.format(date));
}
catch
(ParseException e1) {
}
String url = result.group(3).replaceAll(
","
,
"|"
);
String[] params = url.split(
"&"
);
for
(String param : params) {
String[] p = param.split(
"="
);
if
(p.
length
== 2) {
index.put(p[0], p[1]);
}
}
index.put(
"browser"
, result.group(4).replaceAll(
","
,
"|"
));
for
(String key :
keys
) {
if
(index.containsKey(key)) {
stringBuffer.append(index.get(key) +
","
);
}
else
{
stringBuffer.append(
"~,"
);
}
}
if
(stringBuffer.length() > 0) {
stringBuffer.deleteCharAt(stringBuffer.length() - 1);
}
else
{
stringBuffer.append(
"error="
+ body);
}
if
(date !=
null
) {
headers.put(
"dayStr"
,
sd
.format(date));
headers.put(
"hourStr"
,
sh
.format(date));
Integer m = Integer.parseInt(
sm
.format(date));
String min =
""
;
if
(m >= 0 && m < 10) {
min =
"0"
+ (m / 5) * 5;
}
else
{
min = (m / 5) * 5 +
""
;
}
headers.put(
"minStr"
, min);
}
else
{
headers.put(
"dayStr"
,
"errorLog"
);
}
Event e = EventBuilder.withBody(stringBuffer.toString()
.getBytes(), headers);
return
e;
}
}
}
catch
(Exception e) {
logger
.error(
"LogFormat error!"
, e);
}
return
null
;
}
@Override
public
List<Event> intercept(List<Event> events) {
List<Event> list =
new
LinkedList<Event>();
for
(Event event : events) {
Event e = intercept(event);
if
(e !=
null
) {
list.add(e);
}
}
return
list;
}
/**
* Builder which builds new instances of the HostInterceptor.
*/
public
static
class
Builder
implements
Interceptor.Builder {
private
String
confPath
;
private
boolean
dynamicProp
;
private
String
hostname
;
private
long
propMonitorInterval
;
@Override
public
Interceptor build() {
return
new
LogFormatInterceptor(
confPath
,
dynamicProp
,
hostname
,
propMonitorInterval
);
}
@Override
public
void
configure(Context context) {
confPath
= context.getString(
CONF_PATH
);
dynamicProp
= context.getBoolean(
DYNAMICPROP
,
DYNAMICPROP_DFLT
);
hostname
= context.getString(
HOSTNAME
,
HOSTNAME_DFLT
);
propMonitorInterval
= context.getLong(
PROPMONITORINTERVAL
,
PROPMONITORINTERVAL_DFLT
);
}
}
public
static
class
Constants {
public
static
String
CONF_PATH
=
"confpath"
;
public
static
String
DYNAMICPROP
=
"dynamicprop"
;
public
static
boolean
DYNAMICPROP_DFLT
=
false
;
public
static
String
HOSTNAME
=
"hostname"
;
public
static
String
HOSTNAME_DFLT
=
"hostname"
;
public
static
String
PROPMONITORINTERVAL
=
"prop.monitor.rollInterval"
;
public
static
long
PROPMONITORINTERVAL_DFLT
= 500000l;
}
}
|
至此，获取nginx日志，进行格式化清洗，传输到collector机器，按照格式化的目录和文件名进行输出全部完成