log4j与flume整合配置(实现故障转移FailoverLog4jAppender)及问题总结
实现功能:
该功能重点在log4j端
启动两个相同的flume,log4j往flume1发数据,当flume1冗机后,可以继续往flume2中发送数据.达到故障转移,达到高可用
1、依赖jar包导入，通过maven管理，pom文件增加如下配置：
<dependency>
<groupId>org.apache.flume.flume-ng-clientsgroupId>
<artifactId>flume-ng-log4jappenderartifactId>
<version>1.6.0version>
dependency>
<dependency>
<groupId>log4jgroupId>
<artifactId>log4jartifactId>
<version>1.2.17version>
dependency>
<dependency>
<groupId>org.slf4jgroupId>
<artifactId>slf4j-apiartifactId>
<version>1.7.5version>
dependency>
<dependency>
<groupId>org.slf4jgroupId>
<artifactId>slf4j-log4j12artifactId>
<version>1.7.5version>
dependency> 2、log4j配置文件：
#log4j输出到控制台
log4j.rootLogger=INFO,stdout,writelog,flume
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target = System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%m%n
#log4j输出到flume单机模式配置
log4j.appender.flume = org.apache.flume.clients.log4jappender.Log4jAppender
log4j.appender.flume.layout=org.apache.log4j.PatternLayout
log4j.appender.flume.Hostname = 192.168.109.133
log4j.appender.flume.Port = 6501
log4j.appender.flume.UnsafeMode = true
#log4j输出到flume故障转移方式配置
log4j.appender.writelog = org.apache.flume.clients.log4jappender.FailoverLog4jAppender
log4j.appender.writelog.Hosts = m02:6501 m03:6501
log4j.appender.writelog.MaxAttempts = 2
log4j.appender.writelog.MaxIoWorkers = 2
log4j.appender.writelog.UnsafeMode = false
log4j.appender.writelog.layout=org.apache.log4j.PatternLayout3.问题
采用改方式配置打印日志时，出现循环打印出“Using default maxIOWorkers”导致栈溢出问题：
在getProperties方法中增加对maxIOWorkers初始化：
props.setProperty(RpcClientConfigurationConstants.MAX_IO_WORKERS,
(Runtime.getRuntime().availableProcessors() * 2)+"");需要重写类(FailoverLog4jAppender).重新打包
修改的如下:
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.flume.clients.log4jappender;
import org.apache.commons.lang.StringUtils;
import org.apache.flume.FlumeException;
import org.apache.flume.api.RpcClientConfigurationConstants;
import org.apache.flume.api.RpcClientFactory;
import org.apache.flume.api.RpcClientFactory.ClientType;
import org.apache.log4j.Logger;
import org.apache.log4j.helpers.LogLog;
import org.apache.log4j.spi.LoggingEvent;
import java.util.Properties;
/**
* author:edwardsbean
*/
public class FailoverLog4jAppender extends Log4jAppender {
private String hosts;
private String maxAttempts;
private String maxIoWorkers;
private boolean configured = false;
public void setHosts(String hostNames) {
this.hosts = hostNames;
}
public void setMaxAttempts(String maxAttempts) {
this.maxAttempts = maxAttempts;
}
public void setMaxIoWorkers(String maxIoWorkers) {
this.maxIoWorkers = maxIoWorkers;
}
Logger logger = Logger.getLogger(FailoverLog4jAppender.class);
/**
* 报错机制
* @param event
*/
@Override
public synchronized void append(LoggingEvent event) {
if (!configured) {
String errorMsg = "Flume Log4jAppender not configured correctly! Cannot" +
" send events to Flume.";
LogLog.error(errorMsg);
if (getUnsafeMode()) {
return;
}
//jie- 配置文件报错
throw new FlumeException(errorMsg);
}
super.append(event);
}
/**
* 重连机制
* @throws FlumeException if the FailoverRpcClient cannot be instantiated.
*/
@Override
public void activateOptions() throws FlumeException {
try {
final Properties properties = getProperties(hosts, maxAttempts, maxIoWorkers,getTimeout());
rpcClient = RpcClientFactory.getInstance(properties);
if (layout != null) {
layout.activateOptions();
}
configured = true;
} catch (Exception e) {
String errormsg = "RPC client creation failed! " + e.getMessage();
LogLog.error(errormsg);
logger.error("RPC client creation failed!" + "连接错误");
if (getUnsafeMode()) {
return;
}
throw new FlumeException(e);
}
}
//配置必要的参数 说到底,就是配置文件
/**
* jie:ps自己重写获取配置文件信息,将多个网络端口配置起,是用FAILOVER(故障转移)
* @param hosts 多个主机名
* @param maxAttempts 主机数量
* @param timeout 连接超时时间
* @return
* @throws FlumeException
*/
private Properties getProperties(String hosts, String maxAttempts,String maxIoWorkers, long timeout) throws FlumeException {
//获取多个主机名的hostsname
if (StringUtils.isEmpty(hosts)) {
throw new FlumeException("hosts must not be null");
}
Properties props = new Properties();
//获取hostsname数组,按正则切分, \\s -> 空格 \\s+ -> 多个空格
//datanode4:6501 datanode6:6501
String[] hostsAndPorts = hosts.split("\\s+");
StringBuilder names = new StringBuilder();
for (int i = 0; i < hostsAndPorts.length; i++) {
String hostAndPort = hostsAndPorts[i];
String name = "h" + i;
props.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + name,
hostAndPort);
names.append(name).append(" ");
}
props.put(RpcClientConfigurationConstants.CONFIG_HOSTS, names.toString());
//客户端连接方式: CONFIG_CLIENT_TYPE -> client.type
//DEFAULT_FAILOVER : 故障转移 (NettyAvroRpcClient(普通连接),FAILOVER(故障转移),LOADBALANCE(负载均衡),THRIFT(节约模式?))
props.put(RpcClientConfigurationConstants.CONFIG_CLIENT_TYPE,
ClientType.DEFAULT_FAILOVER.toString());
if (StringUtils.isEmpty(maxAttempts)) {
throw new FlumeException("hosts must not be null");
}
//一样
props.put(RpcClientConfigurationConstants.CONFIG_MAX_ATTEMPTS, maxAttempts);
//一样
props.setProperty(RpcClientConfigurationConstants.CONFIG_CONNECT_TIMEOUT,
String.valueOf(timeout));
//一样
props.setProperty(RpcClientConfigurationConstants.CONFIG_REQUEST_TIMEOUT,
String.valueOf(timeout));
/* props.setProperty(RpcClientConfigurationConstants.MAX_IO_WORKERS,
String.valueOf(maxIoWorkers)); */
//与slf4j框架bug 冲突
//采用负载均衡方式配置打印日志时，出现循环打印出“Using default maxIOWorkers”导致栈溢出问题
//在getProperties方法中增加对maxIOWorkers初始化
props.setProperty(RpcClientConfigurationConstants.MAX_IO_WORKERS,
(Runtime.getRuntime().availableProcessors() * 2)+"");
return props;
}
}测试代码:
package log;
import org.apache.log4j.Logger;
import java.io.IOException;
import java.net.Socket;
import java.text.SimpleDateFormat;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* Created by Administrator on 2016/11/2.
*/
public class log4j {
public static void main(String[] args) throws Exception {
Logger logger = Logger.getLogger(log4j.class);
//通过配置文件，将信息，发送到flume source中
String url = "测试数据";
System.out.println("开始发送");
logger.info("log4j7测试数据" + " thread send message on -");
thread mTh1 = new thread("A");
try {
mTh1.start();
} catch (Exception e) {
logger.info("程序出现错误,骚等片刻100S:" + e.toString());
try {
mTh1.sleep(100000);
} catch (InterruptedException e1) {
e1.printStackTrace();
}
}
}
}
class thread extends Thread {
String message = "----->testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest" +
"testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest";
Logger logger = Logger.getLogger(thread.class);
Logger bpe1 = Logger.getLogger("vehicle_exam_event_his");
Logger bpe2 = Logger.getLogger("vehicle_exam_score_his");
Logger dse1 = Logger.getLogger("device_statushis");
Logger dse2 = Logger.getLogger("device_statushis_invisible");
Logger dse3 = Logger.getLogger("device_statushis_vehicle");
Logger dse4 = Logger.getLogger("device_statushis_vehicle_invisible");
Logger dse5 = Logger.getLogger("vehicle_behavior_point");
Logger dse6 = Logger.getLogger("vehicle_behavior_process");
private String name;
public thread(String name) {
this.name = name;
}
@Override
public void run() {
Socket socket = null;
try {
socket = new Socket("datanode4",41414);
} catch (IOException e) {
e.printStackTrace();
}
SimpleDateFormat time = new SimpleDateFormat("HH:mm:ss");
String format = null;
Long currtime = null;
String test = "是否丢失 数据 log4j7测试数据";
String mess = "thread send message on -";
String what = "- : ---" + name + " for ";
int i = 0;
while (true) {
try {
if(socket.isConnected()){
currtime = System.currentTimeMillis();
format = time.format(currtime);
logger.info(test + mess + format + what + i++ + message);
System.out.println("41414端口开放");
sleep(2000);
}else {
System.out.println("没有连接");
sleep(2000);
}
} catch (Exception e) {
System.out.println("here");
System.out.println(e);
}
}
}
}