使用mina2框架编写分布式日志服务器
日志服务器,那游戏服务器为实例,重要的数据库服务器,游戏服务器,网关服务器...往往不在意日志服务器,而通常情况下,日志简单的通过log4j,c下的log4c,cpp下的log4cpp等日志工具,当然,这类日志工具是主流的开源框架,性能,功能面都不错,但如果,我们需要记录几乎每个玩家,没个操作,那么这类在应用程序中添加的日志记录将会消耗大量系统资源,导致服务器卡慢.顾,分布式日志服务器在某些项目架构下非常有意义.
我的分布式服务器,分为3个版本,c语言版本,原生java nio版本,以及依赖mina2框架实现,由于mina2框架的如日中天,相信,这篇文章对于java编程爱好者有一定帮助.
请看源码:
IoHandler的编写.log4j记录错误等日志,而LogWriter为真正的高频度,大数据日志记录工具.
package cn.vicky.mina.logserver;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import org.apache.mina.core.buffer.IoBuffer;
import org.apache.mina.core.service.IoHandler;
import org.apache.mina.core.session.IdleStatus;
import org.apache.mina.core.session.IoSession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* @author Vicky.H
*/
public class LogIoHandler implements IoHandler {
private static final Logger LOG = LoggerFactory.getLogger(LogIoHandler.class);
private Set sessions = Collections.synchronizedSet(new HashSet(10));
// 日志写入器
private LogWriter logWriter;
public LogIoHandler(LogWriter logWriter) {
this.logWriter = logWriter;
}
public void sessionCreated(IoSession session) throws Exception {
// Do nothing...
}
public void sessionOpened(IoSession session) throws Exception {
sessions.add(session);
}
public void sessionClosed(IoSession session) throws Exception {
sessions.remove(session);
}
public void sessionIdle(IoSession session, IdleStatus status) throws Exception {
// Do nothing...
}
public void exceptionCaught(IoSession session, Throwable cause) throws Exception {
LOG.error("exceptionCaught:", cause);
}
public void messageReceived(IoSession session, Object message) throws Exception {
if (message instanceof IoBuffer) { // 确认数据类型
logWriter.addLog((IoBuffer) message);
}
}
public void messageSent(IoSession session, Object message) throws Exception {
// Do nothing...
}
}
LogWriter 其核心思想是,通过直接缓存区,以及缓存区交换区方式实现数据存储以及写入文件的高性能,高并发,并且通过5分钟左右的写入文件形式,在对日志时间要求不是非常精确下,可以直接通过日志文件名称方式,获得日志范围,可以精简日志文件大小,减少性能消耗.至于此处采用小文件方式记录日志,还有个原因就是方便数据统计.我在日志处理方面,没有编写额外的程序,通过正则表达式等方式拆解,而是通过linux shell命令拆解文件
package cn.vicky.mina.logserver;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import org.apache.mina.core.buffer.IoBuffer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* @author Vicky.H
* 日志写入定时器
*/
public class LogWriter extends Thread {
private static final Logger LOG = LoggerFactory.getLogger(LogWriter.class);
private static final DateFormat DF = new SimpleDateFormat("yyMMddHHmmss"); // 日志时间格式化
private final Object syncObj = new Object(); // 缓冲区同步锁
// 直接缓冲区，它与当前操作系统能够更好的耦合，因此能进一步提高I/O操作的速度。
// 但是分配直接缓冲区的系统开销很大，因此只有在缓冲区较大并且长期存在，或者需要经常重用时，才使用这种缓冲区。
// 故,此处使用直接缓冲区非常合适
private ByteBuffer logcontent;
private ByteBuffer logcontentswap;
private boolean isRunning = true; // 日志线程运行状态
private long interval = 180000; // 180秒间隔
private int capacity = 1024 * 1024 * 18; // 缓冲区大小,预留足够,大约60秒6M
public LogWriter () {
this.logcontent = ByteBuffer.allocateDirect(capacity); // 10M缓冲区
this.logcontentswap = ByteBuffer.allocateDirect(capacity); // 10M缓冲交换区
}
/**
* 设置日志写入工具
* @param capacity 缓冲区大小
* @param interval 写入间隔(最好低于5分钟)
* 缓冲区大小需要预留足够 大约100秒1M capacity = interval / 10000
*/
public LogWriter (int capacity,long interval) {
this.logcontent = ByteBuffer.allocateDirect(capacity);
this.logcontentswap = ByteBuffer.allocateDirect(capacity);
this.interval = interval;
}
public long getInterval() {
return interval;
}
/**
* 该值默认为3分钟,由于通过文件创建时间方式和较短的时间间隔,在不精确每条日志时间的基础上
* 便可以通过日志文件名的方式,大致获得每条日志的记录时间.
* @param interval
*/
public void setInterval(long interval) {
this.interval = interval;
}
public void addLog(IoBuffer log) {
synchronized (syncObj) { // Buffer是非线程安全的
logcontent.put(log.buf()); // 写入日志内容
}
}
public void shutdown() {
LOG.info("关闭日志记录器");
this.isRunning = false;
}
@Override
public void run() {
LOG.info("开启日志记录器");
while (isRunning) {
LOG.info(new Date() + "写入日志");
if (logcontent.position() == 0) {
LOG.info(new Date() + "没有任何日志内容,跳过写入文件");
continue;
}
try {
Thread.sleep(interval);
} catch (InterruptedException e) {
LOG.error("线程休眠异常!", e);
}
try // 处理tempcontent
{
File file = new File("c:/log_" + DF.format(new Date()) + ".log");
file.createNewFile();
FileOutputStream fos = new FileOutputStream(file);
FileChannel fileChannel = fos.getChannel();
synchronized (syncObj) {
// 设置临时缓冲区指向当前接受数据的缓冲区
ByteBuffer tempcontent = logcontent;
// 当前缓冲区指向缓冲区交换区
logcontent = logcontentswap;
// 缓冲区交换区指向原来的缓冲区地址
logcontentswap = tempcontent;
}
logcontentswap.flip();
fileChannel.write(logcontentswap, logcontentswap.position());
fileChannel.close();
fos.close();
// 处理完毕后,重置缓冲区
logcontentswap.clear();
} catch (IOException e) {
LOG.error("将日志写入文件异常!", e);
}
}
}
}
简单的测试:
package cn.vicky.mina.logserver;
import java.util.Date;
import java.util.Scanner;
import org.apache.mina.core.service.IoAcceptor;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
/**
*
* @author Vicky.H
*/
public final class MainApp {
public static void main(String args[]) {
if (System.getProperty("com.sun.management.jmxremote") != null) {
System.out.println("JMX enabled.");
} else {
System.out.println("JMX disabled. Please set the "
+ "'com.sun.management.jmxremote' system property to enable JMX.");
}
ConfigurableApplicationContext applicationContext = new ClassPathXmlApplicationContext("cn/vicky/mina/logserver/LogServerContext.xml");
System.out.println("Listening ...");
IoAcceptor ioAcceptor = (IoAcceptor) applicationContext.getBean("ioAcceptor");
ioAcceptor.setCloseOnDeactivation(false); // 设置unbind()相应方式
// TODO 开启测试
class TestThread extends Thread {
@Override
public void run() {
new TestClient().test();
}
}
TestThread thread1 = new TestThread();
TestThread thread2 = new TestThread();
TestThread thread3 = new TestThread();
TestThread thread4 = new TestThread();
TestThread thread5 = new TestThread();
thread1.start();
thread2.start();
thread3.start();
thread4.start();
thread5.start();
System.out.println("输入exit关闭服务器!");
Scanner scanner = new Scanner(System.in);
String command = scanner.nextLine();
while (!command.equals("exit")) {
command = scanner.nextLine();
}
if (ioAcceptor.isCloseOnDeactivation()) {
applicationContext.close(); // 关闭容器
System.out.println("一旦unbind()将关闭所有连接");
} else {
// 需要2个步骤才能关闭服务器
applicationContext.close();
System.out.println("一旦unbind()将阻止之后的所有连接");
ioAcceptor.dispose(true);
System.out.println("关闭服务器");
}
System.out.println("服务器将在1分钟后完全关闭!" + new Date());
}
}
测试客户端
package cn.vicky.mina.logserver;
import java.net.InetSocketAddress;
import java.nio.charset.Charset;
import org.apache.mina.core.filterchain.DefaultIoFilterChainBuilder;
import org.apache.mina.core.filterchain.IoFilter;
import org.apache.mina.core.future.ConnectFuture;
import org.apache.mina.core.service.IoConnector;
import org.apache.mina.core.service.IoHandler;
import org.apache.mina.core.session.IdleStatus;
import org.apache.mina.core.session.IoSession;
import org.apache.mina.core.session.IoSessionConfig;
import org.apache.mina.filter.codec.ProtocolCodecFactory;
import org.apache.mina.filter.codec.ProtocolCodecFilter;
import org.apache.mina.filter.codec.textline.LineDelimiter;
import org.apache.mina.filter.codec.textline.TextLineCodecFactory;
import org.apache.mina.transport.socket.nio.NioSocketConnector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* @author Vicky.H
*/
public class TestClient {
public void test() {
final IoConnector connector = new NioSocketConnector();
// 2.
IoSessionConfig sessionConfig = connector.getSessionConfig();
sessionConfig.setReaderIdleTime(60000);
sessionConfig.setReadBufferSize(1024 * 2);
// 3
DefaultIoFilterChainBuilder filterChain = connector.getFilterChain();
ProtocolCodecFactory codecFactory = new TextLineCodecFactory(Charset.forName("UTF-8"),
LineDelimiter.WINDOWS.getValue(), LineDelimiter.WINDOWS.getValue());
IoFilter filter = new ProtocolCodecFilter(codecFactory);
filterChain.addLast("codec", filter);
// 5
connector.setHandler(new IoHandler() {
Logger LOGGER = LoggerFactory.getLogger(IoHandler.class);
public void sessionCreated(IoSession session) throws Exception {
LOGGER.debug("创建一个会话链接（未连接）");
}
public void sessionOpened(IoSession session) throws Exception {
LOGGER.debug("服务器与客户端会建立连接");
}
public void sessionClosed(IoSession session) throws Exception {
LOGGER.debug("服务器与客户端会话链接已经关闭");
}
public void sessionIdle(IoSession session, IdleStatus status) throws Exception {
LOGGER.debug("客户端端进入空闲状态");
session.close(false);
}
public void exceptionCaught(IoSession session, Throwable cause) throws Exception {
LOGGER.error("Error:客户端捕获到异常", cause);
session.close(true);
}
public void messageReceived(IoSession session, Object message) throws Exception {
LOGGER.debug("接收到服务器端消息:" + message);
}
/**
* 注意这里是消息发送后回调，非发送消息！！！
*/
public void messageSent(IoSession session, Object message) throws Exception {
// 发送返回消息后就断开与客户端的链接，这就类似HTTP请求一样的短链接模式
// session.close(true)
}
});
// 6
ConnectFuture connectFuture = connector.connect(new InetSocketAddress("192.168.1.77", 1234));
// 7
connectFuture.awaitUninterruptibly();
IoSession session = connectFuture.getSession();
System.out.println("连接成功!");
for (int i = 0; i < 999; i++) {
session.write("测试日志服务器性能!!! " + i);
try {
Thread.sleep(30);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
public static void main(String args[]) {
new TestClient().test();
}
}
关于SHELL拆解日志文件:
pid:72356 item:11217 num:1 cast:2
pid:72529 item:11217 num:1 cast:2
pid:52567 item:262 num:1 cast:20
pid:52567 item:262 num:1 cast:20
pid:72539 item:11217 num:1 cast:2
pid:72151 item:11217 num:1 cast:2
pid:70569 item:11318 num:3 cast:48
如以上内容的日志文件,shell脚本:
#!/bin/bash
# 根据日志文件统计出item:11218 的购买次数以及购买数量
for item in 11218 11219 11217 230
do
awk '$4~/item:'$item'/{count+=1;sub("num:","",$5);sum+=$5}END{print '$item',count == "" ? " 0 " : " "count" ",sum == "" ? 0 : sum}' log.2012-0*(日期范围) >> log.txt
done