生产环境中的Tomcat 大量出现ClientAbortException
前几天生产环境的一台Tomcat 6.0的日志出频繁出现如下异常：
(业务有关代码全部用XXX代替)
ERROR 2012-11-28 22:29:57,053 [TP-Processor4056] xxx.xxx.xxx.ExceptionHandlerImpl - [SYSTEM_ERROR]/xxx/xxx/get_ext_list, errorCode=xxxxx-E001, errorMessage=exception。, params=[], userAgent="Mozilla/5.0 (Linux; U; Android 2.3.5; ja-jp; T-01D Build/F0001) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", throwable: xxx.xxx.xxx.AplRuntimeException: [xxxxxxxxx] at xxxxxxx.SigResolver.processSigApi(SigApiResolver.java:95) at org.t2framework.t2.action.impl.ActionContextBuilderImpl.resolveActionMethod(ActionContextBuilderImpl.java:226) at org.t2framework.t2.action.impl.ActionContextBuilderImpl.buildActionContext(ActionContextBuilderImpl.java:151) at org.t2framework.t2.action.impl.ActionContextBuilderImpl.build(ActionContextBuilderImpl.java:141) at org.t2framework.t2.action.impl.ActionInvokingContextImpl.buildActionContext(ActionInvokingContextImpl.java:338) at org.t2framework.t2.action.impl.PageCreationFilterImpl.invoke(PageCreationFilterImpl.java:71) at org.t2framework.t2.action.impl.ActionFilterChainImpl.invokeChain(ActionFilterChainImpl.java:108) at org.t2framework.t2.action.impl.ExceptionHandlerActionFilterImpl.invoke(ExceptionHandlerActionFilterImpl.java:79) at org.t2framework.t2.action.impl.ActionFilterChainImpl.invokeChain(ActionFilterChainImpl.java:108) at org.t2framework.t2.action.impl.ActionInvokerImpl.invoke(ActionInvokerImpl.java:121) at org.t2framework.t2.filter.T2Filter.invokeAction(T2Filter.java:254) at org.t2framework.t2.filter.T2Filter.doFilter(T2Filter.java:214) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293) at org.apache.jk.server.JkCoyoteHandler.invoke(JkCoyoteHandler.java:190) at org.apache.jk.common.HandlerRequest.invoke(HandlerRequest.java:291) at org.apache.jk.common.ChannelSocket.invoke(ChannelSocket.java:776) at org.apache.jk.common.ChannelSocket.processConnection(ChannelSocket.java:705) at org.apache.jk.common.ChannelSocket$SocketConnection.runIt(ChannelSocket.java:898) at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:690) at java.lang.Thread.run(Thread.java:662)
紧接着上面的log, 有时候会出现以下的异常
ERROR 2012-11-28 22:29:57,054 [TP-Processor4056] xxxx.DefaultExecutor - [APL_SYSTEM_ERROR]/xxx/xxx/xxx, errorCode=xxxxE001, errorMessage=xxx。, params=[cause=xxxx。], userAgent="Mozilla/5.0 (Linux; U; Android 2.3.5; ja-jp; T-01D Build/F0001) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", throwable: ClientAbortException: java.net.SocketException: Broken pipe at org.apache.catalina.connector.OutputBuffer.realWriteBytes(OutputBuffer.java:369) at org.apache.tomcat.util.buf.ByteChunk.flushBuffer(ByteChunk.java:448) at org.apache.catalina.connector.OutputBuffer.doFlush(OutputBuffer.java:318) at org.apache.catalina.connector.OutputBuffer.flush(OutputBuffer.java:296) at org.apache.catalina.connector.CoyoteOutputStream.flush(CoyoteOutputStream.java:98) at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:278) at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:122) at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:212) at xxxxx.json.OrderdJSON.format(OrderdJSON.java:920) at xxxx.DefaultExecutor.execute(DefaultExecutor.java:35) at org.t2framework.t2.filter.T2Filter.executeNavigation(T2Filter.java:277) at org.t2framework.t2.filter.T2Filter.handleNavigation(T2Filter.java:267) at org.t2framework.t2.filter.T2Filter.doFilter(T2Filter.java:215) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293) at org.apache.jk.server.JkCoyoteHandler.invoke(JkCoyoteHandler.java:190) at org.apache.jk.common.HandlerRequest.invoke(HandlerRequest.java:291) at org.apache.jk.common.ChannelSocket.invoke(ChannelSocket.java:776) at org.apache.jk.common.ChannelSocket.processConnection(ChannelSocket.java:705) at org.apache.jk.common.ChannelSocket$SocketConnection.runIt(ChannelSocket.java:898) at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:690) at java.lang.Thread.run(Thread.java:662) Caused by: java.net.SocketException: Broken pipe at java.net.SocketOutputStream.socketWrite0(Native Method) at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:92) at java.net.SocketOutputStream.write(SocketOutputStream.java:136) at org.apache.jk.common.ChannelSocket.send(ChannelSocket.java:539) at org.apache.jk.common.JkInputStream.doWrite(JkInputStream.java:162) at org.apache.coyote.Response.doWrite(Response.java:560) at org.apache.catalina.connector.OutputBuffer.realWriteBytes(OutputBuffer.java:364) ... 35 more
第一个异常出现的原因和业务逻辑有关系， filter在接到http request ，转交给后台的Java对象之前， 根据注解，要做post电文的加密check，check出错，则抛出异常。
第二个异常有趣的紧，
我们用的开源框架的Filter的代码如下：
public void doFilter(final ServletRequest request,
final ServletResponse response, final FilterChain chain)
throws IOException, ServletException {
HttpServletRequest req = (HttpServletRequest) request;
HttpServletResponse res = (HttpServletResponse) response;
final WebConfiguration webConfig = getWebConfig();
final PluginProcessor pluginProcessor = getPluginProcessor();
webConfig.setupRequestAndResponse(req, res);
pluginProcessor.beginRequest(req, res);
WebContext context = null;
Navigation navigation = null;
try {
req = pluginProcessor.createRequest(req, res);
res = pluginProcessor.createResponse(req, res);
final WebApplication webapp = webConfig.getWebApplication();
final FilterConfig filterConfig = webConfig.getFilterConfig();
context = webapp.createContext(req, res, chain, filterConfig);
if (isExcludeRequest(req)) {
handleNotT2Request(context);
return;
}
A navigation = invokeAction(context, webConfig, pluginProcessor);
B handleNavigation(navigation, context, pluginProcessor);
} catch (Throwable t) {
handleException(t, context, navigation);
} finally {
try {
pluginProcessor.endRequest(req, res);
} finally {
webConfig.endRequest();
WebContext.clear();
}
}
}
①的RuntimeException在代码的A处出现
出现之后，框架会将error信息做成http response，然后在代码的B处返回给客户端
之所以会出现②的错误，就是因为在B处，返回http response的时候，框架底层是将error信息，打成字节流，循环用byte传递回来。
如果此时， 客户端关闭浏览器， 就会关闭和服务器之间的tcp通信，此时就会出现socketException
[未完]
为什么会大量的出现这种日志呢
正常的业务操作不会出现。
我怀疑是有人写脚本，大访问量调用我们的API。 一直尝试不同的密钥加密，hacker我们的系统
脚本本身是多线程的， 调用的过程中会出现丢弃tcp connection的可能性
以上 蒋彪@南京