一次线上异常 Connection reset

背景

线上项目一开始是运行一段时间就需要重启，后来重启越来越频繁，查日志也没有任何发现。后来只能开着运行日志看是什么情况，基本经常会看Connection reset 异常，如下：

ClientAbortException: java.net.SocketException: Connection reset

        at org.apache.catalina.connector.OutputBuffer.doFlush(OutputBuffer.java:319)

        at org.apache.catalina.connector.OutputBuffer.flush(OutputBuffer.java:288)

        at org.apache.catalina.connector.CoyoteOutputStream.flush(CoyoteOutputStream.java:98)

        at com.allinpay.fps.fundtxinteract.fsi.controller.ControllerProcessMS11.sendRespToFSI(ControllerProcessMS11.java:230)

        at com.allinpay.fps.fundtxinteract.fsi.controller.ControllerProcessMS11.action00(ControllerProcessMS11.java:173)

        at sun.reflect.GeneratedMethodAccessor97.invoke(Unknown Source)

        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

        at java.lang.reflect.Method.invoke(Method.java:585)

        at org.springframework.web.servlet.mvc.multiaction.MultiActionController.invokeNamedMethod(MultiActionController.java:473)

        at com.hitrust.hap.web.servlet.mvc.multiaction.HapMultiActionController.handleRequestInternal(HapMultiActionController.java:773)

        at org.springframework.web.servlet.mvc.AbstractController.handleRequest(AbstractController.java:153)

        at org.springframework.web.servlet.mvc.SimpleControllerHandlerAdapter.handle(SimpleControllerHandlerAdapter.java:48)

        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:875)

        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:809)

        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:523)

        at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:463)

        at javax.servlet.http.HttpServlet.service(HttpServlet.java:710)

        at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)

        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)

        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)

        at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:654)

        at org.apache.catalina.core.ApplicationDispatcher.processRequest(ApplicationDispatcher.java:445)

        at org.apache.catalina.core.ApplicationDispatcher.doForward(ApplicationDispatcher.java:379)

        at org.apache.catalina.core.ApplicationDispatcher.forward(ApplicationDispatcher.java:292)

        at com.allinpay.fps.fundtxinteract.fsi.receiver.FSIRequestReceiver.doPost(FSIRequestReceiver.java:217)

        at javax.servlet.http.HttpServlet.service(HttpServlet.java:710)

        at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)

        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)

        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)

        at com.hitrust.hap.web.filter.HapLocaleFilter.doFilterInternal(HapLocaleFilter.java:85)

        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76)

        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)

        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)

        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:96)

        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76)

        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)

        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)

        at org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:96)

        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)

        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)

        at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:230)

        at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175)

        at org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:182)

        at org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:84)

        at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)

        at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)

        at org.jboss.web.tomcat.service.jca.CachedConnectionValve.invoke(CachedConnectionValve.java:157)

        at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)

        at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:262)

        at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:844)

        at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:583)

        at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:446)

        at java.lang.Thread.run(Thread.java:595)

Caused by: java.net.SocketException: Connection reset

        at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:96)

        at java.net.SocketOutputStream.write(SocketOutputStream.java:136)

        at org.apache.coyote.http11.InternalOutputBuffer.realWriteBytes(InternalOutputBuffer.java:737)

        at org.apache.tomcat.util.buf.ByteChunk.flushBuffer(ByteChunk.java:434)

        at org.apache.coyote.http11.InternalOutputBuffer.flush(InternalOutputBuffer.java:299)

        at org.apache.coyote.http11.Http11Processor.action(Http11Processor.java:963)

        at org.apache.coyote.Response.action(Response.java:183)

        at org.apache.catalina.connector.OutputBuffer.doFlush(OutputBuffer.java:314)

        ... 52 more

在这篇文章中找到原因

结合这篇文章可以知道是由于某个请求内部处理时间过长导致 socket 中断，根据错误日志看出是在哪个具体接口处理时间过长。
我这边是一个日志记录接口，代码并不复杂， 但是在记录日志之前会有一个查询操作，看了一下日志，有将近两亿条，查询一次至少需要两分钟，而且听前端说这个日志记录接口会经常使用。

原因

可以得出结论，导致系统崩溃的是很多线程阻塞在服务端无法释放，后期会越来越快的导致线程阻塞。

处理方法

清理数据库日志表，以及将查询语句删除。