1 现象
今天在uat环境压力测试的时候报如下错误
java.lang.RuntimeException: Timed out waiting for operation at net.spy.memcached.internal.OperationFuture.get(OperationFuture.java:135) at org.jasig.cas.ticket.registry.MemCacheTicketRegistry.updateTicket(MemCacheTicketRegistry.java:109) at org.jasig.cas.ticket.registry.AbstractDistributedTicketRegistry$TicketDelegator.updateTicket(AbstractDistributedTicketRegistry.java:73) at org.jasig.cas.ticket.registry.AbstractDistributedTicketRegistry$TicketGrantingTicketDelegator.grantServiceTicket_aroundBody2(AbstractDistributedTicketRegistry.java:167) at org.jasig.cas.ticket.registry.AbstractDistributedTicketRegistry$TicketGrantingTicketDelegator.grantServiceTicket_aroundBody3$advice(AbstractDistributedTicketRegistry.java:57) at org.jasig.cas.ticket.registry.AbstractDistributedTicketRegistry$TicketGrantingTicketDelegator.grantServiceTicket(AbstractDistributedTicketRegistry.java:1) at org.jasig.cas.CentralAuthenticationServiceImpl.grantServiceTicket_aroundBody2(CentralAuthenticationServiceImpl.java:243) at org.jasig.cas.CentralAuthenticationServiceImpl.grantServiceTicket_aroundBody3$advice(CentralAuthenticationServiceImpl.java:57) at org.jasig.cas.CentralAuthenticationServiceImpl.grantServiceTicket(CentralAuthenticationServiceImpl.java:1) at org.jasig.cas.CentralAuthenticationServiceImpl.grantServiceTicket_aroundBody4(CentralAuthenticationServiceImpl.java:276) at org.jasig.cas.CentralAuthenticationServiceImpl.grantServiceTicket_aroundBody5$advice(CentralAuthenticationServiceImpl.java:57) at org.jasig.cas.CentralAuthenticationServiceImpl.grantServiceTicket(CentralAuthenticationServiceImpl.java:1) at sun.reflect.GeneratedMethodAccessor63.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:318) at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150) at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:80) at org.perf4j.aop.AbstractTimingAspect$1.proceed(AbstractTimingAspect.java:47) at org.perf4j.aop.AgnosticTimingAspect.runProfiledMethod(AgnosticTimingAspect.java:53) at org.perf4j.aop.AbstractTimingAspect.doPerfLogging(AbstractTimingAspect.java:45) at sun.reflect.GeneratedMethodAccessor55.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:621) at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:610) at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:65) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161) at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:80) at com.github.inspektr.audit.AuditTrailManagementAspect.handleAuditTrail(AuditTrailManagementAspect.java:126) at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:621) at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:610) at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:65) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:90) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172) at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:202) at com.sun.proxy.$Proxy25.grantServiceTicket(Unknown Source) at org.jasig.cas.web.flow.GenerateServiceTicketAction.doExecute(GenerateServiceTicketAction.java:52) at org.springframework.webflow.action.AbstractAction.execute(AbstractAction.java:188) at org.springframework.webflow.execution.ActionExecutor.execute(ActionExecutor.java:51) at org.springframework.webflow.action.EvaluateAction.doExecute(EvaluateAction.java:77) at org.springframework.webflow.action.AbstractAction.execute(AbstractAction.java:188) at org.springframework.webflow.execution.AnnotatedAction.execute(AnnotatedAction.java:145) at org.springframework.webflow.execution.ActionExecutor.execute(ActionExecutor.java:51) at org.springframework.webflow.engine.ActionState.doEnter(ActionState.java:101) at org.springframework.webflow.engine.State.enter(State.java:194) at org.springframework.webflow.engine.Transition.execute(Transition.java:227) at org.springframework.webflow.engine.DecisionState.doEnter(DecisionState.java:51) at org.springframework.webflow.engine.State.enter(State.java:194) at org.springframework.webflow.engine.Transition.execute(Transition.java:227) at org.springframework.webflow.engine.impl.FlowExecutionImpl.execute(FlowExecutionImpl.java:393) at org.springframework.webflow.engine.impl.RequestControlContextImpl.execute(RequestControlContextImpl.java:214) at org.springframework.webflow.engine.TransitionableState.handleEvent(TransitionableState.java:119) at org.springframework.webflow.engine.Flow.handleEvent(Flow.java:555) at org.springframework.webflow.engine.impl.FlowExecutionImpl.handleEvent(FlowExecutionImpl.java:388) at org.springframework.webflow.engine.impl.RequestControlContextImpl.handleEvent(RequestControlContextImpl.java:210) at org.springframework.webflow.engine.ActionState.doEnter(ActionState.java:105) at org.springframework.webflow.engine.State.enter(State.java:194) at org.springframework.webflow.engine.Transition.execute(Transition.java:227) at org.springframework.webflow.engine.impl.FlowExecutionImpl.execute(FlowExecutionImpl.java:393) at org.springframework.webflow.engine.impl.RequestControlContextImpl.execute(RequestControlContextImpl.java:214) at org.springframework.webflow.engine.TransitionableState.handleEvent(TransitionableState.java:119) at org.springframework.webflow.engine.Flow.handleEvent(Flow.java:555) at org.springframework.webflow.engine.impl.FlowExecutionImpl.handleEvent(FlowExecutionImpl.java:388) at org.springframework.webflow.engine.impl.RequestControlContextImpl.handleEvent(RequestControlContextImpl.java:210) at org.springframework.webflow.engine.ActionState.doEnter(ActionState.java:105) at org.springframework.webflow.engine.State.enter(State.java:194) at org.springframework.webflow.engine.Transition.execute(Transition.java:227) at org.springframework.webflow.engine.impl.FlowExecutionImpl.execute(FlowExecutionImpl.java:393) at org.springframework.webflow.engine.impl.RequestControlContextImpl.execute(RequestControlContextImpl.java:214) at org.springframework.webflow.engine.TransitionableState.handleEvent(TransitionableState.java:119) at org.springframework.webflow.engine.Flow.handleEvent(Flow.java:555) at org.springframework.webflow.engine.impl.FlowExecutionImpl.handleEvent(FlowExecutionImpl.java:388) at org.springframework.webflow.engine.impl.RequestControlContextImpl.handleEvent(RequestControlContextImpl.java:210) at org.springframework.webflow.engine.ViewState.handleEvent(ViewState.java:232) at org.springframework.webflow.engine.ViewState.resume(ViewState.java:196) at org.springframework.webflow.engine.Flow.resume(Flow.java:545) at org.springframework.webflow.engine.impl.FlowExecutionImpl.resume(FlowExecutionImpl.java:261) at org.springframework.webflow.executor.FlowExecutorImpl.resumeExecution(FlowExecutorImpl.java:169) at org.springframework.webflow.mvc.servlet.FlowHandlerAdapter.handle(FlowHandlerAdapter.java:183) at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:923) at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:852) at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:882) at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:789) at javax.servlet.http.HttpServlet.service(HttpServlet.java:650) at javax.servlet.http.HttpServlet.service(HttpServlet.java:731) at org.jasig.cas.web.init.SafeDispatcherServlet.service_aroundBody2(SafeDispatcherServlet.java:128) at org.jasig.cas.web.init.SafeDispatcherServlet.service_aroundBody3$advice(SafeDispatcherServlet.java:57) at org.jasig.cas.web.init.SafeDispatcherServlet.service(SafeDispatcherServlet.java:1) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:88) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:259) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) at com.github.inspektr.common.web.ClientInfoThreadLocalFilter.doFilter(ClientInfoThreadLocalFilter.java:63) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:219) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:110) at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:498) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103) at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:1025) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:445) at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1115) at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:637) at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1775) at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1734) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) at java.lang.Thread.run(Thread.java:745) Caused by: net.spy.memcached.internal.CheckedOperationTimeoutException: Timed out waiting for operation - failing node: /10.10.137.107:11211 at net.spy.memcached.internal.OperationFuture.get(OperationFuture.java:160) at net.spy.memcached.internal.OperationFuture.get(OperationFuture.java:133) ... 123 more
2，故障分析
网上搜索了半天，之前压力测试的时候也没有报这样的错误，但是今天出现了。看错误信息,发现是get操作的时候报time out，有可能原因是大量数据并发的时候，有进程一直等待，或者就是并发数量不够
3，处理过程
通过ps -ef|grep memcached 查看memcached进程
app_use+ 12162 1 0 15:07 ? 00:00:00 ./memcached -d -u memcached -p 11211 -m 1024 -c 1024
发现并发连接数是1024是默认值
故调大这个参数为2048试试，结果不再报类似错误了。
4，后续措施
针对此问题，可以调大并发连接数，或者可以考虑使用集群，增加memcached的可用性