2019独角兽企业重金招聘Python工程师标准>>> 
环境: jdk 1.8 + spring cloud + Dalston.M1 (使用的openFeign为9.3.1)
问题描述:
微服务之间使用FeignClient调用,
针对在异步@Async注解标识的方法中请求其它微服务,间歇性出现404错误码.
在logstore日志里面,发现间歇性的,并且是非必现的场景,就让人很头疼.
在同步的方法中,生成环境的日志未出现过404日志.
at java.lang.Thread.run(Thread.java:745)
Caused by: feign.FeignException: status 404 reading XXXXClient#getXXXXX(String); content:
{"timestamp":1507687795225,"status":404,"error":"Not Found","message":"No message available","path":"/api/user/xxxx/yyyyyyy"}
at feign.FeignException.errorStatus(FeignException.java:62)
at feign.codec.ErrorDecoder$Default.decode(ErrorDecoder.java:91)
at feign.SynchronousMethodHandler.executeAndDecode(SynchronousMethodHandler.java:138)
at feign.SynchronousMethodHandler.invoke(SynchronousMethodHandler.java:76)
at feign.hystrix.HystrixInvocationHandler$1.run(HystrixInvocationHandler.java:108)
at com.netflix.hystrix.HystrixCommand$2.call(HystrixCommand.java:302)
at com.netflix.hystrix.HystrixCommand$2.call(HystrixCommand.java:298)
at rx.internal.operators.OnSubscribeDefer.call(OnSubscribeDefer.java:46)跟踪源代码,一进去,就是一大片一大片的代码,
Object executeAndDecode(RequestTemplate template) throws Throwable {
Request request = targetRequest(template);
if (logLevel != Logger.Level.NONE) {
logger.logRequest(metadata.configKey(), logLevel, request);
}
Response response;
long start = System.nanoTime();
try {
response = client.execute(request, options);
// ensure the request is set. TODO: remove in Feign 10
response.toBuilder().request(request).build();
} catch (IOException e) {
if (logLevel != Logger.Level.NONE) {
logger.logIOException(metadata.configKey(), logLevel, e, elapsedTime(start));
}
throw errorExecuting(request, e);
}
long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
boolean shouldClose = true;
try {
if (logLevel != Logger.Level.NONE) {
response =
logger.logAndRebufferResponse(metadata.configKey(), logLevel, response, elapsedTime);
// ensure the request is set. TODO: remove in Feign 10
response.toBuilder().request(request).build();
}
if (Response.class == metadata.returnType()) {
if (response.body() == null) {
return response;
}
if (response.body().length() == null ||
response.body().length() > MAX_RESPONSE_BUFFER_SIZE) {
shouldClose = false;
return response;
}
// Ensure the response body is disconnected
byte[] bodyData = Util.toByteArray(response.body().asInputStream());
return response.toBuilder().body(bodyData).build();
}
if (response.status() >= 200 && response.status() < 300) {
if (void.class == metadata.returnType()) {
return null;
} else {
return decode(response);
}
} else if (decode404 && response.status() == 404) {
return decoder.decode(response, metadata.returnType());
} else {
throw errorDecoder.decode(metadata.configKey(), response);
}
} catch (IOException e) {
if (logLevel != Logger.Level.NONE) {
logger.logIOException(metadata.configKey(), logLevel, e, elapsedTime);
}
throw errorReading(request, response, e);
} finally {
if (shouldClose) {
ensureClosed(response.body());
}
}
}
就是这鬼地方,会执行到:
throw errorDecoder.decode(metadata.configKey(), response);添加监视日志:
用来打印所有模块出现非200~300的错误码
package com.demo.core.server.filter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.*;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* 过滤特殊状态码,打印日志
*
*/
public class BadRequestFilter implements Filter {
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
private String appName ;
public BadRequestFilter(String currentAppName){
appName = currentAppName;
}
protected static List patterns = new ArrayList();
@Override
public void init(FilterConfig filterConfig) throws ServletException {
}
@Override
public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain chain) throws IOException, ServletException {
HttpServletRequest httpRequest = (HttpServletRequest) servletRequest;
HttpServletResponse httpResponse = (HttpServletResponse) servletResponse;
String url = httpRequest.getRequestURL().toString();
chain.doFilter(httpRequest, httpResponse);
if(httpResponse != null && !(httpResponse.getStatus() >= 200 && httpResponse.getStatus() < 300)) {
// appName|method|url|status
if(!StringUtils.isEmpty(httpRequest.getQueryString())){
logger.error("{}|{}|{}|{}", appName, httpRequest.getMethod(),
String.format("%s?%s",url, httpRequest.getQueryString()),
httpResponse.getStatus());
} else {
logger.error("{}|{}|{}|{}", appName, httpRequest.getMethod(), url, httpResponse.getStatus());
}
}
}
@Override
public void destroy() {
}
}
config
package com.demo.srv.user.config;
import com.demo.core.server.filter.BadRequestFilter;
import org.springframework.boot.web.servlet.FilterRegistrationBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import javax.servlet.Filter;
@Configuration
public class BadFilterConfig {
@Bean
public FilterRegistrationBean someFilterRegistration() {
FilterRegistrationBean registration = new FilterRegistrationBean();
registration.setFilter(badRequestFilter());
registration.addUrlPatterns("/*");
registration.setName("badRequestFilter");
registration.setOrder(Integer.MAX_VALUE - 10);
return registration;
}
/**
* 创建一个bean
* @return
*/
@Bean(name = "badRequestFilter")
public Filter badRequestFilter() {
return new BadRequestFilter("user");
}
}
注意, 设置当前模块为user模块,这样在日志里面可以打印出是哪个模块出现了404, 微服务之间调用没有经过gateway没地方看请求日志.
Filter的序号放到比较大.这样可以确保都最后执行该Filter.
然后使用ab压力测试,因为是post请求,所以需要在执行命令的目录下,创建一个pos.json文件.里面存放改接口需要的数据.
ab -c 100 -n 100000 -p pos.json -T "application/json" http://127.0.0.1:8092/api/user/testapi-c并发数, -n 所有并发请求总共要调用多少次
请求User微服务,
User微服务会异步调用 A-SERVICE和B-SERVICE微服务.
通过以上日志分析, 最终调用A,B微服务的地址会错乱
在请求A微服务的时候,出现一定概率请求到B微服务中,所以出现404
解决方法:
升级spring cloud : Dalston.RC1 不敢升级大版本,等时间充足测试充分再升级大版本
分析依赖,发现openFeign升级到9.4.0
https://github.com/OpenFeign/feign/releases/tag/9.4.0
压力测试后未发现404的情况.