基于springcloud服务灰度发布(一)

1 解决的问题

为了解决越来越频繁的服务更新上线，版本回退，快速迭代；服务灰度发很好地解决该问题；
该系列文章主要介绍基于springcloud实现服务灰度发布，下图大致灰度发布系统各个组成部分及路由过程；

2 灰度发布实现构成

2.1 注册中心(eureka)

不解释

2.2 网关(cloudegateway)

处理请求灰度打标：根据请求信息，如ip,用户id等信息，如果该请求符合灰度策略的用户，那么给该请求添加一个标识为灰度用户的请求头信息；

2.3 服务实例管理后台

给服务实例打标签，服务灰度策略配置；
给服务打标是怎么样实现? 据于eureka服务实及eureka客户端实现，灰度的服务标签信息主要保存于服务实例的metadata里；通过eureka的/eureka/apps/appId/instanceId/metadata?接口设置；然后eureka客户端通过定时拉取服务实例信息，可得到服务实例里的metadata信息；

2.4 负载均衡器(ribbon)

据网关处理请求头标签结果，改写路由规则；
ribbon据请求头标签信息，如何寻找对应服务？由上面可知，eureka 客户端在拉取服务信息列表时，可得服务实例里的metadata里的标签，根据该标签信息，ribbon可判断那个服务实例是正常实例，那些实例是灰度例实；由此再根据请求头灰度标签信息决定路由到那个服务实例

3 ribbon路由规则改写

3.1 修改路由规则

通过分析ribbon实例源码，发现ribbon为每个服务都会创建相应的负载均衡器及负载策略，而这些信息都在RibbonClientConfiguration去被始化，那么改写该配置类则可以修改其负载策略了；下面为改写负载策略的代码
RibbonClientGrayConfig

public class RibbonClientGrayConfig extends RibbonClientConfiguration {
    //具体某个服务名称
    @Value("${ribbon.client.name}")
    private String name = "client";
    //当前应用名称
    @Value("${spring.application.name}")
    private  String appName;

    @Autowired
    private PropertiesFactory propertiesFactory;

    @Bean
    @Primary
    @ConditionalOnMissingBean
    @Override
    public IRule ribbonRule(IClientConfig config) {
        if (this.propertiesFactory.isSet(IRule.class, name)) {
            return this.propertiesFactory.get(IRule.class, config, name);
        }
        //GrayRule自定路由策略，取代原来的策略
        GrayRule rule = new GrayRule(appName,name);
        rule.setEurekaUrls(eurekaUrls);
        rule.initWithNiwsConfig(config);
        return rule;
    }

    //注册一个feign拦截器，处理通过feign远程调用灰度标签传递
    @Bean
    GrayFeignInterceptor grayFeiginInterceptor() {
        return new GrayFeignInterceptor();
    }
}

GrayRule

public class GrayRule extends PredicateBasedRule {
    final static Logger logger = LoggerFactory.getLogger(GrayRule.class);
    private AbstractServerPredicate predicate = new GrayPredicate();
    private RoundRobinRule roundRobinRule;
    private String appName;
    private String clientName;
    private String eurekaUrls;
    public void setEurekaUrls(String eurekaUrls) {
        this.eurekaUrls = eurekaUrls;
    }
    public GrayRule() {
    }
    public GrayRule(String appName, String clientName) {
        this.appName = appName;
        this.clientName = clientName;
    }
    @Override
    public void initWithNiwsConfig(IClientConfig clientConfig) {
        super.initWithNiwsConfig(clientConfig);
        roundRobinRule = new RoundRobinRule();
    }
    @Override
    public void setLoadBalancer(ILoadBalancer lb) {
        super.setLoadBalancer(lb);
        roundRobinRule.setLoadBalancer(lb);
    }
    @Override
    public AbstractServerPredicate getPredicate() {
        return predicate;
    }
    @Override
    public Server choose(Object key) {
        boolean gray = GrayUtils.isGray();
        String routeStatus = gray ? "灰度服务" : "正常服务";
        logger.debug("[{}]将路由到[{}]的{}", appName, clientName, routeStatus);
        Server server = super.choose(gray ? ServerStatus.GRAY : ServerStatus.NORMAL);
        if (gray && server == null) {
            logger.debug("[{}]没有灰度服务实例，回退路由到正常服务", clientName);
            server = super.choose(ServerStatus.NORMAL);
        }
        if (server == null) {
            logger.error("[{}]没有{}实例,请检查服务环境当前注册中心:{}", clientName, routeStatus, eurekaUrls);
        } else {
            logger.debug("当前选择的服务[{}]:{}", clientName, server.getHostPort());
            GrayUtils.setTestServer(server);
        }
        return server;
    }
}

灰度服务判断处理

public class GrayPredicate extends AbstractServerPredicate {

    @Override
    public boolean apply(PredicateKey input) {
        Server server = input.getServer();
        ServerInstanceStatus routTo = (ServerInstanceStatus) input.getLoadBalancerKey();
        if (server instanceof DiscoveryEnabledServer) {
            DiscoveryEnabledServer enabledServer = (DiscoveryEnabledServer) server;
            InstanceInfo instanceInfo = enabledServer.getInstanceInfo();
            //从metadata得到服务是否为灰度服务
            String serverStatus = instanceInfo.getMetadata().get(Constant.INSTANCE_STATUS);
            switch (routTo) {
                case GRAY:
                    return ServerInstanceStatus.GRAY.getValue().equals(serverStatus);
                case NORMAL:
                    if (ServerInstanceStatus.GRAY.getValue().equals(serverStatus) || ServerInstanceStatus.DISABLE.getValue().equals(serverStatus)) {
                        return false;
                    }
                    return true;
                default:
                    return false;
            }
        }
        return false;
    }
}

public class GrayFeignInterceptor implements RequestInterceptor {

    @Override
    public void apply(RequestTemplate template) {
       //传递灰度标签
        template.header(Constant.ROUTE_TO_GRAY, String.valueOf(GrayUtils.isGray()));
    }
}

3.2 替换ribbon client 配置

从源码分析可知ribbon通过SpringClientFactory 为每个远程服务创建一个ribbon client相关实例信息，而SpringClientFactory 创建ribbon client默认配置类为RibbonClientConfiguration，那么只需将该类配置类替换成上面自定义RibbonClientGrayConfig 配置类，即可原配置的替换；
实现代码GrayAutoConfiguration

Configuration
public class GrayAutoConfiguration implements ApplicationContextAware, InitializingBean {

    private ApplicationContext applicationContext;

    @Autowired
    private SpringClientFactory springClientFactory;

    @Autowired(required = false)
    private ApplicationInfoManager applicationInfoManager;

    @Override
    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
        this.applicationContext = applicationContext;
    }

     @Override
    public void afterPropertiesSet() throws Exception {
        //修改负载config
        SpringClientFactory clientFactory = applicationContext.getBean(SpringClientFactory.class);
        Field defaultConfigType = NamedContextFactory.class.getDeclaredField("defaultConfigType");
        defaultConfigType.setAccessible(true);
        //自定义配置替换掉原来配置
        defaultConfigType.set(clientFactory, RibbonClientGrayConfig.class);
        //初始化应用metadata
        initGrayMetadata();
    }

    /**
     * 把contextPath保存到metadata，
     * 供网关据contextPath 自动匹配对应的服务并改写路由请求url
     */
    private void initGrayMetadata() {
        Map metadata = new HashMap<>();
        metadata.put("supportGray", "true");
        metadata.put("contextPath", contextPath);
        if (StringUtils.isEmpty(contextPath)) {
            if (StringUtils.isEmpty(contextPath2)) {
                metadata.put("contextPath", "/");
            } else {
                metadata.put("contextPath", contextPath2);
            }
        }
        if(applicationInfoManager!= null ){
            applicationInfoManager.registerAppMetadata(metadata);
        }
    }
}

4 网关(cloud-gateway)改写请求路径

4.1 覆盖原DispatcherHandler 改写请求path

为什么要改写请求path?通常在实际应用中，我们都是以请求path前辍区分为不同服务；如:
http://com.baidu.com/customer/xxx/xxx 表示请求到customer后台应用
http://com.baidu.com/merchant/xxx/xxx 表示请求到merchant后台应用
所以在实践中，网关获取到请求path 前辍作为应用的contextPath，从注册中拉取服务id及其存在metadata里的contextPath,即可据contextPath映射到具体的那个服务了

/**
 * 覆盖原DispatcherHandler 并据contextPath(requst path prefix) matcher service
 * @author xieyang 
 */
public class RewritePathDispatcherHandler extends DispatcherHandler implements ApplicationContextAware {
        //略去一些代码
	@Override
	public Mono handle(ServerWebExchange exchange) {
		if (logger.isDebugEnabled()) {
			ServerHttpRequest request = exchange.getRequest();
			logger.debug("Processing " + request.getMethodValue() + " request for [" + request.getURI() + "]");
		}
		if (this.handlerMappings == null) {
			return Mono.error(HANDLER_NOT_FOUND_EXCEPTION);
		}
		//改写请求path
		ServerWebExchange nExchange = rewritePath(exchange);
		return Flux.fromIterable(this.handlerMappings)
				.concatMap(mapping -> mapping.getHandler(nExchange))
				.next()
				.switchIfEmpty(Mono.error(HANDLER_NOT_FOUND_EXCEPTION))
				.flatMap(handler -> invokeHandler(nExchange, handler))
				.flatMap(result -> handleResult(nExchange, result));
	}

	/**
	 * 改写path
	 * @param exchange
	 * @return
	 */
	private ServerWebExchange rewritePath(ServerWebExchange exchange){
		GatewayRouteContext context =new GatewayRouteContext(exchange);
		String requestContextPath = serviceHandler.getRequestContextPath(context);
		//通常请求会有前置nginx ,为了布多套环境会加上一个环境参数
		//我们在实践过中采用不同的环境给服务加上不同前辍
		//用于区分服务所处不同环境
		String env = exchange.getRequest().getHeaders().getFirst("host_env");
		String serviceId = serviceHandler.mappingServiceIdByContextPath(requestContextPath,env);
		if(serviceId== null){
			exchange.getAttributes().put(ROUTE_CONTEXT,context);
			return exchange;
		}else {
			ServerHttpRequest req = exchange.getRequest();
			addOriginalRequestUrl(exchange, req.getURI());
			String path = req.getURI().getRawPath();
			String newPath = "/"+serviceId+path;
			ServerHttpRequest newRequest = req.mutate()
					.path(newPath)
					.build();
			ServerWebExchange newExchange = exchange.mutate().request(newRequest).build();
			newExchange.getAttributes().put(SERVICE_ID,serviceId);
			newExchange.getAttributes().put(ROUTE_CONTEXT,context);
			context.setExchange(newExchange);
			return newExchange;
		}
	}
      //省去一些代码
}

    /**
     * 换掉原来DispatherHandler
     * @return
     */
    @Bean
    @Primary
    public DispatcherHandler webHandler() {
        return new RewritePathDispatcherHandler();
    }

4.2 contextPath 映射服务Id

实际的开发中，我们可能需要会配置多套开发或测试环境，通常的做法是通过给应用加不同的前辍或后辍去表示应用属于不同的环境

@Component
public class ServiceHandler implements InitializingBean {

    private static final Logger logger = LoggerFactory.getLogger(ServiceHandler.class);

    private  ScheduledExecutorService scheduler;

    private  ThreadPoolExecutor serviceMappingExecutor;

    @Resource
    private EurekaDiscoveryClient discoveryClient;

    /**
     * key: contextPath 应对前端转发
     * value: 服务原id,无区分环境
     */
    private volatile Map contextPathServiceMap = new HashMap<>();

    /**
     * key: env 对应的环境
     * value: 服务id前辍
     */
    private volatile Map envPrefixMap = new HashMap<>();

    private Pattern pattern = Pattern.compile("[0-9]*");


    @Autowired
    private EnvironmentProperties environmentProperties;

    private void start(){
        scheduler = Executors.newScheduledThreadPool(1,
                new ThreadFactoryBuilder()
                        .setNameFormat("RefreshMapping-%d")
                        .setDaemon(true)
                        .build());
        serviceMappingExecutor = new ThreadPoolExecutor(
                1, 1, 0, TimeUnit.SECONDS,
                new SynchronousQueue(),
                new ThreadFactoryBuilder()
                        .setNameFormat("RefreshMapping-%d")
                        .setDaemon(true)
                        .build()
        );
        int expBackOffBound = 10;
        scheduler.schedule(
                new TimedSupervisorTask(
                        "heartbeat",
                        scheduler,
                        serviceMappingExecutor,
                        10,
                        TimeUnit.SECONDS,
                        expBackOffBound,
                        new RenewServiceMappingCache()
                ),
                10, TimeUnit.SECONDS);
    }

    @Override
    public void afterPropertiesSet() throws Exception {
        start();
    }


    private   class  RenewServiceMappingCache implements Runnable{
        @Override
        public void run() {
            loadContextPathServiceMapping();
            loadEnvHostMapping();
        }
    }

    /**
     * 通过请求环境参数，把服务映射到服务环境服务的
     * @param env
     * @param serviceId
     * @return
     */
    private String convertServerId2MatcherEnv(String serviceId,String env) {
        if (StringUtils.isEmpty(serviceId)) {
            return "--";
        }
        if (envPrefixMap.isEmpty()) {
            loadEnvHostMapping();
        }
        String envPrefix = envPrefixMap.get(env);
        if (envPrefix == null) {
            return serviceId;
        }
        return envPrefix + "-" + serviceId;
    }

    private  synchronized void loadContextPathServiceMapping() {
        List services = discoveryClient.getServices();
        for (String appId : services) {
            List instances = discoveryClient.getInstances(appId);
            if (instances.isEmpty()) {
                continue;
            }
            ServiceInstance instance = instances.get(0);
            EurekaDiscoveryClient.EurekaServiceInstance eurekaServiceInstance = (EurekaDiscoveryClient.EurekaServiceInstance) instance;
            InstanceInfo instanceInfo = eurekaServiceInstance.getInstanceInfo();
            String contextPath = instanceInfo.getMetadata().get("contextPath");
            if (contextPath == null) {
                logger.warn("{} 没加载到contextPath", instanceInfo.getAppName());
                continue;
            }
            if ("/".equals(contextPath)) {
                logger.warn("{} 没加载到contextPath 为 /", instanceInfo.getAppName());
                continue;
            }
            String appName = instanceInfo.getAppName();
            String[] split = appName.split("-");
            if (split == null || split.length == 0) {
                contextPathServiceMap.put(contextPath, appName);
                continue;
            }
            String serverIdPreFix = split[0];
            if (isNumeric(serverIdPreFix)) {
                contextPathServiceMap.put(contextPath, appName.substring(serverIdPreFix.length() + 1, appName.length()));
            } else {
                contextPathServiceMap.put(contextPath, appName);
            }
        }
    }

    /**
     * 获取当前请求的contextPath
     * @param context
     * @return
     */
    public String getRequestContextPath(RouteContext context) {
        String servletPath = context.getPath();
        String[] split = servletPath.split("/");
        String incomeContextPath = "/" + split[1];
        return incomeContextPath;
    }

    /**
     * 通过contextPath映射出服务id
     *
     * @param requestContextPath
     * @return
     */
    public String mappingServiceIdByContextPath(String requestContextPath,String env) {
        if (contextPathServiceMap.isEmpty()) {
            loadContextPathServiceMapping();
        }
        String serviceId = contextPathServiceMap.get(requestContextPath);
        if(serviceId == null){
            return null;
        }
        return convertServerId2MatcherEnv(serviceId,env);
    }


    private boolean isNumeric(String str){
        Matcher isNum = pattern.matcher(str);
        if( !isNum.matches() ){
            return false;
        }
        return true;
    }

    /**
     * 加载环境映射配置
     */
    private synchronized void loadEnvHostMapping() {
        envPrefixMap = environmentProperties.getHostMap();
    }
}

添加灰度路由头

public class GrayHeaderFilter implements GlobalFilter, Ordered {

    @Autowired
    private StrategyContextFactory clientContext;

    @Override
    public Mono filter(ServerWebExchange exchange, GatewayFilterChain chain) {
        GrayRouteContext context = (GrayRouteContext) exchange.getAttributes().get(ROUTE_CONTEXT);
        //据灰度策略，判断是否为走灰度服务
        boolean gray =  clientContext.get(context).isGray(context);
        if(gray){
            ServerHttpRequest request = exchange.getRequest();
            //给请求头添加灰度标标
            ServerHttpRequest newRequest = request.mutate().header(ROUTE_TO_GRAY,String.valueOf(gray)).build();
            ServerWebExchange nExchange = exchange.mutate().request(newRequest).build();
            return chain.filter(nExchange);
        }else {
            return chain.filter(exchange);
        }
    }

    @Override
    public int getOrder() {
        return 2;
    }
}

5 网关(zuul)改写请求路径

@Component
public class RewritePathFilter implements Filter {

    private static final Logger log = LoggerFactory.getLogger(RewritePathFilter.class);
    
    @Autowired
    private ServiceHandler serviceHandler;
    
    @Override
    public void init(FilterConfig filterConfig) throws ServletException {

    }
    @Override
    public  void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain)
            throws IOException, ServletException {
        HttpServletRequest request = (HttpServletRequest) servletRequest;
        String requestContextPath = serviceHandler.getRequestContextPath(request);
        if (requestContextPath.startsWith("/admin")) {
            filterChain.doFilter(servletRequest, servletResponse);
            return;
        }
        String env = request.getHeader("host_env");
        String envServiceId = serviceHandler.mappingServiceIdByContextPath(requestContextPath, env);
        if (StringUtils.isEmpty(envServiceId)) {
            log.warn("据contextPath[{}]没匹配到相应的服务", requestContextPath);
        }
        HttpServletRequest newRequest = new HttpServletRequestWraper(request, envServiceId);
        filterChain.doFilter(newRequest, servletResponse);
    }
    
    @Override
    public void destroy() {

    }
}

HttpServletRequestWraper 改写请求path

    @Override
    public String getServletPath() {
        return "/"+serviceId+request.getServletPath();
    }

6 灰度策略实现

6.1灰度context 接口

public interface GrayRouteContext  {

    String getServiceId();

    String getPath();

    V get(Object key);

    V put(String key, Object object);

    E getExchange();

    void setExchange(E exchange);

    String getRemoteAddr();
}

6.2灰度context 接口实现

cloud gateway的实现

public class GatewayRouteContext   implements GrayRouteContext {

    private ServerWebExchange exchange;

    public GatewayRouteContext(ServerWebExchange exchange){
        this.exchange = exchange;
    }
    @Override
    public String getServiceId() {
      return (String) exchange.getAttributes().get(SERVICE_ID);
    }

    @Override
    public String getPath() {
        return exchange.getRequest().getURI().getPath();
    }

    @Override
    public Object get(Object key) {
        return null;
    }

    @Override
    public Object put(String key, Object object) {
        return null;
    }

    @Override
    public ServerWebExchange getExchange() {
        return exchange;
    }

    @Override
    public void setExchange(ServerWebExchange exchange) {
        this.exchange = exchange;
    }

    @Override
    public String getRemoteAddr() {
        InetSocketAddress remoteAddress = exchange.getRequest().getRemoteAddress();
        return  remoteAddress.getAddress().getHostAddress();
    }
}

zuul的实现

public class GatewayRouteContext implements GrayRouteContext {


    private RequestContext context;

    public GatewayRouteContext(RequestContext context) {
        this.context = context;
    }


    @Override
    public String getServiceId() {
        return (String) context.get(SERVICE_ID_KEY);
    }

    @Override
    public Object get(Object key) {
        return context.get(key);
    }

    @Override
    public Object put(String key, Object object) {
        return context.put(key, object);
    }

    @Override
    public RequestContext getExchange() {
        return context;
    }
    @Override
    public String getRemoteAddr() {
       return context.getRequest().getRemoteAddr();
    }
   
}

6.3 策略接口

public interface GrayStrategy extends Cloneable {

    /**
     *  请求 context 信息
     * @param context
     * @return
     */
    boolean isGray(T context);
    
    /**
     *  策略所有属的服务
     * @param serviceId
     */
    default void setServiceId(String serviceId){}

    /**
     *  策略类型
     * @return
     */
    default StrategyType getType(){
        return null;
    }
}

6.4 策略接口实现

组合策略
将各具体策略实现组合在一起使用

public class CompositeGrayStrategy extends GrayBaseStrategy implements ICompositeGray {

    private volatile Map grayStrategies = new HashMap<>();

    public CompositeGrayStrategy() {
    }

    @Override
    public boolean isGray(GrayRouteContext t) {
        if (grayStrategies.isEmpty()) {
            return false;
        }
        Collection values = grayStrategies.values();
        for (GrayStrategy strategy : values) {
            if (strategy.isGray(t)) {
                return true;
            }
        }
        return false;
    }
    
    @Override
    public StrategyType getType() {
        return COMPOSITE;
    }
    
    @Override
    public void add(GrayStrategy strategy) {
        grayStrategies.put(strategy.getType(), strategy);
    }
}

ip过滤策略实现

public class IpStrategy extends GrayBaseStrategy implements GrayStrategy {

    @Autowired
    private IpDao dao;

    @Override
    public boolean isGray(GatewayRouteContext context) {
        String ip = context.getRemoteAddr();
        return dao.exist(getServiceId()+ip);
    }

    @Override
    public StrategyType getType() {
        return StrategyType.IP;
    }
}

token过滤策略实现(cloud gateway)

(cloud gateway 的实现)
public class TokenStrategy extends GrayBaseStrategy implements GrayStrategy {

    @Autowired
    private TokenDao dao;

    @Override
    public boolean isGray(GatewayRouteContext context) {
        String token = context.getExchange().getRequest().getHeaders().getFirst(TOKEN);
        if (StringUtils.isEmpty(token)) {
            return false;
        }
        return dao.exist(getServiceId()+token);
    }

    @Override
    public StrategyType getType() {
        return StrategyType.TOKEN;
    }
}

(zuul 的实现)
public class TokenStrategy extends GrayBaseStrategy implements GrayStrategy {

    private String TOKEN="token";

    @Autowired
    private TokenDao dao;

    @Override
    public boolean isGray(GatewayRouteContext exchange) {
        String token = exchange.getExchange().getRequest().getHeader(TOKEN);
        if (StringUtils.isEmpty(token)) {
            return false;
        }
        return dao.exist(getServiceId()+token);
    }


    @Override
    public StrategyType getType() {
        return StrategyType.TOKEN;
    }
}