Spring Cloud Stream的基本使用

Spring Cloud Stream

概述

Spring Cloud Stream 是一个用于构建基于消息的微服务应用框架。它基于 SpringBoot 来创建具有生产级别的单机 Spring 应用，并且使用 Spring Integration 与 Broker 进行连接。

Spring Cloud Stream是Spring Cloud Alibaba提供的组件，基于Spring Cloud Stream的编程模型，接入 RocketMQ作为消息中间件，实现消息驱动的微服务。

Spring Cloud Stream 提供了消息中间件配置的统一抽象，推出了 publish-subscribe、consumer groups、partition 这些统一的概念。

官网: https://spring.io/projects/spring-cloud-stream

核心概念

两个核心概念：Binder 和 Binding

Binder: 跟外部消息中间件集成的组件，用来创建 Binding，各消息中间件都有自己的 Binder 实现。

比如 Kafka 的实现 KafkaMessageChannelBinder，RabbitMQ 的实现 RabbitMessageChannelBinder 以及 RocketMQ 的实现 RocketMQMessageChannelBinder。

Binding: 包括 Input Binding 和 Output Binding。

Binding 在消息中间件与应用程序提供的 Provider 和 Consumer 之间提供了一个桥梁，实现了开发者只需使用应用程序的 Provider 或 Consumer 生产或消费数据即可，屏蔽了开发者与底层消息中间件的接触。

常用注解

@Input

    /**
     * 用于接收消息
     * 为每个binding生成channel实例
     * 指定channel名称,在spring容器中生成一个名为my-input，类型为SubscribableChannel的bean
     * 在spring容器中生成一个类，实现Barista接口。
     * @return
     */
    @Input("my-input")
    SubscribableChannel input();

@Output

	/**
     * 用来生产消息
     * 为每个binding生成channel实例
     * 指定channel名称,在spring容器中生成一个名为my-output，类型为MessageChannel的bean
     * 在spring容器中生成一个类，实现Barista接口。
     * @return
     */
	@Output("my-output")
    MessageChannel output();

@StreamListener

/**
     * 用于消费消息
     * 
     * condition：符合条件，才进入处理方法
     *  condition生效前提条件：
     *      注解的方法没有返回值
     *      方法是一个独立方法，不支持Reactive API
     * @param msg
     */
    @StreamListener(value = Sink.INPUT, condition = "headers['header-tag']=='header-tag1'")
    public void receiveHeaderTag1(String msg) {
        System.out.println("Stream receiveHeaderTag1 接收消息: " + msg);
    }

@SendTo

    /**
     *  接收INPUT这个channel的消息，并将返回值发送到OUTPUT这个channel
     * @param msg
     * @return
     */
    @StreamListener(Sink.INPUT)
    @SendTo(Source.OUTPUT)
    public String receive(String msg) {
        return "handle...";
    }

@InboundChannelAdapter

    /**
     * 让定义的方法生产消息
     * 不接受任何参数
     * fixedDelay：多少毫秒发送1次
     * maxMessagesPerPoll：一次发送几条消息
     *
     * @return
     */
    @Bean
    @InboundChannelAdapter(value = Source.OUTPUT, poller = @Poller(fixedDelay = "1000", maxMessagesPerPoll = "2"))
    public MessageSource<String> testInboundChannelAdapter() {
        return () -> {
            Map<String, Object> map = new HashMap<>(1);
            map.put("header-tag", "header-tag1");
            return new GenericMessage<>("map", map);
        };
    }

@ServiceActivator

    /**
     * 方法能够处理消息或消息有效内容
     * 监听input消息，用方法体的代码处理，然后输出到output中
     *
     * @param payload
     * @return
     */
    @ServiceActivator(inputChannel = Sink.INPUT, outputChannel = Source.OUTPUT)
    public String transform(String payload) {
        return payload.toUpperCase();
    }

@Transformer

    /**
     * 和ServiceActivator类似
     * 
     * 方法能够转换消息，消息头，或消息有效内容
     * @param message
     * @return
     */
    @Transformer(inputChannel = Processor.INPUT, outputChannel = Processor.OUTPUT)
    public Object transform(String message) {
        return message.toUpperCase();
    }

Source与Sink内置接口

通过Source与Sink 接口进行消息的发送与接收

Processor 由于继承Source与Sink两个接口，所以可以进行消息的发送与接收

public interface Processor extends Source, Sink {
}

生产者

    <dependency>
            <groupId>com.alibaba.cloud</groupId>
            <artifactId>spring-cloud-starter-stream-rocketmq</artifactId>
        </dependency>

spring:
  cloud:
    stream:
      rocketmq:
        binder:
          name-server: IP:9876
      bindings:
        output:
          destination: stream-test-topic

@EnableBinding({Source.class})
public class Application {
}

	@Autowired
    private Source source;

    @GetMapping("/testStream")
    public String testStream() {
        HashMap<String, Object> map = new HashMap<>();
        map.put("name", "小白");
        map.put("age", 22);
        map.put("sex", "男");

        source.output().send(MessageBuilder.withPayload(map).build());
        return "success";
    }

消费者

    <dependency>
            <groupId>com.alibaba.cloud</groupId>
            <artifactId>spring-cloud-starter-stream-rocketmq</artifactId>
        </dependency>

spring:
  cloud:
    stream:
      rocketmq:
        binder:
          name-server: IP:9876
      bindings:
        input:
          destionation: stream-test-topic
          group: stream-group

@EnableBinding({Sink.class})
public class Application {
}

@Service
public class TestStreamConsumer {

    @StreamListener(Sink.INPUT)
    public void receive(String msg){
        System.out.println("Stream接收消息: " + msg);
    }
}

Stream接收消息: {"sex":"男","name":"小白","age":22}

自定义接口

生产者

public interface MySource {
    @Output("my-output")
    MessageChannel output();
}

@EnableBinding({Source.class, MySource.class})
public class Application {
}

	 @Autowired
    private MySource mySource;
    
    @GetMapping("/testStream")
    public String testStream() {
        HashMap<String, Object> map = new HashMap<>();
        map.put("name", "小白");
        map.put("age", 22);
        map.put("sex", "男");

        mySource.output().send(MessageBuilder.withPayload(map).build());
        return "success";
    }

    stream:
      rocketmq:
        binder:
          name-server: IP:9876
      bindings:
        output:
          destination: stream-test-topic
        my-output:
          destination: stream-my-topic

消费者

public interface MySink {
    @Input("my-input")
    SubscribableChannel input();
}

@EnableBinding({Sink.class, MySink.class})
public class Application {
}

@Service
public class TestStreamConsumer {

    @StreamListener("my-input")
    public void receive(String msg) {
        System.out.println("自定义接口接收消息: " + msg);
    }
}

    stream:
      rocketmq:
        binder:
          name-server: IP:9876
      bindings:
        input:
          destination: stream-test-topic
          group: stream-group
        my-input:
          destination: stream-my-topic
          group: stream-my-group

自定义接口接收消息: {"sex":"男","name":"小白","age":22}

消息异常处理

全局异常处理

@Service
public class TestStreamConsumer {

    @StreamListener("my-input")
    public void receive(String msg) {
        System.out.println("自定义接口接收消息: " + msg);
        throw new IllegalArgumentException("出现异常");
    }

    @StreamListener("errorChannel")
    public void error(Message<?> message){
         ErrorMessage errorMessage=(ErrorMessage)message;
        System.out.println("errorMessage = " + errorMessage);
    }
}

局部异常处理

    @ServiceActivator(inputChannel = "stream-test-topic.stream-my-topic.errors")
    public void handleError(ErrorMessage message) {
        Throwable throwable = message.getPayload();
        System.out.println("获取异常:"+throwable);
    }

自定义接口接收消息: {"sex":"男","name":"小白","age":15}
自定义接口接收消息: {"sex":"男","name":"小白","age":15}
自定义接口接收消息: {"sex":"男","name":"小白","age":15}
INFO 68080 --- [MessageThread_1] o.s.i.h.s.MessagingMethodInvokerHelper   : Overriding default instance of MessageHandlerMethodFactory with provided one.
获取异常:org.springframework.messaging.MessagingException: Exception thrown while invoking TestStreamConsumer#receive[1 args]; nested exception is java.lang.RuntimeException: 出现异常, failedMessage=GenericMessage [payload=byte[38], headers={rocketmq_QUEUE_ID=1, rocketmq_TOPIC=stream-test-topic1, rocketmq_FLAG=0, rocketmq_RECONSUME_TIMES=0, rocketmq_MESSAGE_ID=C0A81E1909F000B4AAC230BD1461010B, rocketmq_SYS_FLAG=0, id=77dced0c-166b-1130-b198-fa083c1c3bdd, CLUSTER=DefaultCluster, rocketmq_BORN_HOST=125.71.203.164, contentType=application/json, rocketmq_BORN_TIMESTAMP=1639105697889, timestamp=1639105697932}]

重试消费

        my-input:
          destination: stream-test-topic1
          group: stream-my-topic
          consumer:
            # 最多尝试处理几次，默认3
            maxAttempts:  2
            # 重试时初始避退间隔，单位毫秒，默认1000
            backOffInitialInterval: 1000
            # 重试时最大避退间隔，单位毫秒，默认10000
            backOffMaxInterval: 10000
            # 避退乘数，默认2.0
            backOffMultiplier: 2.0
            # 当listen抛出retryableExceptions未列出的异常时，是否要重试
            defaultRetryable: true
            # 异常是否允许重试的map映射
            retryableExceptions:
              java.lang.RuntimeException: true
              java.lang.IllegalStateException: false

自定义接口接收消息: {"sex":"男","name":"小白","age":33}
自定义接口接收消息: {"sex":"男","name":"小白","age":33}
INFO 68080 --- [MessageThread_1] o.s.i.h.s.MessagingMethodInvokerHelper   : Overriding default instance of MessageHandlerMethodFactory with provided one.
获取异常:org.springframework.messaging.MessagingException: Exception thrown while invoking TestStreamConsumer#receive[1 args]; nested exception is java.lang.RuntimeException: 出现异常, failedMessage=GenericMessage [payload=byte[41], headers={rocketmq_QUEUE_ID=1, rocketmq_TOPIC=stream-test-topic1, rocketmq_FLAG=0, rocketmq_RECONSUME_TIMES=0, rocketmq_MESSAGE_ID=C0A81E1909F000B4AAC230B900E00108, rocketmq_SYS_FLAG=0, id=437c402b-9126-287e-b712-653158a97582, CLUSTER=DefaultCluster, rocketmq_BORN_HOST=125.71.203.164, contentType=application/json, rocketmq_BORN_TIMESTAMP=1639105430752, timestamp=1639105430795}]

Spring Cloud Stream+RocketMQ实现分布式事务

生产者

    @GetMapping("/testRocketTx")
    public String testRocketTx() {
        HashMap<String, Object> map = new HashMap<>();
        map.put("name", "小白");
        map.put("age", 22);
        map.put("sex", "男");

        source.output().send(
                MessageBuilder.withPayload(map)
                        .setHeader(RocketMQHeaders.TRANSACTION_ID, UUID.randomUUID().toString())
                        .setHeader("my_data", map)
                        .build()
        );

        return "success";
    }

消费者

@RocketMQTransactionListener(txProducerGroup = "stream-tx-group")
@RequiredArgsConstructor
public class AddBonusTransactionListener implements RocketMQLocalTransactionListener {
    private final ShareService shareService;
    private final RocketmqTransactionLogMapper rocketmqTransactionLogMapper;

    /**
     * 消费消息,消费成功提交事务,失败则回滚丢弃消息不消费
     *
     * @param msg
     * @param arg
     * @return
     */
    @Transactional(rollbackFor = Exception.class)
    @Override
    public RocketMQLocalTransactionState executeLocalTransaction(Message msg, Object arg) {
        MessageHeaders headers = msg.getHeaders();

        byte[] payloadByte = (byte[]) msg.getPayload();
        String payload = new String(payloadByte);
        System.out.println("payload = " + payload);

        String transactionId = (String) headers.get(RocketMQHeaders.TRANSACTION_ID);
        System.out.println("transactionId = " + transactionId);

        String myData = headers.get("my_data").toString();
        System.out.println("myData = " + myData);

        try {
            // TODO 记录MQ日志
            int a = 10;
            int b = a / 0;
            //可以消费该消息
            return RocketMQLocalTransactionState.COMMIT;
        } catch (Exception e) {
            // 继续查询该消息的状态
            return RocketMQLocalTransactionState.UNKNOWN;
        }
    }

    /**
     * 检查本地事务状态
     *
     * @param msg
     * @return
     */
    @Override
    public RocketMQLocalTransactionState checkLocalTransaction(Message msg) {
        MessageHeaders headers = msg.getHeaders();
        String transactionId = (String) headers.get(RocketMQHeaders.TRANSACTION_ID);

        // TODO 根据事务id查询数据库,判断消息是否消费
        Object DbRocketMQLog = "DB RocketMQ Log Object";

        if (DbRocketMQLog != null) {
            // 消息已被消费,删除该消息
            return RocketMQLocalTransactionState.ROLLBACK;
        }
        // TODO 进行消费逻辑,记录MQ日志
        return RocketMQLocalTransactionState.COMMIT;
    }
}

配置

  stream:
      rocketmq:
        binder:
          name-server: IP:9876
        bindings:
          output:
            producer:
              transactional: true
              group: stream-tx-group
      bindings:
        output:
          destination: stream-test-topic

消息消费过滤

Header过滤

生产者生成消息，设置不同的header，多个消费者根据不同的header进行消费处理

    @GetMapping("/testStream/{id}")
    public String testStream(@PathVariable Integer id) {
        HashMap<String, Object> map = new HashMap<>();
        map.put("name", "小白");
        map.put("age", 22);
        map.put("sex", "男");

        String headerTag="header-tag"+id;
        source.output().send(
                MessageBuilder.withPayload(map)
                        .setHeader("header-tag",headerTag )
                        .build());

        return "success";
    }

@SpringBootApplication
@EnableBinding({Source.class, Sink.class})
public class Application {

    public static void main(String[] args) {
        SpringApplication.run(Application.class, args);
    }
}

@Component
public class TestStreamConsumer {
    @StreamListener(value = Sink.INPUT, condition = "headers['header-tag']=='header-tag1'")
    public void receiveHeaderTag1(String msg) {
        System.out.println("Stream receiveHeaderTag1 接收消息: " + msg);
    }

    @StreamListener(value = Sink.INPUT, condition = "headers['header-tag']=='header-tag2'")
    public void receiveHeaderTag2(String msg) {
        System.out.println("Stream receiveHeaderTag2 接收消息: " + msg);
    }
}

spring:
  cloud:
    stream:
      rocketmq:
        binder:
          name-server: IP:9876
      bindings:
        output:
          destination: stream-test-topic
        input:
          destination: stream-test-topic
          group: stream-group

访问http://localhost:8080/testStream/2

Stream receiveHeaderTag1 接收消息: {"sex":"男","name":"小白","age":22}
Stream receiveHeaderTag2 接收消息: {"sex":"男","name":"小白","age":22}

Tags过滤

该方式只支持RoketMQ

    @GetMapping("/testStream/{id}")
    public String testStream(@PathVariable Integer id) {
        HashMap<String, Object> map = new HashMap<>();
        map.put("name", "小白");
        map.put("age", 22);
        map.put("sex", "男");

        String tag="tag"+id;
        source.output().send(
                MessageBuilder.withPayload(map)
                        // 只能设置1个tag
                        .setHeader(RocketMQHeaders.TAGS, tag)
                        .build());

        return "success";
    }

public interface MySink {
    String INPUT1 = "input1";
    String INPUT2 = "input2";

    @Input(INPUT1)
    SubscribableChannel input();

    @Input(INPUT2)
    SubscribableChannel input2();
}

@SpringBootApplication
@EnableBinding({Source.class, Sink.class, MySink.class})
public class Application {

    public static void main(String[] args) {
        SpringApplication.run(Application.class, args);
    }

}

@Component
public class TestStreamConsumer {

    @StreamListener(MySink.INPUT1)
    public void receive1(String msg) {
        System.out.println("消费tag1的消息: "+ msg);
    }


    @StreamListener(MySink.INPUT2)
    public void receive2(String msg) {
        System.out.println("消费tag2/tag3的消息: "+ msg);
    }

}

spring:
  cloud:
    stream:
      rocketmq:
        binder:
          name-server: 112.74.96.150:9876
        bindings:
          input1:
            consumer:
              tags: tag1
          input2:
            consumer:
              tags: tag2 || tag3
      bindings:
        input1:
          destination: stream-test-topic
          group: stream-group1
        input2:
          destination: stream-test-topic
          group: stream-group2
        output:
          destination: stream-test-topic
        input:
          destination: stream-test-topic
          group: stream-group

消费tag1的消息: {"sex":"男","name":"小白","age":22}
消费tag2/tag3的消息: {"sex":"男","name":"小白","age":22}
消费tag2/tag3的消息: {"sex":"男","name":"小白","age":22}