flink接入mqtt数据源

flink没有原生的mqtt数据源，但可以通过自定义数据源进行添加mqtt的数据源。

package com.agioe.flink.source.mqtt;
 
import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
import org.eclipse.paho.mqttv5.client.*;
import org.eclipse.paho.mqttv5.client.persist.MemoryPersistence;
import org.eclipse.paho.mqttv5.common.MqttException;
import org.eclipse.paho.mqttv5.common.MqttMessage;
import org.eclipse.paho.mqttv5.common.packet.MqttProperties;

import java.io.ByteArrayOutputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;

 
public class MqttSource extends RichSourceFunction {
    //存储服务
    private static MqttClient client;
    //存储订阅主题
    private static MqttTopic mqttTopic;
    //阻塞队列存储订阅的消息
    private BlockingQueue queue = new ArrayBlockingQueue<>(10);

    private StartupParams startupParams;

    public MqttSource(StartupParams startupParams) {
        this.startupParams = startupParams;
    }

    //包装连接的方法
    private void connect() throws MqttException {
        String username = startupParams.pro.getProperty("mqtt.username");
        String password = startupParams.pro.getProperty("mqtt.password");


        //配置连接参数
        MqttConfig mqttConfigBean = new MqttConfig(null, null, "tcp://192.168.15.13:1883", "mqtt-client", "testtopic/#");
        //连接mqtt服务器
        client = new MqttClient(mqttConfigBean.getHostUrl(), mqttConfigBean.getClientId(), new MemoryPersistence());
        MqttConnectionOptions options = new MqttConnectionOptions();

        options.setUserName(mqttConfigBean.getUsername());
        options.setPassword(mqttConfigBean.getPassword() == null ? null : mqttConfigBean.getPassword().getBytes(StandardCharsets.UTF_8));
//        options.setCleanSession(false);   //是否清除session
//        options.setSessionExpiryInterval();
        // 设置超时时间
        options.setConnectionTimeout(30);
        // 设置会话心跳时间
        options.setKeepAliveInterval(20);
        try {
            String[] msgtopic = mqttConfigBean.getMsgTopic();
            //订阅消息
            int[] qos = new int[msgtopic.length];
            for (int i = 0; i < msgtopic.length; i++) {
                qos[i] = 0;
            }
            client.setCallback(new MsgCallback(client, options, msgtopic, qos) {
            });
            client.connect(options);
            client.subscribe(msgtopic, qos);
            System.out.println("MQTT连接成功:" + mqttConfigBean.getClientId() + ":" + client);
        } catch (Exception e) {
            System.out.println("MQTT连接异常：" + e);
        }
    }

    //实现MqttCallback，内部函数可回调
    class MsgCallback implements MqttCallback {
        private MqttClient client;
        private MqttConnectionOptions options;
        private String[] topic;
        private int[] qos;

        public MsgCallback() {
        }

        public MsgCallback(MqttClient client, MqttConnectionOptions options, String[] topic, int[] qos) {
            this.client = client;
            this.options = options;
            this.topic = topic;
            this.qos = qos;
        }

        //连接失败回调该函数
        @Override
        public void disconnected(MqttDisconnectResponse mqttDisconnectResponse) {
            System.out.println("MQTT连接断开，发起重连");
            while (true) {
                try {
                    Thread.sleep(1000);
                    client.connect(options);
                    //订阅消息
                    client.subscribe(topic, qos);
                    System.out.println("MQTT重新连接成功:" + client);
                    break;
                } catch (Exception e) {
                    e.printStackTrace();
                    continue;
                }
            }
        }

        @Override
        public void mqttErrorOccurred(MqttException e) {

        }

        //收到消息回调该函数
        @Override
        public void messageArrived(String s, MqttMessage message) throws Exception {
            System.out.println();
            //订阅消息字符
            String msg = new String(message.getPayload());
            byte[] bymsg = getBytesFromObject(msg);
            System.out.println("topic:" + topic);
            queue.put(msg);

        }

        @Override
        public void deliveryComplete(IMqttToken iMqttToken) {
        }

        @Override
        public void connectComplete(boolean b, String s) {
            System.out.println("MQTT重新连接成功:" + client);
        }

        @Override
        public void authPacketArrived(int i, MqttProperties mqttProperties) {

        }

        //对象转化为字节码
        public byte[] getBytesFromObject(Serializable obj) throws Exception {
            if (obj == null) {
                return null;
            }
            ByteArrayOutputStream bo = new ByteArrayOutputStream();
            ObjectOutputStream oo = new ObjectOutputStream(bo);
            oo.writeObject(obj);
            return bo.toByteArray();
        }
    }

    //flink线程启动函数
    @Override
    public void run(final SourceContext ctx) throws Exception {
        connect();
        //利用死循环使得程序一直监控主题是否有新消息
        while (true) {
            //使用阻塞队列的好处是队列空的时候程序会一直阻塞到这里不会浪费CPU资源
            ctx.collect(queue.take());
        }
    }


    @Override
    public void cancel() {

    }

    /**
     * 订阅某个主题
     *
     * @param topic
     * @param qos
     */
    public void subscribe(String topic, int qos) {
        try {
            System.out.println("topic:" + topic);
            client.subscribe(topic, qos);
        } catch (MqttException e) {
            e.printStackTrace();
        }
    }

    public MqttClient getClient() {
        return client;
    }

    public void setClient(MqttClient client) {
        this.client = client;
    }

    public MqttTopic getMqttTopic() {
        return mqttTopic;
    }

    public void setMqttTopic(MqttTopic mqttTopic) {
        this.mqttTopic = mqttTopic;
    }


}

public class MqttConfig implements Serializable {

    public MqttConfig(String username, String password, String hostUrl, String clientId, String msgTopic) {
        this.username = username;
        this.password = password;
        this.hostUrl = hostUrl;
        this.clientId = clientId;
        this.msgTopic = msgTopic;
    }
    //连接名称
    private String username;
    //连接密码
    private String password;
    //ip地址以及端口号
    private String hostUrl;
    //服务器ID注意不能与其他连接重复，否则会连接失败
    private String clientId;
    //订阅的主题
    private String msgTopic;


    //获得用户名
    public String getUsername() {
        return username;
    }
    //获得密码
    public String getPassword() {
        return password;
    }
    //获得客户端id
    public String getClientId() {
        return clientId;
    }
    //获得服务端url
    public String getHostUrl() {
        return hostUrl;
    }
    //获得订阅
    public String[] getMsgTopic() {
        String[] topic = msgTopic.split(",");
        return topic;
    }

}

在flink的main函数中增加

    stream = env.addSource(new MqttSource(StartupParams.instance));