Flink 消费 Kafka, 实时修改 Doris 表数据.

2022/11/08 菜鸟记录.

场景: Flink 消费 Kafka 数据, 修改 Doris 表数据.

背景: Flink版本 1.14.4

         Doris版本 1.1.0

代码: 

public class DorisUpdateDemo {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.enableCheckpointing(5000);
        env.setParallelism(5);
        CheckpointConfig checkpointConfig = env.getCheckpointConfig();
        checkpointConfig.setCheckpointingMode(CheckpointingMode.AT_LEAST_ONCE);
        checkpointConfig.setExternalizedCheckpointCleanup(CheckpointConfig.ExternalizedCheckpointCleanup.DELETE_ON_CANCELLATION);

        KafkaSource kafkaSource = KafkaUtils.getKafkaSource("192.168.**.**:9092", "test", "test");
        DataStreamSource kafkaDS = env.fromSource(kafkaSource, WatermarkStrategy.noWatermarks(), "kafkaSource");

        SingleOutputStreamOperator jsonDS = kafkaDS.process(new ProcessFunction() {
            @Override
            public void processElement(String json, Context context, Collector collector) throws Exception {
                try {
                    JSONObject jsonObject = JSON.parseObject(json);
                    JSONObject result = new JSONObject();
                    result.put("id", jsonObject.get("id"));
                    result.put("handle", "1");
                    collector.collect(result);
                } catch (Exception ignored) {
                }
            }
        });
        jsonDS.print();
        Properties pro = new Properties();
        pro.setProperty("jdbc_driver", "com.mysql.jdbc.Driver");
        pro.setProperty("db_url_pattern", "jdbc:mysql://%s:%s/%s?useSSL=false");
        pro.setProperty("host", "192.168.**.**");
        pro.setProperty("port", "9030");
        pro.setProperty("db", "demo");
        pro.setProperty("user", "******");
        pro.setProperty("passwd", "******");

        String sql = "update" + " demo.event" + " set handle=? where id=?";
        String strings = "handle,id"; //按占位符?顺序
        jsonDS.addSink(new JdbcDorisSink<>(sql,strings,pro));
        
        env.execute();
    }
}

public class KafkaUtils {

    public static KafkaSource getKafkaSource(String brokers, String topic, String groupId) {
        KafkaSource kafkaSource = KafkaSource.builder()
                .setBootstrapServers(brokers)
                .setTopics(topic)
                .setGroupId(groupId)
                .setStartingOffsets(OffsetsInitializer.latest())
                .setValueOnlyDeserializer(new SimpleStringSchema())
                .build();
        return kafkaSource;
    }

    public static KafkaSink getKafkaSink(String brokers, String topic) {
        KafkaSink kafkaSink = KafkaSink.builder()
                .setBootstrapServers(brokers)
                .setRecordSerializer(KafkaRecordSerializationSchema.builder()
                        .setTopic(topic)
                        .setValueSerializationSchema(new SimpleStringSchema())
                        .build()
                )
                .setDeliverGuarantee(DeliveryGuarantee.AT_LEAST_ONCE)
                .build();
        return kafkaSink;
    }


}

public class JdbcDorisSink extends RichSinkFunction {
    private DruidDataSource druidDataSource;
    private DruidPooledConnection conn;
    private PreparedStatement preparedStatement;
    private String sql;
    private String strings;
    private Properties pro;

    public JdbcDorisSink(String sql, String strings, Properties pro) {
        this.sql = sql;
        this.strings = strings;
        this.pro = pro;
    }

    @Override
    public void open(Configuration parameters) throws Exception {
        super.open(parameters);
        // 创建连接池
        DruidUtils druidUtil = new DruidUtils(pro);
        druidDataSource = druidUtil.createDorisDataSource();
    }

    @Override
    public void invoke(T bean, Context context) throws Exception {
        Class clazz = bean.getClass();
        try {
            // 获取数据库操作对象（预编译）
            conn = druidDataSource.getConnection();
            preparedStatement = conn.prepareStatement(sql);
        } catch (SQLException sqlException) {
            System.out.println("数据库操作对象获取异常~");
            sqlException.printStackTrace();
        }
        //注入sql
        try {
            Field[] declaredFields = clazz.getDeclaredFields();
            Field declaredField = declaredFields[declaredFields.length - 1];
            declaredField.setAccessible(true);
            HashMap map = (HashMap) declaredField.get(bean);
            String[] fields = strings.split(",");
            int index = 1;
            for (String field : fields) {
                preparedStatement.setObject(index, map.get(field));
                index++;
            }
        } catch (SecurityException e) {
            e.printStackTrace();
        } catch (IllegalArgumentException e) {
            e.printStackTrace();
        } catch (IllegalAccessException e) {
            e.printStackTrace();
        } catch (SQLException e) {
            e.printStackTrace();
        }

        //写入
        try {
            preparedStatement.execute();
        } catch (SQLException sqlException) {
            System.out.println("Doris 写入 SQL 执行异常~");
            sqlException.printStackTrace();
        }

        Thread.sleep(100);  //防止连续update导致资源来不及释放报错.
        closeResource(preparedStatement, conn);
        Thread.sleep(100);
    }

    //资源释放方法
    private void closeResource(PreparedStatement preparedStatement, DruidPooledConnection conn) {
        if (preparedStatement != null) {
            try {
                preparedStatement.close();
            } catch (SQLException sqlException) {
                System.out.println("数据库操作对象释放异常~");
                sqlException.printStackTrace();
            }
        }
        if (conn != null) {
            try {
                conn.close();
            } catch (SQLException sqlException) {
                System.out.println("德鲁伊连接对象释放异常~");
                sqlException.printStackTrace();
            }
        }
    }

    @Override
    public void close() throws Exception {
        super.close();
        if (preparedStatement != null) {
            try {
                preparedStatement.close();
            } catch (SQLException sqlException) {
                System.out.println("数据库操作对象释放异常~");
                sqlException.printStackTrace();
            }
        }
        if (conn != null) {
            try {
                conn.close();
            } catch (SQLException sqlException) {
                System.out.println("德鲁伊连接对象释放异常~");
                sqlException.printStackTrace();
            }
        }
        if (druidDataSource != null) {
            try {
                druidDataSource.close();
            } catch (Exception e) {
                System.out.println("德鲁伊连接池释放异常~");
                e.printStackTrace();
            }

        }
    }
}

public class DruidUtils {
    private DruidDataSource druidDataSource;
    private String jdbc_driver;
    private String db_url_pattern;
    private String host;
    private String port;
    private String db;
    private String user;
    private String passwd;

    public DruidUtils(Properties pro) {
        jdbc_driver = pro.getProperty("jdbc_driver");
        db_url_pattern = pro.getProperty("db_url_pattern");
        host = pro.getProperty("host");
        port = pro.getProperty("port");
        db = pro.getProperty("db");
        user = pro.getProperty("user");
        passwd = pro.getProperty("passwd");
    }

    public DruidDataSource getDataSource(String driverClassName, String url) {
        // 创建连接池
        druidDataSource = new DruidDataSource();
        // 设置驱动全类名
        druidDataSource.setDriverClassName(driverClassName);
        // 设置连接 url
        druidDataSource.setUrl(url);
        druidDataSource.setUsername(user);
        druidDataSource.setPassword(passwd);
        // 设置初始化连接池时池中连接的数量
        druidDataSource.setInitialSize(5);
        // 设置同时活跃的最大连接数
        druidDataSource.setMaxActive(20);
        // 设置空闲时的最小连接数，必须介于 0 和最大连接数之间，默认为 0
        druidDataSource.setMinIdle(1);
        // 设置没有空余连接时的等待时间，超时抛出异常，-1 表示一直等待
        druidDataSource.setMaxWait(-1);
        // 验证连接是否可用使用的 SQL 语句
        druidDataSource.setValidationQuery("select 1");
        // 指明连接是否被空闲连接回收器（如果有）进行检验，如果检测失败，则连接将被从池中去除
        // 注意，默认值为 true，如果没有设置 validationQuery，则报错
        // testWhileIdle is true, validationQuery not set
        druidDataSource.setTestWhileIdle(true);
        // 借出连接时，是否测试，设置为 false，不测试，否则很影响性能
        druidDataSource.setTestOnBorrow(false);
        // 归还连接时，是否测试
        druidDataSource.setTestOnReturn(false);
        // 设置空闲连接回收器每隔 30s 运行一次
        druidDataSource.setTimeBetweenEvictionRunsMillis(30 * 1000L);
        // 设置池中连接空闲 30min 被回收，默认值即为 30 min
        druidDataSource.setMinEvictableIdleTimeMillis(30 * 60 * 1000L);

        return druidDataSource;
    }

    public DruidDataSource createDorisDataSource() {
        return getDataSource(jdbc_driver, String.format(db_url_pattern, host, port, db));
    }
}