Flume自定义Source之MysqlSource

场景描述：公司项目某一模块是数据源之间数据迁移，场景之一就是从Mysql中读取数据写入到其他的数据源，架构采用的的flume+kafa的形式，Kafka作为Channel，然后通过自定Source和Sink，将数据在不同的数据源之间迁移，而且效率还比较高，也不会丢数据；

整理项目知识点的过程中打算写一下博客，结合一些机构的资料简单写一下自定义MysqlSource；

主要考虑的几点：

1、实时监控mysql数据库某张表的数据变化；

2、根据offset和maxRecords去查询数据；

3、将数据封装成event，写入Channel;

4、写入成功之后更新offset,失败回滚（这里就不说回滚的问题，Source这里也没什么要说的在Sink那里着重考虑）

maven依赖：

    org.apache.flume
    flume-ng-core
    1.9.0



    mysql
    mysql-connector-java
    5.1.27

maven中添加打包的插件：

        
            
                maven-assembly-plugin
                
                    
                        jar-with-dependencies
                    
                    
                        
                            
                        
                    
                
                
                    
                        make-assembly
                        package
                        
                            single
                        
                    
                
            
        
    

为了方便操作定义MySQLSourceHandler

import org.apache.flume.Context;
import org.apache.flume.conf.ConfigurationException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.sql.*;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;

public class MySQLSQLSourceHandler {
    private static final Logger LOG = LoggerFactory.getLogger(MySQLSQLSourceHandler.class);
    private Integer runQueryDelay,//每两次查询的时间间隔
                    startFrom,//开始id
                    currentIndex,//当前id
                    recordSize,//每次查询返回结果的条数
                    maxRow;//每次查询的最大条数

    private String table,//要操作的表
                   columnsToSelect,//用户传入的查询的样例
                   customQuery,//用户传入的查询语句
                   query,//构建的查询语句
                   defaultCharsetResultSet;//编码集

    //上下文 用来获取配置文件
    private Context context;

    //为定义的变量赋值（默认值），可在flume任务的配置文件中修改
    private static final Integer DEFAULT_QUERY_DELAY = 10000;
    private static final Integer DEFAULT_START_VALUE=0;
    private static final Integer DEFAULT_MAX_ROWS = 5000;
    private static final String DEFAULT_COLUMNS_SELECT = "*";
    private static final String DEFAULT_CHARSET_RESULTSET = "UTF-8";

    private static Connection conn = null;
    private static PreparedStatement ps = null;
    private static String connectionURL,connectionUserName,connectionPassword;

    //加载静态资源
    static{
        try {
            Properties prop = new Properties();
            prop.load(MySQLSQLSourceHandler
                    .class.getClassLoader()
                    .getResourceAsStream("jdbc.properties"));
            connectionURL = prop.getProperty("dbUrl");
            connectionUserName = prop.getProperty("dbUser");
            connectionPassword = prop.getProperty("dbPassword");
            Class.forName(prop.getProperty("dbDriver"));

        } catch (IOException e) {
            e.printStackTrace();
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        }
    }

    //获取jdbc连接
    private static Connection InitConnection(String url,String user,String password){

        try {
            Connection conn = DriverManager.getConnection(url,user,password);
            if (conn == null)
                throw new SQLException();
            return conn;
        } catch (SQLException e) {
            e.printStackTrace();
        }
        return null;
    }

    //构造方法
    MySQLSQLSourceHandler (Context context) throws ParseException {
        //初始化上下文
        this.context = context;

        //有默认值参数：获取flume任务配置文件中的参数，读不到的采用默认值
        this.columnsToSelect = context.getString("columns.to.select", DEFAULT_COLUMNS_SELECT);
        this.runQueryDelay = context.getInteger("run.query.delay", DEFAULT_QUERY_DELAY);
        this.startFrom = context.getInteger("start.from", DEFAULT_START_VALUE);
        this.maxRow = context.getInteger("max.row", DEFAULT_MAX_ROWS);
        this.defaultCharsetResultSet = context.getString("default.charset.resultset", DEFAULT_CHARSET_RESULTSET);


        //无默认值参数：获取flume任务配置文件中的参数
        this.table = context.getString("table");
        this.customQuery = context.getString("custom.query");
        connectionURL = context.getString("connection.url");
        connectionUserName = context.getString("connection.user");
        connectionPassword = context.getString("connection.password");
        conn = InitConnection(connectionURL, connectionUserName, connectionPassword);

        //校验相应的配置信息，如果没有默认值的参数也没赋值，抛出异常
        checkMandatoryProperties();
        //获取当前的id
        currentIndex = getStatusDBIndex(startFrom);
        //构建查询语句
        query = buildQuery();
    }

    //校验相应的配置信息（表，查询语句以及数据库连接的参数）
    private void checkMandatoryProperties() {
        if (table == null) {
            throw new ConfigurationException("property table not set");
        }
        if (connectionURL == null) {
            throw new ConfigurationException("connection.url property not set");
        }
        if (connectionUserName == null) {
            throw new ConfigurationException("connection.user property not set");
        }
        if (connectionPassword == null) {
            throw new ConfigurationException("connection.password property not set");
        }
    }

    //构建sql语句
    private String buildQuery() {
        String sql = "";
        //获取当前id
        currentIndex = getStatusDBIndex(startFrom);
        LOG.info(currentIndex + "");
        if (customQuery == null) {
            sql = "SELECT " + columnsToSelect + " FROM " + table;
        } else {
            sql = customQuery;
        }

        StringBuilder execSql = new StringBuilder(sql);
        //以id作为offset
        if (!sql.contains("where")) {
            execSql.append(" where ");
            execSql.append("id").append(">").append(currentIndex);
            execSql.append(" and id").append("<=").append(currentIndex+maxRow);
            LOG.info("execSql:" + execSql.toString() );
            return execSql.toString();
        } else {
            String  oldSql = execSql.toString();
            int num = KMPFunction.evaluate(oldSql,"where");
            String no_where = oldSql.substring(0,num);
            execSql = new StringBuilder(no_where);
            execSql.append(" where ");
            execSql.append("id").append(">").append(currentIndex);
            execSql.append(" and id").append("<=").append(currentIndex+maxRow);
            LOG.info("execSql:" + execSql.toString() );
            return execSql.toString();
        }
    }

    //执行查询
    List> executeQuery() {
        try {
            //每次执行查询时都要重新生成sql，因为id不同
            customQuery = buildQuery();
            //存放结果的集合
            List> results = new ArrayList<>();
            if (ps == null) {
                //
                ps = conn.prepareStatement(customQuery);
            }
            ResultSet result = ps.executeQuery(customQuery);
            while (result.next()) {
                //存放一条数据的集合（多个列）
                List