datax扩展vertica插件

1.idea里面新建module

2.新建verticareader模块

3.复制mysqlreader下的pom.xml到本项目的pom.xml中

    4.0.0
    
        com.alibaba.datax
        datax-all
        0.0.1-SNAPSHOT
    
    verticareader
    verticareader
    jar

    
        
            com.alibaba.datax
            datax-common
            ${datax-project-version}
            
                
                    slf4j-log4j12
                    org.slf4j
                
            
        
        
            org.slf4j
            slf4j-api
        
        
            ch.qos.logback
            logback-classic
        

        
            com.alibaba.datax
            plugin-rdbms-util
            ${datax-project-version}
        
        
            com.vertica
            vertica-jdbc
            9.3.1-0
        


    

    
        
            
            
                maven-compiler-plugin
                
                    ${jdk-version}
                    ${jdk-version}
                    ${project-sourceEncoding}
                
            
            
            
                maven-assembly-plugin
                
                    
                        src/main/assembly/package.xml
                    
                    datax
                
                
                    
                        dwzip
                        package
                        
                            single
                        
                    
                
            
        
    

4.新建assembly目录

5.目录下新建package.xml,将此文件粘贴进去

    
    
        dir
    
    false
    
        
            src/main/resources
            
                plugin.json
                plugin_job_template.json
            
            plugin/reader/verticareader
        
        
            target/
            
                verticareader-0.0.1-SNAPSHOT.jar
            
            plugin/reader/verticareader
        
    

    
        
            false
            plugin/reader/verticareader/libs
            runtime
        
    

新建plugin.json

{
  "name": "verticareader",
  "class": "com.xxxx.VerticaReader",
  "description": "vertica reader",
  "developer": "cx"
}

新建plugin_job_template.json

{
    "name": "verticareader",
    "parameter": {
        "username": "",
        "password": "",
        "column": [],
        "connection": [
            {
            "jdbcUrl": [],
            "table": []
            }
        ],
        "where": ""
    }
}

 6.src目录新建java文件，包名可以随便取

import com.alibaba.datax.common.element.StringColumn;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.plugin.RecordSender;
import com.alibaba.datax.common.spi.Reader;
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.common.element.Record;
import com.alibaba.datax.plugin.rdbms.reader.CommonRdbmsReader;
import com.alibaba.datax.plugin.rdbms.reader.Constant;
import com.alibaba.datax.plugin.rdbms.reader.Key;
import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode;
import com.alibaba.datax.plugin.rdbms.util.DataBaseType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;

public class VerticaReader extends Reader {

    private static final DataBaseType DATABASE_TYPE = DataBaseType.Vertica;

    public static class Job extends Reader.Job {

        private static final Logger LOG = LoggerFactory
                .getLogger(Job.class);
        private Configuration originalConfig = null;
        private CommonRdbmsReader.Job commonRdbmsReaderJob;

        @Override
        public void init() {
            this.originalConfig = super.getPluginJobConf();
            Integer userConfigedFetchSize = this.originalConfig.getInt(Constant.FETCH_SIZE);
            if (userConfigedFetchSize != null) {
                LOG.warn("对 mysqlreader 不需要配置 fetchSize, mysqlreader 将会忽略这项配置. 如果您不想再看到此警告,请去除fetchSize 配置.");
            }
            this.originalConfig.set(Constant.FETCH_SIZE, Integer.MIN_VALUE);

            this.commonRdbmsReaderJob = new CommonRdbmsReader.Job(DATABASE_TYPE);
//            this.commonRdbmsReaderJob.init(this.originalConfig);
        }

        @Override
        public void prepare() {
            LOG.info("vertica prepare....");
            init();
//            this.commonRdbmsReaderJob.preCheck(this.originalConfig,DATABASE_TYPE);
        }

        @Override
        public List split(int adviceNumber) {
            LOG.info("vertica split...."+adviceNumber);
            List configurations = new ArrayList<>();
            Integer partitions = 1;
            for (int i = 0; i < partitions; i++) {
                configurations.add(this.originalConfig.clone());
            }
            return configurations;
        }

        @Override
        public void post() {
            LOG.info("vertica post....");
            this.commonRdbmsReaderJob.post(this.originalConfig);
        }

        @Override
        public void destroy() {
            LOG.info("vertica destroy....");
            this.commonRdbmsReaderJob.destroy(this.originalConfig);
        }

    }

    public static class Task extends Reader.Task {
        private static final Logger LOG = LoggerFactory
                .getLogger(Task.class);
        private Configuration readerSliceConfig;
        private CommonRdbmsReader.Task commonRdbmsReaderTask;

        static final String JDBC_DRIVER="com.vertica.jdbc.Driver";

        static  String  DB_URL="jdbc:vertica://xxxxxx:5433/test1";

        static  String USER = "xxxx";

        static  String PASSWORD = "xxxxx";

        static String sql = "";

        Connection conn = null;
        Statement stmt = null;

        public void doCreateParams(Configuration readerSliceConfig) {
            this.USER = readerSliceConfig.getString(Key.USERNAME);
            this.PASSWORD = readerSliceConfig.getString(Key.PASSWORD);

            List