使用kettle进行数据标准化

整个kettle调度

1-0获取数据库名称数据

获取数据库名称数据

select '' AS DB_ID,
       '' as db_name,
       '' AS DW_CODE,
       '' AS DW_NAME,
       '' AS ZT_CODE,
       '' AS ZT_NAME,
       t1.begin_year,
       t1.end_year,
       t1.version,
       t1.xzqh_code,
       '' AS XZQH_NAME,
       t1.DB_NAME    as JCZL_USER,
       t2.DB_NAME    as JZHS_USER
  from (SELECT '' AS DB_ID,
               T1.USERNAME AS DB_NAME,
               '' AS DW_CODE,
               '' AS DW_NAME,
               '' AS ZT_CODE,
               '' AS ZT_NAME,
               SUBSTR(T1.USERNAME, INSTR(T1.USERNAME, '_', 1，3) + 1, 4) AS BEGIN_YEAR,
               SUBSTR(T1.USERNAME, INSTR(T1.USERNAME, '_', -1，1) + 1, 4) AS END_YEAR,
               '集中_中科BS' AS VERSION,
               substr(T1.USERNAME, 1, INSTR(T1.USERNAME, '_', 1，1) - 1) AS XZQH_CODE,
               '' AS XZQH_NAME
          FROM DBA_USERS T1
         WHERE T1.USERNAME IN ('330104_ZK2_JCZL_2017_2018')) t1
  left join (SELECT '' AS DB_ID,
                    T1.USERNAME AS DB_NAME,
                    '' AS DW_CODE,
                    '' AS DW_NAME,
                    '' AS ZT_CODE,
                    '' AS ZT_NAME,
                    SUBSTR(T1.USERNAME,INSTR(T1.USERNAME, '_', 1，3) + 1,4) AS BEGIN_YEAR,
                    SUBSTR(T1.USERNAME,INSTR(T1.USERNAME, '_', -1，1) + 1, 4) AS END_YEAR,
                    '集中_中科BS' AS VERSION,
                    substr(T1.USERNAME, 1, INSTR(T1.USERNAME, '_', 1，1) - 1) AS XZQH_CODE,
                    '' AS XZQH_NAME
               FROM DBA_USERS T1
              WHERE T1.USERNAME IN ('330104_ZK2_JZHS_2017_2018')) t2
    on t1.begin_year = t2.begin_year
   and t1.end_year = t2.end_year
   and t1.version = t2.version
   and t1.xzqh_code = t2.xzqh_code

获得了查询的结果数据之后用控件复制记录到结果；
设置变量；

var prevRow=previous_result.getRows();

if (prevRow == null&&(prevRow.size()=0))
{

   false;

}else{

     parent_job.setVariable("tables",prevRow);//ArrayList存储表名变量

     parent_job.setVariable("size",prevRow.size()*13);//存储执行表的总数
     parent_job.setVariable("i",0);//循环控制变量

     parent_job.setVariable("DB_ID",prevRow.get(0).getString("DB_ID",""));
     parent_job.setVariable("DB_NAME",prevRow.get(0).getString("DB_NAME",""));
     parent_job.setVariable("DW_CODE",prevRow.get(0).getString("DW_CODE",""));
     parent_job.setVariable("DW_NAME",prevRow.get(0).getString("DW_NAME",""));
     parent_job.setVariable("ZT_CODE",prevRow.get(0).getString("ZT_CODE",""));
     parent_job.setVariable("ZT_NAME",prevRow.get(0).getString("ZT_NAME",""));
     parent_job.setVariable("BEGIN_YEAR",prevRow.get(0).getString("BEGIN_YEAR",""));
     parent_job.setVariable("END_YEAR",prevRow.get(0).getString("END_YEAR",""));
     parent_job.setVariable("VERSION",prevRow.get(0).getString("VERSION",""));
     parent_job.setVariable("XZQH_CODE",prevRow.get(0).getString("XZQH_CODE",""));
     parent_job.setVariable("XZQH_NAME",prevRow.get(0).getString("XZQH_NAME",""));
     parent_job.setVariable("JCZL_USER",prevRow.get(0).getString("JCZL_USER",""));
     parent_job.setVariable("JZHS_USER",prevRow.get(0).getString("JZHS_USER",""));

     true;
}

获取之前查询到的所有字段，设置i的初始值为0，用以控制循环
size乘以字段数量

检验字段的值，用i和size来控制循环

插入DB_list表

INSERT INTO DB_LIST_ZKBS
SELECT '${DB_ID}'      as DB_ID,
       '${DB_NAME}'    as DB_NAME,
       '${DW_CODE}'    as DW_CODE,
       '${DW_NAME}'    as DW_NAME,
       '${ZT_CODE}'    as ZT_CODE,
       '${ZT_NAME}'    as ZT_NAME,
       '${BEGIN_YEAR}' as BEGIN_YEAR,
       '${END_YEAR}'   as END_YEAR,
       '${VERSION}'    as VERSION,
       '${XZQH_CODE}'    as XZQH_CODE,
       '${XZQH_NAME}'    as XZQH_NAME,
       '${JCZL_USER}'    as JCZL_USER,
       '${JZHS_USER}'    as JZHS_USER
  FROM DUAL;

计数器的自加

var list_tables = parent_job.getVariable("tables").replace("[","").replace("]","").split(",");

var size = new Number(parent_job.getVariable("size"));
var i = new Number(parent_job.getVariable("i"))+13;
if (i<size){

      
      parent_job.setVariable("DB_ID",list_tables[i].trim());
      parent_job.setVariable("DB_NAME",list_tables[i+1].trim());
      parent_job.setVariable("DW_CODE",list_tables[i+2].trim());
      parent_job.setVariable("DW_NAME",list_tables[i+3].trim());
      parent_job.setVariable("ZT_CODE",list_tables[i+4].trim());
      parent_job.setVariable("ZT_NAME",list_tables[i+5].trim());
      parent_job.setVariable("BEGIN_YEAR",list_tables[i+6].trim());
      parent_job.setVariable("END_YEAR",list_tables[i+7].trim());
      parent_job.setVariable("VERSION",list_tables[i+8].trim());
      parent_job.setVariable("XZQH_CODE",list_tables[i+9].trim());
      parent_job.setVariable("XZQH_NAME",list_tables[i+10].trim());
      parent_job.setVariable("JCZL_USER",list_tables[i+11].trim());
      parent_job.setVariable("JZHS_USER",list_tables[i+12].trim());


}
parent_job.setVariable("i",i);
true;