4月14日 从文件(*.csv或*.txt)导入Oracle数据库Java源代码

近期由于项目需要，实现了从.csv或者.txt文件中读取每行信息后插入到Oracle数据库中的功能，特共享出来。用到第三方开源工具包：opencsv-1.8.jar和commons-dbutils-1.1.jar，需单独下载。
首先需要配置XML文件，如下：
<?xml version="1.0" encoding="UTF-8"?>
<root>
<DefaultImport>Employee</DefaultImport>
<TargetTable tableName="EMPLOYEE_TEST">
<!--表中对应文件的列名和类型信息-->
<ColumnFileInfo>
   <Column name="ID" type="String"/>
   <Column name="DEP" type="String"/>
   <Column name="NAME" type="String"/>
   <Column name="AREA" type="String"/>
   <Column name="AGE" type="Number"/>
   <Column name="SEX" type="String"/>
   <Column name="XUELI" type="String"/>
   <Column name="SALARY" type="Number"/>
   <Column name="PRIX" type="Date"/>
</ColumnFileInfo>
</TargetTable>

<TargetTable tableName="YFJBXX_TEST">
   <ColumnFileInfo>
    <Column name="GLGXZJ" type="String"/>
   <Column name="YXTZJ" type="String"/>  
   <Column name="YFXM" type="String"/>   
   <Column name="JZDZ" type="String"/>   
   <Column name="JZDXZQH_DM" type="String"/>   
   <Column name="HJDZ" type="String"/>   
   <Column name="HJDXZQH_DM" type="String"/>   
   <Column name="SFZH" type="String"/>     
   <Column name="CSRQ" type="String"/>  
   <Column name="MZ_DM" type="String"/>   
   <Column name="WHCD_DM" type="String"/>
   <Column name="HKXZ_DM" type="String"/>
   <Column name="HYZK_DM" type="String"/>
   <Column name="CHRQ" type="String"/>   
   <Column name="JRXTRQ" type="String"/>
   <Column name="TCXTRQ" type="String"/>
   <Column name="TCYY_DM" type="String"/>
   <Column name="ZF_XM" type="String"/>  
   <Column name="ZF_JJDZ" type="String"/>   
   <Column name="ZF_JJDXZQHDM_DM" type="String"/>   
   <Column name="ZF_HJDZ" type="String"/>   
   <Column name="ZF_HJDXZQH_DM" type="String"/>   
   <Column name="ZF_SFZH" type="String"/>   
   <Column name="ZF_CSRQ" type="String"/>   
   <Column name="ZF_MZ_DM" type="String"/>   
   <Column name="ZF_WHCD_DM" type="String"/>
   <Column name="ZF_HKXZ_DM" type="String"/>
   <Column name="LGRZRQ" type="String"/>      
</ColumnFileInfo>
<ColumnCodeInfo>
    <Column name="LRSJ" type="Date"/>
   <Column name="SBPCH" type="String"/>
   <Column name="SBDWXZQH_DM" type="String"/>
</ColumnCodeInfo>
</TargetTable>

<ImportDealClassList>
   <Bean name="ImportDirectImpl" class="fileimport.ImportDirectImpl"/>
   <Bean name="ImportWISImpl" class="fileimport.ImportWISImpl"/>
</ImportDealClassList>
<MaxCommitNumber>5000</MaxCommitNumber>
</root>

文件中描述了2个导入工作：(1)表EMPLOYEE_TEST(2)表YFJBXX_TEST，文件中每张表的列名和类型需要和导入的目标表一致，且顺序和文件中要导入的内容相匹配，例如下面的csv文件：

"ID","DEP","NAME","AREA","AGE","SEX","XUELI","SALARY","PRIX"
"001","研发","张三","北京","34","女","本科","4546","1"
"002","销售","李四","天津","45","男","专科","4456","2"

可以看到导入的顺序和上述XML文件中的列名一致。

如果从文件导入到数据库中完全匹配，只需配置文件中的<Bean name="ImportDirectImpl" class="fileimport.ImportDirectImpl"/>即可，代码中调用示例：
    //设置XML配置文件所在位置
    FileImportInitFactory.setConfigFileName("E:/EclipseProjects/WISImport/bin/fileimport/FileImportConfig.xml");
&nb

sp;   FileImportInitFactory.init();
    HashMap h = new HashMap(1,1);
    //如果有日期型的列，需要设置DateFormat并放入HashMap中
    h.put("DateFormat","yyyy-MM-dd HH:mm:ss");
    //执行导入工作
    new ImportFileEntry().importFile("导入的文件路径及名称.csv","YFJBXX_TEST",false,"ImportDirectImpl",h);

ImportFileEntry()的importFile方法说明：
/**
   * 从文件导入到指定表中
   * @param fileName String 要导入文件名
   * @param tableName String 导入目标表名
   * @param firstLineRead boolean 是否读取第一行
   * @param dealClass 处理类名称（例如配置文件中Bean name="ImportDirectImpl"）
   * @param aHashMap 扩展用，需特殊处理时可置入变量
   * @throws Exception
   */
public void importFile(String fileName, String tableName, boolean firstLineRead,String dealClass,HashMap aHashMap) throws Exception ;

至此，一个简单的不需做任何处理直接从文件导入数据库对应表的功能就实现了。

但是有些时候我们需要进行特殊的处理，例如表中的当前操作日期列在导入文件中没有，需要在代码中加入，这时就需要实现FileImportInterface接口并加入到配置文件中例如：<Bean name="ImportWISImpl" class="fileimport.ImportWISImpl"/>，ImportWISImpl的实现代码见后续代码清单。

t;);
            columnListInfoCode[i][0] = columnName;
            columnListInfoCode[i][1] = columnType;
            System.out.println("Code columnName:" + columnName + " columnType:" + columnType);
          }
        }
        //生成实例
        ImportTableInfoBO importAction = new ImportTableInfoBO();
        importAction.setTableName(tableName);
        importAction.setColumnNamesFile(columnListInfoFile);
        if (columnListInfoCode != null) {
          importAction.setColumnNamesCode(columnListInfoCode);
        }
        //放入静态容器中
        importJobList.put(tableName, importAction);
      }
      //2.其他配置信息
      Element importDealClassList = (Element) eroot.getElementsByTagName("ImportDealClassList").item(0);
      String className;
      String classFullName;
      NodeList beanList = importDealClassList.getElementsByTagName("Bean");
      for (int j=0; j < beanList.getLength(); j++){
        className = ( (Element) beanList.item(j)).getAttribute("name");
        classFullName = ( (Element) beanList.item(j)).getAttribute("class");
        dealClassList.put(className,classFullName);
      }
     
      System.out.println("importJobList.size()" + importJobList.size());
      System.out.println("dealClassList.size()" + dealClassList.size());

 

;      insertNum = 1;
        } else {
          insertNum++;
        }
      }
    } catch (Exception e) {
      e.printStackTrace();
      //---写入数据库待加入----
      //log.error(e.getMessage());
    }
}

public void close() {
    if (conn != null) {
      try {
        if (conn != null) {
          DbUtils.commitAndClose(conn);
        System.out.println("close end");
        }
      } catch (SQLException e) {
        e.printStackTrace();
        //log.error(e.getCause());
      }
    }
}

/**
   *
   * @param dateString String
   * @param format String
   * @return Date
   */
public Date formatDate(String dateString,String format){
    try{
      SimpleDateFormat f = new SimpleDateFormat(format);
      return f.parse(dateString);
    }catch(Exception e){
      return null;
    }
}

}

八、调用示例
package fileimport;

import java.util.HashMap;

public class ImportFileExample {
public ImportFileExample() {
}
public static void main(String[] args) {
   try {
     String ls = "c:/temp/employee_test.csv";
     FileImportInitFactory.setConfigFileName("c:/temp/FileImportConfig.xml");
     FileImportInitFactory.init();
     HashMap h = new HashMap(1,1);
     h.put("DateFormat","yyyy-MM-dd HH:mm:ss");
     new ImportFileEntry().importFile(ls,"EMPLOYEE_TEST",false,"ImportDirectImpl",h);
     //new ImportFileEntry().importFile(ls,"YFJBXX_TEST",false,"ImportWISImpl",h);
   } catch (Exception e) {
     e.printStackTrace();
   }
}
}

文章出处：http://www.diybl.com/course/3_program/java/javashl/2008521/117640_8.html

 

源代码清单：
一、读取配置文件，初始化参数
package fileimport;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import java.io.File;
import java.util.HashMap;

/**
* 配置文件中读取信息赋值到创建实例，放入容器中
* @author lihf
* @version 1.0
*/
public class FileImportInitFactory {
private static HashMap importJobList = new HashMap();
private static HashMap dealClassList = new HashMap();
private static String configFileName="./FileImportConfig.xml";

public static void setConfigFileName(String fileName){
    configFileName = fileName;
}

/**
   * 读取配置文件初始化ImportAction
   * @param fileName String 配置文件名
   */
public static void init() {
    try {
      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
      DocumentBuilder db = dbf.newDocumentBuilder();
      Document doc = db.parse(new File(configFileName));
      Element eroot = (Element) doc.getElementsByTagName("root").item(0);

      int talbeListNum = eroot.getElementsByTagName("TargetTable").getLength();
      System.out.println("talbeListNum:" + talbeListNum);
      //1、表信息
      for (int j = 0; j < talbeListNum; j++) {
        Element targetTable = (Element) eroot.getElementsByTagName("TargetTable").item(j);
        String tableName = targetTable.getAttribute("tableName");
        System.out.println("TableName:" + tableName);

        //取得文件与表中列匹配的信息列表
        Element eColumnFileInfo = (Element) targetTable.getElementsByTagName("ColumnFileInfo").item(0);
        NodeList tableColumnList = eColumnFileInfo.getElementsByTagName("Column");
        int ColumnListNum = tableColumnList.getLength();
        String columnName, columnType;
        String columnListInfoFile[][] = new String[ColumnListNum][2];
        for (int i = 0; i < ColumnListNum; i++) {
          columnName = ( (Element) tableColumnList.item(i)).getAttribute("name");
          columnType = ( (Element) tableColumnList.item(i)).getAttribute("type");
          columnListInfoFile[i][0] = columnName;
          columnListInfoFile[i][1] = columnType;
          System.out.println("File columnName:" + columnName + " columnType:" + columnType);
        }
        //取得文件中没有的列，需特殊处理入库的列信息列表
        Element eColumnCodeInfo = (Element) targetTable.getElementsByTagName("ColumnCodeInfo").item(0);
        String columnListInfoCode[][] = null;
        if (eColumnCodeInfo != null) {
          tableColumnList = eColumnCodeInfo.getElementsByTagName("Column");
          ColumnListNum = tableColumnList.getLength();

          columnListInfoCode = new String[ColumnListNum][2];
          for (int i = 0; i < ColumnListNum; i++) {
            columnName = ( (Element) tableColumnList.item(i)).getAttribute("name");
            columnType = ( (Element) tableColumnList.item(i)).getAttribute("type&quo

 

    } catch (Exception e) {
      e.printStackTrace();
    }
}

/**
   * 根据表名返回实例（表列表信息）
   * @param tableName String
   * @return ImportAction
   */
public static ImportTableInfoBO getImportAction(String tableName) {
    return (ImportTableInfoBO) importJobList.get(tableName);
}

/**
   * 取得默认的导入表操作类实例
   * @return ImportAction
   */
public static ImportTableInfoBO getDefaultImportAction() {
    try {
      //取得节点值
      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
      DocumentBuilder db = dbf.newDocumentBuilder();
      Document doc = db.parse(new File(configFileName));
      Element eroot = (Element) doc.getElementsByTagName("root").item(0);

      String defaultImport = eroot.getElementsByTagName("DefaultImport").item(0).getFirstChild().getNodeValue();
      System.out.println("DefaultImport:" + defaultImport);
      return (ImportTableInfoBO) importJobList.get(defaultImport);
    } catch (Exception e) {
      e.printStackTrace();
      return null;
    }
}
/**
   * 根据表名返回实例（表列表信息）
   * @param tableName String
   * @return ImportAction
   */
public static String getDealClass(String className) {
    return (String) dealClassList.get(className);
}

public static void main(String args[]) {
    new FileImportInitFactory().init();
}

}

二、保存配置文件中的信息类
package fileimport;

/**
* 配置文件中对应一张表的导入信息,表名、列名、列数据类型
* @author
* @version 1.0
*/
public class ImportTableInfoBO{
private String tableName;
private String columnNamesFile[][] = null;
private String columnNamesCode[][] = null;

public String[][] getColumnNamesFile() {
    return this.columnNamesFile;
}

public void setColumnNamesFile(String[][] columnNamesFile) {
    this.columnNamesFile = columnNamesFile;
}

public void setColumnNamesCode(String[][] columnNamesCode) {
    this.columnNamesCode = columnNamesCode;
}

public void setTableName(String tableName) {
    this.tableName = tableName;
}

public String[][] getColumnNamesCode() {
      return this.columnNamesCode;
}

public String getTableName() {
    return tableName;
}
}

三、导入主入口程序
package fileimport;

import java.io.FileNotFoundException;
import java.io.FileReader;
import au.com.bytecode.opencsv.CSVReader;
import java.util.HashMap;
/**
* 逐行读取文件内容，

prepareSQL(importTableInfoBO, tableName);
    }
    //---------------------------------------------------------------------------
    //如果接口实现类未实例化，则根据setDealClass()方法设置的处理类名进行实例化
    if (imp == null) {
      try {
        imp = (FileImportInterface) Class.forName(dealClass).newInstance();
        //imp = (FileImportInterface)new ImportDirectImpl();
        //System.out.println("ImportDirectImpl:" + imp);
        //nextLine数组内对象均为String类型，处理前需要对nextLine进行：String、Number、Date判断，并转换
        //convert(columnValues);
      } catch (Exception e) {
        e.printStackTrace();
        throw e;
      }
    }
    //执行数据库插入操作------------------------------------------------------------
    imp.insert(nextLine, sql, aHashMap,importTableInfoBO);
}
/**
   * 生成预处理SQL语句，例如 -- insert into emp(ID,NAME)values(?,?)
   * @param columnNamesFile String[][] 插入表中列名数组
   * @param tableName String 插入目标表名
   */
public void prepareSQL(ImportTableInfoBO importTableInfoBO,String tableName) {
// this.tableName = tableName;
// this.columnNamesFile = columnNamesFile;
    String[][] columnNamesFile = importTableInfoBO.getColumnNamesFile();
    String[][] columnNamesCode = importTableInfoBO.getColumnNamesCode();
    int fileColumnLen = columnNamesFile.length;
    int codeColumnLen = 0;
    if (columnNamesCode != null)
      codeColumnLen = columnNamesCode.length;

    StringBuffer sqlBuff = new StringBuffer(200);

    //拼装PreparedStatement使用SQL语句------------------------------
    sqlBuff.append("insert into " + tableName + "(");
    int i = 0;
    //文件与表匹配的列名
    for (i = 0; i < fileColumnLen - 1; i++) {
      sqlBuff.append(columnNamesFile[i][0]);
      sqlBuff.append(",");
    }
    sqlBuff.append(columnNamesFile[i][0]);
    //需特殊处理的列名
    if (columnNamesCode != null){
      sqlBuff.append(",");
      for (i = 0; i < codeColumnLen - 1; i++) {
        sqlBuff.append(columnNamesCode[i][0]);
        sqlBuff.append(",");
      }
      sqlBuff.append(columnNamesCode[i][0]);
    }
    sqlBuff.append(")values(");
    //列值?
    for (i = 0; i < fileColumnLen + codeColumnLen - 1; i++) {
      sqlBuff.append("?,");
    }
    sqlBuff.append("?)");

    sql = sqlBuff.toString();
    System.out.println(sql);
    //log.debug("sql:"+sql);

}
/**
   * 释放数据库连接并提交
   */
public void close() {
    if (imp != null) {
      imp.close();
    }
}
}

五、插入数据库操作需实现的接口
package fileimport;

import java.util.HashMap;

/**
* @version 1.0
*/
public interface FileImportInterface {
public void insert(Object[] nextLine,String tableName,HashMap aHashMap,ImportTableInfoBO importTableInfoBO);
public void close();
}

六、直接导入到表程序
package fileimport;

import org.apache.commons.dbutils.QueryRunner;
import java.sql.Connection;
import java.util.HashMap;
import org.apache.commons.dbutils.DbUtils;
import java.sql.SQLException;
import java.sql.DriverManager;
import java.util.Date;
import java.text.SimpleDateFormat;

/**
* 直接导入到表中，不做特殊处理.
* 如需特殊处理，可实现接口FileImportInterface，改造insert方法即可
* @author lihf
* @version 1.0
*/
public class ImportDirectImpl implements FileImportInterface {
QueryRunner qRunner;
private long maxCommitNumber = 5000;
private long insertNum = 1; //已插入但未提交的数据数量
Connection conn = null;

public ImportDirectImpl() {
    qRunner = new QueryRunner();
    if (conn == null) {
      try {
        long l1 = System.currentTimeMillis();
        Class.forName("oracle.jdbc.driver.OracleDriver");
        conn = DriverManager.getConnection("jdbc:oracle:thin:user/[email protected]:1521:dbsid", "user", "pwd");
        conn.setAutoCommit(false);
        long l2 = System.currentTimeMillis();
        System.out.println("connect db time:"+(l2 -

 

/**
   * 文件和表列完全匹配，没有任何处理直接入库
   * @param columnValues 文件对应表中列名及类型二维数组
   * @param sql 预处理SQL语句
   * @param aHashMap 扩展用，需特殊处理时置入变量
   * @param importTableInfoBO 保存导入目标表的名称、字段名、类型等信息的BO
   * @throws Exception
   */
public void insert(Object[] columnValues, String sql, HashMap aHashMap,ImportTableInfoBO importTableInfoBO) {
    String dateFormat = null;
    Object[] columnValuesInsert = new Object[columnValues.length];
    //日期型数据处理-----------------------------------------------------------------
    if (aHashMap != null){
      dateFormat = (String) aHashMap.get("DateFormat");
      //System.out.println("dateFormat:" + dateFormat);
      //System.out.println("importTableInfoBO:" + importTableInfoBO);
    }
    if (importTableInfoBO != null) {
      String[][] columnList = importTableInfoBO.getColumnNamesFile();
      //System.out.println("columnList.length:"+columnList.length);
      for (int i = 0; i < columnList.length; i++) {
        String dataType = columnList[i][1];
        if (dataType.equalsIgnoreCase("Date")) {
          Date d = formatDate(((String)columnValues[i]), dateFormat);
          //System.out.println("d:"+d);
          java.sql.Date sd = new java.sql.Date(d.getTime());
          columnValuesInsert[i] = (Object)sd;
        }else{
          columnValuesInsert[i] = (Object) columnValues[i];
          //System.out.println("columnValuesInsert["+i+"]:"+columnValuesInsert[i]);
        }
      }
    }

    //-----------------------------------------------------------------------------
    try {
      //System.out.println("columnValuesInsert.length:"+columnValuesInsert.length);
      int result = qRunner.update(conn, sql, columnValuesInsert);
      if (result > 0) {
        //log.debug("第"+(insertNum)+"条数据插入成功！");
        //System.out.println("第"+(insertNum)+"条数据插入成功！");
        if (insertNum >= maxCommitNumber) { //WIASystemConfig.getInstance().getLong("5000")
          conn.commit();
          //log.info(insertNum+" numbers committed succeed...insertNum归零");
          //System.out.println(insertNum+" numbers committed succeed...insertNum归零");
          insertNum = 1;
        } else {
          insertNum++;
        }
      }
    } catch (Exception e) {
      e.printStackTrace();
      //---写入数据库待加入----
      //log.error(e.getMessage());
    }
}

public void close() {
    if (conn != null) {
      try {
        if (conn != null) {
          DbUtils.commitAndClose(conn);
        System.out.println("close end");
        }
      } catch (SQLException e) {
        e.printStackTrace();
        //log.error(e.getCause());
      }
    }
}

/**
   *
   * @param dateString String
   * @param format String
   * @return Date
   */
public Date formatDate(String dateString,String format){
    try{
      SimpleDateFormat f = new SimpleDateFormat(format);
      return f.parse(dateString);
    }catch(Exception e){
      return null;
    }
}

}

七、某些列需特殊处理后插入程序
package fileimport;

import org.apache.commons.dbutils.QueryRunner;
import java.sql.Connection;
import java.util.HashMap;
import org.apache.commons.dbutils.DbUtils;
import java.sql.SQLException;
import java.sql.DriverManager;
import java.util.Date;
import java.text.SimpleDateFormat;

 

l1));
      } catch (Exception ex) {
        ex.printStackTrace();
      }
    }
}