Java nio实现的PostgrelSql数据快速导入
1.PostgreSql快速复制的实现
Data.Java
public void setTableData(String tableName, InputStream pis, char copyDelimiter)
throws Exception
{
PGConnection copyConn = (PGConnection)this.conn.getConnection();
CopyManager copy = copyConn.getCopyAPI();
copy.copyIn("COPY " + tableName + " FROM STDIN WITH DELIMITER AS '" + copyDelimiter + "'", pis);
}
2.InputStream的构造
MTK.java
public LargeBufferInputStream getTableData(
MTKConstants.DATABASE_TYPE sourceDBType,
MTKConstants.DATABASE_TYPE targetDBType, Table table,
char copyDelimeter, boolean escapeTabDelimiter) throws Exception {
LargeBufferInputStream pis = new LargeBufferInputStream(MaxBufferSize);
PipedOutputStream pos = new PipedOutputStream();
String newLineSequence = "\\\\n";
String carriageReturnSequence = "\\\\r";
String columnTabDelimiter = "\\\\t";
pis.connect(pos);
try {
if (!this.hasMoreData) {
if (this.lastTable.compareToIgnoreCase(table.getFullName()) != 0) {
this.stmt = this.dbConn.createStatement();
if ((sourceDBType == MTKConstants.DATABASE_TYPE.POSTGRESQL)
|| (sourceDBType == MTKConstants.DATABASE_TYPE.ENTERPRISEDB)) {
this.defaultAutoCommitMode = this.dbConn
.getAutoCommit();
this.dbConn.setAutoCommit(false);
this.stmt.setFetchSize(100);
}
this.rs = this.stmt
.executeQuery(sourceDBType == MTKConstants.DATABASE_TYPE.ORACLE ? table
.getSelectSQL(sourceDBType) : table
.getSelectSQL());
this.lastTable = table.getFullName();
} else {
return null;
}
}
while (this.rs.next()) {
for (int i = 0; i < table.getColumns().size(); i++) {
String lastField = null;
boolean largeObjFlag = false;
Column column = table.getColumns().get(i);
switch (column.getDataType().ordinal()) {
case 1:
if (this.rs.getObject(i + 1) != null) {
lastField = String.valueOf(this.rs.getInt(i + 1));
}
break;
case 2:
case 3:
case 4:
case 5:
case 6:
case 7:
case 8:
lastField = this.rs.getString(i + 1);
if (lastField != null) {
if (MigrationToolkit.replaceNullChar()) {
lastField = lastField.replaceAll("", String
.valueOf(MigrationToolkit
.nullReplacementChar()));
} else {
lastField = lastField.replaceAll("", "");
}
lastField = lastField.replace("\\", "\\\\");
lastField = lastField.replaceAll("\n",
newLineSequence);
lastField = lastField.replaceAll("\r",
carriageReturnSequence);
if ((escapeTabDelimiter) && (copyDelimeter == '\t')) {
lastField = lastField.replaceAll("\t",
columnTabDelimiter);
}
if ((sourceDBType == MTKConstants.DATABASE_TYPE.POSTGRESQL)
|| (sourceDBType == MTKConstants.DATABASE_TYPE.ENTERPRISEDB)) {
if ((column.getDataTypeString()
.equalsIgnoreCase("TEXT"))
|| (column.getDataTypeString()
.equalsIgnoreCase("XML"))
|| (column.getDataTypeString()
.equalsIgnoreCase("JSON"))) {
table.setPgTextColSize(lastField.length());
}
}
}
break;
case 9:
if (this.rs.getObject(i + 1) != null) {
lastField = new Double(this.rs.getDouble(i + 1))
.toString();
}
break;
case 10:
case 11:
case 12:
case 13:
case 14:
case 15:
if (this.rs.getObject(i + 1) != null) {
if ((column.isReal()) || (column.isBinaryFloat())) {
lastField = new Float(this.rs.getFloat(i + 1))
.toString();
} else if (column.isDouble()) {
lastField = new Double(this.rs.getDouble(i + 1))
.toString();
} else {
lastField = this.rs.getBigDecimal(i + 1)
.toString();
}
}
break;
case 16:
if (this.rs.getTime(i + 1) != null) {
if (((sourceDBType == MTKConstants.DATABASE_TYPE.POSTGRESQL) || (sourceDBType == MTKConstants.DATABASE_TYPE.ENTERPRISEDB))
&& ((targetDBType == MTKConstants.DATABASE_TYPE.POSTGRESQL) || (targetDBType == MTKConstants.DATABASE_TYPE.ENTERPRISEDB))) {
lastField = this.rs.getString(i + 1);
} else {
lastField = this.rs.getTime(i + 1).toString();
}
}
break;
case 17:
if (this.rs.getTimestamp(i + 1) != null) {
lastField = this.rs.getTimestamp(i + 1).toString();
}
break;
case 18:
if (this.rs.getTimestamp(i + 1) != null) {
if (((sourceDBType == MTKConstants.DATABASE_TYPE.POSTGRESQL) || (sourceDBType == MTKConstants.DATABASE_TYPE.ENTERPRISEDB))
&& ((targetDBType == MTKConstants.DATABASE_TYPE.POSTGRESQL) || (targetDBType == MTKConstants.DATABASE_TYPE.ENTERPRISEDB))) {
lastField = this.rs.getString(i + 1);
} else {
lastField = this.rs.getTimestamp(i + 1)
.toString();
}
}
break;
case 19:
largeObjFlag = true;
break;
case 20:
if (this.rs.getString(i + 1) != null) {
if ((sourceDBType == MTKConstants.DATABASE_TYPE.ORACLE)
&& ((targetDBType == MTKConstants.DATABASE_TYPE.ENTERPRISEDB) || (targetDBType == MTKConstants.DATABASE_TYPE.POSTGRESQL))) {
lastField = RedwoodBridge
.getPGIntervalString(this.rs
.getString(i + 1));
} else {
lastField = this.rs.getString(i + 1);
}
}
break;
case 21:
lastField = String.valueOf(this.rs.getBoolean(i + 1));
break;
case 22:
lastField = this.rs.getString(i + 1);
break;
case 23:
if (this.rs.getArray(i + 1) != null) {
lastField = this.rs.getArray(i + 1).toString();
}
break;
case 24:
case 25:
case 26:
lastField = this.rs.getString(i + 1);
break;
case 27:
case 28:
lastField = this.rs.getString(i + 1);
break;
default:
if ((this.rs.getMetaData().getColumnTypeName(i + 1)
.equalsIgnoreCase("CLOB"))
|| (this.rs.getMetaData().getColumnTypeName(
i + 1).equalsIgnoreCase("NCLOB"))) {
throw new Exception(
"This Table Contains CLOB data, Marked for Bulk Insert Loading");
}
if ((this.rs.getMetaData().getColumnType(i + 1) == 2002)
&& (!column.getDataTypeString()
.equalsIgnoreCase("SDO_GEOMETRY"))) {
lastField = RedwoodBridge
.convertToPGObjectTypeFormat(this.rs
.getObject(i + 1));
} else if (column.getDataTypeString().equalsIgnoreCase(
"SDO_GEOMETRY")) {
lastField = null;
} else {
throw new Exception(this.rs.getMetaData()
.getColumnTypeName(i + 1)
+ " is Not Supported by COPY");
}
break;
}
if ((this.rs.wasNull()) || (largeObjFlag)
|| (lastField == null)) {
if (targetDBType != MTKConstants.DATABASE_TYPE.NPS) {
pos.write(new String("\\N").getBytes());
}
} else if (column.getDataType() == Column.DataType.NVARCHAR) {
pos.write(lastField.getBytes("UTF-8"));
} else {
pos.write(lastField.getBytes(this.sourceDBNamedCharSet));
}
if (i < table.getColumns().size() - 1) {
pos.write(copyDelimeter);
}
}
pos.write(10);
pis.addRowNum();
if (MaxBufferSize - pis.getLength() < table.getMaxRowSize()) {
pos.flush();
pos.close();
this.hasMoreData = true;
return pis;
}
}
this.rs.close();
this.stmt.close();
this.hasMoreData = false;
if ((sourceDBType == MTKConstants.DATABASE_TYPE.POSTGRESQL)
|| (sourceDBType == MTKConstants.DATABASE_TYPE.ENTERPRISEDB)) {
if (this.dbConn != null) {
this.dbConn.setAutoCommit(this.defaultAutoCommitMode);
}
}
pos.flush();
pos.close();
} catch (Exception ex) {
if (this.rs != null) {
this.rs.close();
}
if (this.stmt != null) {
this.stmt.close();
}
if ((sourceDBType == MTKConstants.DATABASE_TYPE.POSTGRESQL)
|| (sourceDBType == MTKConstants.DATABASE_TYPE.ENTERPRISEDB)) {
if (this.dbConn != null) {
this.dbConn.setAutoCommit(this.defaultAutoCommitMode);
}
}
throw ex;
}
return pis;
}
3.总结
数据迁移时需要对大表数据切分成小块，如果直接使用分页效率非常低下，而采用流的模式进程自动从流中读取数据并写入到流中，避免了分页。