重写JdbcRDD支持Sql命名参数和分区
Spark提供的JdbcRDD很不好用，没法指定命名参数，而且必须要提供两个Long类型的参数表示分区的范围，如果数据表没有long类型的字段或者不需要条件，那就不能用JdbcRDD了。这里我简单重写了一个GenericJdbcRDD，支持命名参数和分区，仅仅作为参考哦。
项目结构如下：
GenericJdbcRDD.scala
package yay.jdbc.rdd import java.sql.{Connection, ResultSet} import org.apache.spark.annotation.DeveloperApi import org.apache.spark.rdd.{JdbcRDD, RDD} import org.apache.spark.{Logging, Partition, SparkContext, TaskContext} import scala.reflect.ClassTag /** * Created by ${yuananyun} on 2015/2/7. */ private[yay] class JdbcPartition(idx: Int, val lower: Long, val upper: Long) extends Partition { override def index = idx } class GenericJdbcRDD[T: ClassTag]( sc: SparkContext, getConnection: () => Connection, sql: String, paramsMap: Map[String, Object], skip: Long, take: Long, numPartitions: Int, mapRow: (ResultSet) => T = JdbcRDD.resultSetToObjectArray _) extends RDD[T](sc, Nil) with Logging { @DeveloperApi override def compute(thePart: Partition, context: TaskContext) = new JdbcNextIterator[T] { context.addTaskCompletionListener { context => closeIfNeeded()} val part = thePart.asInstanceOf[JdbcPartition] val conn = getConnection() // val stmt = conn.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY) var parsedSql = "" if (conn.getMetaData.getURL.matches("jdbc:mysql:.*")) { parsedSql = sql+" limit "+ part.lower+","+ part.upper } val stmt = new NamedParameterStatement(conn, parsedSql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY) // setFetchSize(Integer.MIN_VALUE) is a mysql driver specific way to force streaming results, // rather than pulling entire resultset into memory. // see http://dev.mysql.com/doc/refman/5.0/en/connector-j-reference-implementation-notes.html if (conn.getMetaData.getURL.matches("jdbc:mysql:.*")) { stmt.setFetchSize(Integer.MIN_VALUE) logInfo("statement fetch size set to: " + stmt.getFetchSize + " to force MySQL streaming ") } if (paramsMap != null && paramsMap.size > 0) { val paramsIter = paramsMap.iterator while (paramsIter.hasNext) { val (key, value) = paramsIter.next() stmt.setObject(key, value) } } val rs = stmt.executeQuery() override def getNext: T = { if (rs.next()) { mapRow(rs) } else { finished = true null.asInstanceOf[T] } } override def close() { try { if (null != rs && !rs.isClosed()) { rs.close() } } catch { case e: Exception => logWarning("Exception closing resultset", e) } try { if (null != stmt && !stmt.isClosed()) { stmt.close() } } catch { case e: Exception => logWarning("Exception closing statement", e) } try { if (null != conn && !conn.isClosed()) { conn.close() } logInfo("closed connection") } catch { case e: Exception => logWarning("Exception closing connection", e) } } } override protected def getPartitions: Array[Partition] = { take > 0 match { case false => throw new IllegalArgumentException("take 参数不能小于0") case _ => { val step = take / numPartitions (0 until numPartitions).map(i => { val start = (skip + i * step) val end = start + step new JdbcPartition(i, start, end) }).toArray } } } }
JdbcNextIterator.scala
package yay.jdbc.rdd /** * Created by ${yuananyun} on 2015/2/8. */ abstract private[yay] class JdbcNextIterator[U] extends Iterator[U] { private var gotNext = false private var nextValue: U = _ private var closed = false protected var finished = false /** * Method for subclasses to implement to provide the next element. * * If no next element is available, the subclass should set `finished` * to `true` and may return any value (it will be ignored). * * This convention is required because `null` may be a valid value, * and using `Option` seems like it might create unnecessary Some/None * instances, given some iterators might be called in a tight loop. * * @return U, or set 'finished' when done */ protected def getNext(): U /** * Method for subclasses to implement when all elements have been successfully * iterated, and the iteration is done. * * <b>Note:</b> `NextIterator` cannot guarantee that `close` will be * called because it has no control over what happens when an exception * happens in the user code that is calling hasNext/next. * * Ideally you should have another try/catch, as in HadoopRDD, that * ensures any resources are closed should iteration fail. */ protected def close() /** * Calls the subclass-defined close method, but only once. * * Usually calling `close` multiple times should be fine, but historically * there have been issues with some InputFormats throwing exceptions. */ def closeIfNeeded() { if (!closed) { close() closed = true } } override def hasNext: Boolean = { if (!finished) { if (!gotNext) { nextValue = getNext() if (finished) { closeIfNeeded() } gotNext = true } } !finished } override def next(): U = { if (!hasNext) { throw new NoSuchElementException("End of stream") } gotNext = false nextValue } }
NamedParameterStatement.java
package yay.jdbc.rdd; import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; import java.net.URL; import java.sql.*; import java.sql.Date; import java.util.*; /** * This class wraps around a {@link PreparedStatement} and allows the programmer to set parameters by name instead * of by index. This eliminates any confusion as to which parameter index represents what. This also means that * rearranging the SQL statement or adding a parameter doesn't involve renumbering your indices. * Code such as this: * <p/> * <pre><code> * Connection conn = getConnection(); * String sql = "select * from my_table where name=? or address=?"; * PreparedStatement p = conn.prepareStatement(sql); * p.setString(1, "bob"); * p.setString(2, "123"); * ResultSet rs = p.executeQuery(); * </code></pre> * <p/> * Can be replaced with: * <p/> * <pre><code> * Connection conn = getConnection(); * String sql = "select * from my_table where name=:name or address=:address"; * NamedParameterStatement p = new NamedParameterStatement(conn, sql); * p.setString("name", "bob"); * p.setString("address", "123"); * ResultSet rs = p.executeQuery(); * </code></pre> */ public class NamedParameterStatement extends PreparedStatementWrapper { private static final HashMap<String, Map<String, List<Integer>>> nameIndexCache = new HashMap<String, Map<String, List<Integer>>>(); private static final HashMap<String, String> parsedSqlCache = new HashMap<String, String>(); private final String parsedSql; private final Map<String, List<Integer>> nameIndexMap; /** * Creates a NamedParameterStatement. Wraps a call to * c.{@link Connection#prepareStatement(java.lang.String) prepareStatement}. * * @param conn the database connection * @param sql the parameterized sql * @throws SQLException if the statement could not be created */ public NamedParameterStatement(Connection conn, String sql,int resultSetType,int resultSetConcurrency) throws SQLException { if (nameIndexCache.containsKey(sql)) { nameIndexMap = nameIndexCache.get(sql); parsedSql = parsedSqlCache.get(sql); } else { nameIndexMap = new HashMap<String, List<Integer>>(); parsedSql = parseNamedSql(sql, nameIndexMap); nameIndexCache.put(sql, nameIndexMap); parsedSqlCache.put(sql, parsedSql); } ps = conn.prepareStatement(parsedSql,resultSetType,resultSetConcurrency); } /** * Returns the indexes for a parameter. * * @param name parameter name * @return parameter indexes * @throws IllegalArgumentException if the parameter does not exist */ private List<Integer> getIndexes(String name) { List<Integer> indexes = nameIndexMap.get(name); if (indexes == null) { throw new IllegalArgumentException("Parameter not found: " + name); } return indexes; } /** * Parses a sql with named parameters. The parameter-index mappings * are put into the map, and the parsed sql is returned. * * @param sql sql with named parameters * @return the parsed sql */ private static String parseNamedSql(String sql, Map<String, List<Integer>> nameIndexMap) { // I was originally using regular expressions, but they didn't work well for ignoring // parameter-like strings inside quotes. int length = sql.length(); StringBuffer parsedSql = new StringBuffer(length); boolean inSingleQuote = false; boolean inDoubleQuote = false; int index = 1; for (int i = 0; i < length; i++) { char c = sql.charAt(i); if (inSingleQuote) { if (c == '\'') { inSingleQuote = false; } } else if (inDoubleQuote) { if (c == '"') { inDoubleQuote = false; } } else { if (c == '\'') { inSingleQuote = true; } else if (c == '"') { inDoubleQuote = true; } else if (c == ':' && i + 1 < length && Character.isJavaIdentifierStart(sql.charAt(i + 1))) { int j = i + 2; while (j < length && Character.isJavaIdentifierPart(sql.charAt(j))) { j++; } String name = sql.substring(i + 1, j); c = '?'; // replace the parameter with a question mark i += name.length(); // skip past the end if the parameter List<Integer> indexList = nameIndexMap.get(name); if (indexList == null) { indexList = new LinkedList<Integer>(); nameIndexMap.put(name, indexList); } indexList.add(index); index++; } } parsedSql.append(c); } return parsedSql.toString(); } public void setArray(String name, Array value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setArray(index, value); } } public void setAsciiStream(String name, InputStream value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setAsciiStream(index, value); } } public void setAsciiStream(String name, InputStream value, int length) throws SQLException { for (Integer index : getIndexes(name)) { ps.setAsciiStream(index, value, length); } } public void setBigDecimal(String name, BigDecimal value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setBigDecimal(index, value); } } public void setBinaryStream(String name, InputStream value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setBinaryStream(index, value); } } public void setBinaryStream(String name, InputStream value, int length) throws SQLException { for (Integer index : getIndexes(name)) { ps.setBinaryStream(index, value, length); } } public void setBinaryStream(String name, InputStream value, long length) throws SQLException { for (Integer index : getIndexes(name)) { ps.setBinaryStream(index, value, length); } } public void setBlob(String name, Blob value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setBlob(index, value); } } public void setBlob(String name, InputStream value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setBlob(index, value); } } public void setBlob(String name, InputStream value, long length) throws SQLException { for (Integer index : getIndexes(name)) { ps.setBlob(index, value, length); } } public void setBoolean(String name, boolean value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setBoolean(index, value); } } public void setByte(String name, byte value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setByte(index, value); } } public void setBytes(String name, byte[] value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setBytes(index, value); } } public void setCharacterStream(String name, Reader value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setCharacterStream(index, value); } } public void setCharacterStream(String name, Reader value, int length) throws SQLException { for (Integer index : getIndexes(name)) { ps.setCharacterStream(index, value, length); } } public void setCharacterStream(String name, Reader value, long length) throws SQLException { for (Integer index : getIndexes(name)) { ps.setCharacterStream(index, value, length); } } public void setClob(String name, Clob value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setClob(index, value); } } public void setClob(String name, Reader value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setClob(index, value); } } public void setClob(String name, Reader value, long length) throws SQLException { for (Integer index : getIndexes(name)) { ps.setClob(index, value, length); } } public void setDate(String name, Date value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setDate(index, value); } } public void setDate(String name, Date value, Calendar cal) throws SQLException { for (Integer index : getIndexes(name)) { ps.setDate(index, value, cal); } } public void setDouble(String name, double value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setDouble(index, value); } } public void setFloat(String name, float value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setFloat(index, value); } } public void setInt(String name, int value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setInt(index, value); } } public void setLong(String name, long value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setLong(index, value); } } public void setNCharacterStream(String name, Reader value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setNCharacterStream(index, value); } } public void setNCharacterStream(String name, Reader value, long length) throws SQLException { for (Integer index : getIndexes(name)) { ps.setNCharacterStream(index, value, length); } } public void setNClob(String name, NClob value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setNClob(index, value); } } public void setNClob(String name, Reader value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setNClob(index, value); } } public void setNClob(String name, Reader value, long length) throws SQLException { for (Integer index : getIndexes(name)) { ps.setNClob(index, value, length); } } public void setNString(String name, String value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setNString(index, value); } } public void setNull(String name, int sqlType) throws SQLException { for (Integer index : getIndexes(name)) { ps.setNull(index, sqlType); } } public void setObject(String name, Object value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setObject(index, value); } } public void setObject(String name, Object value, int targetSqlType) throws SQLException { for (Integer index : getIndexes(name)) { ps.setObject(index, value, targetSqlType); } } public void setObject(String name, Object value, int targetSqlType, int scaleOrLength) throws SQLException { for (Integer index : getIndexes(name)) { ps.setObject(index, value, targetSqlType, scaleOrLength); } } public void setRef(String name, Ref value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setRef(index, value); } } public void setRowId(String name, RowId value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setRowId(index, value); } } public void setShort(String name, short value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setShort(index, value); } } public void setSQLXML(String name, SQLXML value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setSQLXML(index, value); } } public void setString(String name, String value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setString(index, value); } } public void setTime(String name, Time value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setTime(index, value); } } public void setTime(String name, Time value, Calendar cal) throws SQLException { for (Integer index : getIndexes(name)) { ps.setTime(index, value, cal); } } public void setTimestamp(String name, Timestamp value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setTimestamp(index, value); } } public void setTimestamp(String name, Timestamp value, Calendar cal) throws SQLException { for (Integer index : getIndexes(name)) { ps.setTimestamp(index, value, cal); } } @SuppressWarnings("deprecation") public void setUnicodeStream(String name, InputStream value, int length) throws SQLException { for (Integer index : getIndexes(name)) { ps.setUnicodeStream(index, value, length); } } public void setURL(String name, URL value) throws SQLException { for (Integer index : getIndexes(name)) { ps.setURL(index, value); } } }
PreparedStatementWrapper.java
package yay.jdbc.rdd; import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; import java.net.URL; import java.sql.*; import java.util.Calendar; public class PreparedStatementWrapper implements PreparedStatement { protected PreparedStatement ps; @Override public <T> T unwrap(Class<T> iface) throws SQLException { return ps.unwrap(iface); } @Override public ResultSet executeQuery(String sql) throws SQLException { return ps.executeQuery(sql); } @Override public ResultSet executeQuery() throws SQLException { return ps.executeQuery(); } @Override public boolean isWrapperFor(Class<?> iface) throws SQLException { return ps.isWrapperFor(iface); } @Override public int executeUpdate(String sql) throws SQLException { return ps.executeUpdate(sql); } @Override public int executeUpdate() throws SQLException { return ps.executeUpdate(); } @Override public void setNull(int parameterIndex, int sqlType) throws SQLException { ps.setNull(parameterIndex, sqlType); } @Override public void close() throws SQLException { ps.close(); } @Override public int getMaxFieldSize() throws SQLException { return ps.getMaxFieldSize(); } @Override public void setBoolean(int parameterIndex, boolean x) throws SQLException { ps.setBoolean(parameterIndex, x); } @Override public void setByte(int parameterIndex, byte x) throws SQLException { ps.setByte(parameterIndex, x); } @Override public void setMaxFieldSize(int max) throws SQLException { ps.setMaxFieldSize(max); } @Override public void setShort(int parameterIndex, short x) throws SQLException { ps.setShort(parameterIndex, x); } @Override public int getMaxRows() throws SQLException { return ps.getMaxRows(); } @Override public void setInt(int parameterIndex, int x) throws SQLException { ps.setInt(parameterIndex, x); } @Override public void setMaxRows(int max) throws SQLException { ps.setMaxRows(max); } @Override public void setLong(int parameterIndex, long x) throws SQLException { ps.setLong(parameterIndex, x); } @Override public void setEscapeProcessing(boolean enable) throws SQLException { ps.setEscapeProcessing(enable); } @Override public void setFloat(int parameterIndex, float x) throws SQLException { ps.setFloat(parameterIndex, x); } @Override public void setDouble(int parameterIndex, double x) throws SQLException { ps.setDouble(parameterIndex, x); } @Override public int getQueryTimeout() throws SQLException { return ps.getQueryTimeout(); } @Override public void setQueryTimeout(int seconds) throws SQLException { ps.setQueryTimeout(seconds); } @Override public void setBigDecimal(int parameterIndex, BigDecimal x) throws SQLException { ps.setBigDecimal(parameterIndex, x); } @Override public void setString(int parameterIndex, String x) throws SQLException { ps.setString(parameterIndex, x); } @Override public void setBytes(int parameterIndex, byte[] x) throws SQLException { ps.setBytes(parameterIndex, x); } @Override public void cancel() throws SQLException { ps.cancel(); } @Override public SQLWarning getWarnings() throws SQLException { return ps.getWarnings(); } @Override public void setDate(int parameterIndex, Date x) throws SQLException { ps.setDate(parameterIndex, x); } @Override public void setTime(int parameterIndex, Time x) throws SQLException { ps.setTime(parameterIndex, x); } @Override public void clearWarnings() throws SQLException { ps.clearWarnings(); } @Override public void setCursorName(String name) throws SQLException { ps.setCursorName(name); } @Override public void setTimestamp(int parameterIndex, Timestamp x) throws SQLException { ps.setTimestamp(parameterIndex, x); } @Override public void setAsciiStream(int parameterIndex, InputStream x, int length) throws SQLException { ps.setAsciiStream(parameterIndex, x, length); } @Override public boolean execute(String sql) throws SQLException { return ps.execute(sql); } @Override @SuppressWarnings("deprecation") public void setUnicodeStream(int parameterIndex, InputStream x, int length) throws SQLException { ps.setUnicodeStream(parameterIndex, x, length); } @Override public ResultSet getResultSet() throws SQLException { return ps.getResultSet(); } @Override public void setBinaryStream(int parameterIndex, InputStream x, int length) throws SQLException { ps.setBinaryStream(parameterIndex, x, length); } @Override public int getUpdateCount() throws SQLException { return ps.getUpdateCount(); } @Override public boolean getMoreResults() throws SQLException { return ps.getMoreResults(); } @Override public void clearParameters() throws SQLException { ps.clearParameters(); } @Override public void setObject(int parameterIndex, Object x, int targetSqlType) throws SQLException { ps.setObject(parameterIndex, x, targetSqlType); } @Override public void setFetchDirection(int direction) throws SQLException { ps.setFetchDirection(direction); } @Override public int getFetchDirection() throws SQLException { return ps.getFetchDirection(); } @Override public void setObject(int parameterIndex, Object x) throws SQLException { ps.setObject(parameterIndex, x); } @Override public void setFetchSize(int rows) throws SQLException { ps.setFetchSize(rows); } @Override public int getFetchSize() throws SQLException { return ps.getFetchSize(); } @Override public int getResultSetConcurrency() throws SQLException { return ps.getResultSetConcurrency(); } @Override public boolean execute() throws SQLException { return ps.execute(); } @Override public int getResultSetType() throws SQLException { return ps.getResultSetType(); } @Override public void addBatch(String sql) throws SQLException { ps.addBatch(sql); } @Override public void clearBatch() throws SQLException { ps.clearBatch(); } @Override public void addBatch() throws SQLException { ps.addBatch(); } @Override public int[] executeBatch() throws SQLException { return ps.executeBatch(); } @Override public void setCharacterStream(int parameterIndex, Reader reader, int length) throws SQLException { ps.setCharacterStream(parameterIndex, reader, length); } @Override public void setRef(int parameterIndex, Ref x) throws SQLException { ps.setRef(parameterIndex, x); } @Override public void setBlob(int parameterIndex, Blob x) throws SQLException { ps.setBlob(parameterIndex, x); } @Override public void setClob(int parameterIndex, Clob x) throws SQLException { ps.setClob(parameterIndex, x); } @Override public Connection getConnection() throws SQLException { return ps.getConnection(); } @Override public void setArray(int parameterIndex, Array x) throws SQLException { ps.setArray(parameterIndex, x); } @Override public ResultSetMetaData getMetaData() throws SQLException { return ps.getMetaData(); } @Override public boolean getMoreResults(int current) throws SQLException { return ps.getMoreResults(current); } @Override public void setDate(int parameterIndex, Date x, Calendar cal) throws SQLException { ps.setDate(parameterIndex, x, cal); } @Override public ResultSet getGeneratedKeys() throws SQLException { return ps.getGeneratedKeys(); } @Override public void setTime(int parameterIndex, Time x, Calendar cal) throws SQLException { ps.setTime(parameterIndex, x, cal); } @Override public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException { return ps.executeUpdate(sql, autoGeneratedKeys); } @Override public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) throws SQLException { ps.setTimestamp(parameterIndex, x, cal); } @Override public void setNull(int parameterIndex, int sqlType, String typeName) throws SQLException { ps.setNull(parameterIndex, sqlType, typeName); } @Override public int executeUpdate(String sql, int[] columnIndexes) throws SQLException { return ps.executeUpdate(sql, columnIndexes); } @Override public void setURL(int parameterIndex, URL x) throws SQLException { ps.setURL(parameterIndex, x); } @Override public int executeUpdate(String sql, String[] columnNames) throws SQLException { return ps.executeUpdate(sql, columnNames); } @Override public ParameterMetaData getParameterMetaData() throws SQLException { return ps.getParameterMetaData(); } @Override public void setRowId(int parameterIndex, RowId x) throws SQLException { ps.setRowId(parameterIndex, x); } @Override public void setNString(int parameterIndex, String value) throws SQLException { ps.setNString(parameterIndex, value); } @Override public boolean execute(String sql, int autoGeneratedKeys) throws SQLException { return ps.execute(sql, autoGeneratedKeys); } @Override public void setNCharacterStream(int parameterIndex, Reader value, long length) throws SQLException { ps.setNCharacterStream(parameterIndex, value, length); } @Override public void setNClob(int parameterIndex, NClob value) throws SQLException { ps.setNClob(parameterIndex, value); } @Override public void setClob(int parameterIndex, Reader reader, long length) throws SQLException { ps.setClob(parameterIndex, reader, length); } @Override public boolean execute(String sql, int[] columnIndexes) throws SQLException { return ps.execute(sql, columnIndexes); } @Override public void setBlob(int parameterIndex, InputStream inputStream, long length) throws SQLException { ps.setBlob(parameterIndex, inputStream, length); } @Override public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException { ps.setNClob(parameterIndex, reader, length); } @Override public boolean execute(String sql, String[] columnNames) throws SQLException { return ps.execute(sql, columnNames); } @Override public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException { ps.setSQLXML(parameterIndex, xmlObject); } @Override public void setObject(int parameterIndex, Object x, int targetSqlType, int scaleOrLength) throws SQLException { ps.setObject(parameterIndex, x, targetSqlType, scaleOrLength); } @Override public int getResultSetHoldability() throws SQLException { return ps.getResultSetHoldability(); } @Override public boolean isClosed() throws SQLException { return ps.isClosed(); } @Override public void setPoolable(boolean poolable) throws SQLException { ps.setPoolable(poolable); } @Override public boolean isPoolable() throws SQLException { return ps.isPoolable(); } @Override public void closeOnCompletion() throws SQLException { ps.closeOnCompletion(); } @Override public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException { ps.setAsciiStream(parameterIndex, x, length); } @Override public boolean isCloseOnCompletion() throws SQLException { return ps.isCloseOnCompletion(); } @Override public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException { ps.setBinaryStream(parameterIndex, x, length); } @Override public void setCharacterStream(int parameterIndex, Reader reader, long length) throws SQLException { ps.setCharacterStream(parameterIndex, reader, length); } @Override public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException { ps.setAsciiStream(parameterIndex, x); } @Override public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException { ps.setBinaryStream(parameterIndex, x); } @Override public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException { ps.setCharacterStream(parameterIndex, reader); } @Override public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException { ps.setNCharacterStream(parameterIndex, value); } @Override public void setClob(int parameterIndex, Reader reader) throws SQLException { ps.setClob(parameterIndex, reader); } @Override public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException { ps.setBlob(parameterIndex, inputStream); } @Override public void setNClob(int parameterIndex, Reader reader) throws SQLException { ps.setNClob(parameterIndex, reader); } }
具体的用法：
object SparkToJDBC { def main(args: Array[String]): Unit = { val sc = new SparkContext("local", "mySql") val paramsMap = Map[String, Object]("msgType" -> "99", "sendTime" -> "1419821878146") val rdd = new GenericJdbcRDD(sc, () => { Class.forName("com.mysql.jdbc.Driver").newInstance() DriverManager.getConnection("jdbc:mysql://xxxxxxxx?characterEncoding=utf-8", "root", "xxxxx") }, "SELECT * FROM emb_message where msg_type=:msgType and send_time>=:sendTime",
paramsMap, 0, 1000, 3, r => (r.getString(6),r.getString(11))) // rdd.foreach(x => println(x)) rdd.saveAsTextFile("c:\\temp\\test") sc.stop() } }