Calcite是一种动态数据管理系统,它具有标准SQL、连接不同前端和后端、可定制的逻辑规划器、物化视图、多维数据分析和流查询等诸多能力,使其成为大数据领域中非常有吸引力的查询引擎.
看了好多Github 都没有一些合适的例子,自己花了一段时间整理的,希望以后会用到,代码中有很多不足的地方,就不多说,直接在本地做了一个DEMO
BaseConnection: 定义接口,所有jdbc 都需要实现这个方法
public interface BaseConnection {
public Connection getConnection();
}
JDBCConnection:implements BaseConnection
public class JDBCConnection extends FileLogger implements BaseConnection {
// 静态成员变量,支持单态模式
public static JDBCConnection jdbc = null;
public static DataSource dataSource =
JdbcSchema.dataSource(ConfigurationUtils.getProperty("calcite.url"), ConfigurationUtils.getProperty("calcite.class"),
ConfigurationUtils.getProperty("calcite.user"), ConfigurationUtils.getProperty("calcite.password"));;
private JDBCConnection() {
}
public static JDBCConnection getInstance() {
if (jdbc == null) {
synchronized (JDBCConnection.class) {
if (jdbc == null) {
jdbc = new JDBCConnection();
}
}
}
return jdbc;
}
@Override
public Connection getConnection() {
try {
if(ConfigurationUtils.getProperty("calcite.type").equals("mysql")){
return dataSource.getConnection();
}else{
Properties properties = new Properties();
properties.setProperty("spark", "true");
Connection aConnection;
try {
aConnection = DriverManager.getConnection("jdbc:calcite:", properties);
return aConnection.unwrap(CalciteConnection.class);
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
} catch (SQLException e) {
error_log.error(e.getStackTrace());
e.printStackTrace();
}
return null;
}
private static Config buildParserConfig(DatabaseProduct dbms, String parserSqlQuoting) {
ConfigBuilder aParserConfig = SqlParser.configBuilder();
switch (dbms) {
case HIVE:
case MYSQL:
aParserConfig.setLex(Lex.MYSQL);
break;
case POSTGRESQL:
aParserConfig.setCaseSensitive(false).setQuoting(Quoting.DOUBLE_QUOTE).setQuotedCasing(Casing.UNCHANGED)
.setUnquotedCasing(Casing.TO_LOWER);
break;
case MSSQL:
aParserConfig.setLex(Lex.SQL_SERVER);
break;
default:
aParserConfig.setLex(Lex.ORACLE);
break;
}
if ("ANSI".equals(parserSqlQuoting)) {
aParserConfig.setQuoting(Quoting.DOUBLE_QUOTE);
}
return aParserConfig.build();
}
/**
* 获取SchemaName
* @Title: getSchemaName
* @Description: TODO(方法简要描述,必须以句号为结束)
* @since: (开始使用的版本)
* @param theDatabaseProduct
* @param conn
* @return
* @throws SQLException
*/
private static String getSchemaName(DatabaseProduct theDatabaseProduct, final Connection conn) throws SQLException {
if (theDatabaseProduct == SqlDialect.DatabaseProduct.MYSQL) {
return conn.getCatalog();
} else {
String schemaName = conn.getSchema();
if (schemaName == null) {
schemaName = "PUBLIC";
}
return schemaName;
}
}
/**
* 创建Config 不同的数据库之间转移
* @Title: createConfig
* @Description: TODO(方法简要描述,必须以句号为结束)
* @since: (开始使用的版本)
* @param theDataSource
* @param theDatabaseProduct
* @param theCatalog
* @param theSchemaName
* @return
*/
private static FrameworkConfig createConfig(final DataSource theDataSource,
final SqlDialect.DatabaseProduct theDatabaseProduct, final String theCatalog, final String theSchemaName) {
SchemaPlus aRootSchema = Frameworks.createRootSchema(true);
aRootSchema.add(theSchemaName,
JdbcSchema.create(aRootSchema, theSchemaName, theDataSource, theCatalog, theSchemaName));
return Frameworks.newConfigBuilder().parserConfig(buildParserConfig(theDatabaseProduct, "ANSI"))
.defaultSchema(aRootSchema.getSubSchema(theSchemaName))
.traitDefs(ConventionTraitDef.INSTANCE, RelCollationTraitDef.INSTANCE).context(Contexts.EMPTY_CONTEXT)
.typeSystem(RelDataTypeSystem.DEFAULT).build();
}
/**
* 根据类型来匹配不同的转换形式
* @Title: convertSqlByType
* @Description: TODO(方法简要描述,必须以句号为结束)
* @since: (开始使用的版本)
* @param sql
* @return
* @throws SQLException
* @throws SqlParseException
* @throws ValidationException
* @throws RelConversionException
*/
public static String convertSqlByType(String sql)
throws SQLException, SqlParseException, ValidationException, RelConversionException {
if(ConfigurationUtils.getProperty("calcite.type").equals("mysql")){
return convertMySql(sql);
}else{
return sql;
}
}
/**
* Mysql以及一些支持的格式转化 为可以执行的calcite
* @Title: convertSql
* @Description: TODO(方法简要描述,必须以句号为结束)
* @since: (开始使用的版本)
* @return
* @throws SQLException
* @throws SqlParseException
* @throws ValidationException
* @throws RelConversionException
*/
public static String convertMySql(String sql)
throws SQLException, SqlParseException, ValidationException, RelConversionException {
Connection aConnection = JDBCConnection.getInstance().getConnection();
try{
SqlDialectFactory aSqlDialectFactory = new SqlDialectFactoryImpl();
SqlDialect aSqlDialect = aSqlDialectFactory.create(aConnection.getMetaData());
SqlDialect.DatabaseProduct aDatabaseProduct =
SqlDialect.getProduct(aConnection.getMetaData().getDatabaseProductName(), "UNUSED");
String schemaName = getSchemaName(aDatabaseProduct, aConnection);
String catalog = aConnection.getCatalog();
FrameworkConfig aConfig = createConfig(dataSource, aDatabaseProduct, catalog, schemaName);
Planner aPlanner = Frameworks.getPlanner(aConfig);
SqlNode aQuery = aPlanner.parse(sql);
aQuery = aPlanner.validate(aQuery);
RelNode aRelNode = aPlanner.rel(aQuery).project();
RelToSqlConverter aSqlConverter = new RelToSqlConverter(aSqlDialect);
SqlNode aSqlNode = aSqlConverter.visitChild(0, aRelNode).asStatement();
System.out.println(sql);
System.out.println(RelOptUtil.toString(aRelNode));
System.out.println(aSqlNode);
return sql;
}catch(Exception e){
e.getStackTrace();
}
return null;
}
/**
* @Title: close
* @Description: TODO(方法简要描述,必须以句号为结束)
* @since: (开始使用的版本)
* @param rs
* @param pst
* @param conn
*/
public static void close(ResultSet rs, PreparedStatement pst, Connection conn) {
if (rs != null) {
try {
rs.close();
} catch (SQLException e) {
error_log.error(e.getStackTrace());
}
rs = null;
}
if (pst != null) {
try {
pst.close();
} catch (SQLException e) {
error_log.error(e.getStackTrace());
}
pst = null;
}
if (conn != null) {
try {
conn.close();
} catch (SQLException e) {
error_log.error(e.getStackTrace());
}
conn = null;
}
}
}
BaseDao : 定义接口
public interface BaseDao {
/**
* 查询
* @Title: query
* @Description: TODO(方法简要描述,必须以句号为结束)
* @since: (开始使用的版本)
* @param sql
* @return
*/
public List
}
BaseDaoImpl : implements BaseDao
public class BaseDaoImpl extends FileLogger implements BaseDao {
private JDBCConnection jdbcConn = JDBCConnection.getInstance();
public static final Pattern sLimitPattern =
Pattern.compile("\\s*\\d+\\s*(,\\s*\\d+\\s*)?");
@Override
public List
return queryBySql(sql, null);
}
@Override
public List
return queryBySql(sql, (ArrayList
}
@Override
public List
Connection conn = jdbcConn.getConnection();
PreparedStatement pst = null;
ResultSet rs = null;
try {
String newSQL = JDBCConnection.convertSqlByType(sql);
info_log.info(newSQL);
pst = conn.prepareStatement(newSQL);
int parameterIndex = 1;
if (null != params && params.size() > 0) {
for (Object object : params) {
pst.setObject(parameterIndex++, object);
}
}
rs = pst.executeQuery();
ResultSetMetaData rst = rs.getMetaData();
int column = rst.getColumnCount();
List
while (rs.next()) {
Map
for (int i = 1; i <= column; i++) {
m.put(rst.getColumnName(i), rs.getObject(i));
}
rstList.add(m);
}
return rstList;
} catch (SQLException | SqlParseException | ValidationException | RelConversionException e) {
error_log.error(e.getMessage());
e.printStackTrace();
return null;
} finally {
JDBCConnection.close(rs, pst, conn);
}
}
@Override
public String getDBSQL(String sql) {
ConfigBuilder builder = SqlParser.configBuilder();
builder.setQuotedCasing(Casing.TO_LOWER);
builder.setLex(Lex.MYSQL);
Config config = builder.build();
SqlParser sqlParser = SqlParser.create(sql, config);
SqlString sqlString = null;
try {
SqlNode sqlNode = sqlParser.parseStmt();
sqlString = sqlNode.toSqlString(CalciteSqlDialect.DEFAULT);
return sqlString.getSql();
} catch (SqlParseException e) {
error_log.error(e.getStackTrace());
e.printStackTrace();
}
return null;
}
@Override
public List
String whereClause = "";
List
if (whereMap != null && whereMap.size() > 0) {
Iterator
int i = 0;
while (iterator.hasNext()) {
String key = iterator.next();
whereClause += (i == 0 ? "" : " AND ");
whereClause += (key + " = ? ");
whereArgs.add(whereMap.get(key));
i++;
}
}
return queryByTable(tableName, false, null, whereClause, whereArgs, null, null, null, null);
}
@Override
public List
throws SQLException {
return queryByTable(tableName, false, null, whereClause, whereArgs, null, null, null, null);
}
@Override
public List
String selection, List
throws SQLException {
String sql = buildQueryString(distinct, tableName, columns, selection, groupBy, having, orderBy, limit);
return executeQuery(sql, selectionArgs);
}
@Override
public int insertBySql(String sql) throws SQLException {
return executeUpdate(sql,null);
}
@Override
public int insertBySql(String sql,List
return executeUpdate(sql,bindArgs);
}
@Override
public int insertByTable(String tableName, Map
/** 获取数据库插入的Map的键值对的值 **/
Set
Iterator
/** 要插入的字段sql,其实就是用key拼起来的 **/
StringBuilder columnSql = new StringBuilder();
/** 要插入的字段值,其实就是? **/
StringBuilder unknownMarkSql = new StringBuilder();
List
int i = 0;
while (iterator.hasNext()) {
String key = iterator.next();
columnSql.append(i == 0 ? "" : ",");
columnSql.append(key);
unknownMarkSql.append(i == 0 ? "" : ",");
unknownMarkSql.append("?");
bindArgs.add(valueMap.get(key));
i++;
}
/** 开始拼插入的sql语句 **/
StringBuilder sql = new StringBuilder();
sql.append("INSERT INTO ");
sql.append(tableName);
sql.append(" (");
sql.append(columnSql);
sql.append(" ) VALUES (");
sql.append(unknownMarkSql);
sql.append(" )");
return executeUpdate(sql.toString(), bindArgs);
}
@Override
public int insertBatchByTable(String tableName, List
/** 影响的行数 **/
int affectRowCount = -1;
Connection connection = jdbcConn.getConnection();
PreparedStatement preparedStatement = null;
try {
/** 从数据库连接池中获取数据库连接 **/
//connection = DBConnectionPool.getInstance().getConnection();
Map
/** 获取数据库插入的Map的键值对的值 **/
Set
Iterator
/** 要插入的字段sql,其实就是用key拼起来的 **/
StringBuilder columnSql = new StringBuilder();
/** 要插入的字段值,其实就是? **/
StringBuilder unknownMarkSql = new StringBuilder();
Object[] keys = new Object[valueMap.size()];
int i = 0;
while (iterator.hasNext()) {
String key = iterator.next();
keys[i] = key;
columnSql.append(i == 0 ? "" : ",");
columnSql.append(key);
unknownMarkSql.append(i == 0 ? "" : ",");
unknownMarkSql.append("?");
i++;
}
/** 开始拼插入的sql语句 **/
StringBuilder sql = new StringBuilder();
sql.append("INSERT INTO ");
sql.append(tableName);
sql.append(" (");
sql.append(columnSql);
sql.append(" ) VALUES (");
sql.append(unknownMarkSql);
sql.append(" )");
String newSQL = JDBCConnection.convertSqlByType(sql.toString());
preparedStatement = connection.prepareStatement(newSQL);
/** 设置不自动提交,以便于在出现异常的时候数据库回滚 **/
connection.setAutoCommit(false);
System.out.println(sql.toString());
for (int j = 0; j < datas.size(); j++) {
for (int k = 0; k < keys.length; k++) {
preparedStatement.setObject(k + 1, datas.get(j).get(keys[k]));
}
preparedStatement.addBatch();
}
int[] arr = preparedStatement.executeBatch();
connection.commit();
affectRowCount = arr.length;
System.out.println("成功了插入了" + affectRowCount + "行");
System.out.println();
} catch (Exception e) {
if (connection != null) {
connection.rollback();
}
e.printStackTrace();
} finally {
if (preparedStatement != null) {
preparedStatement.close();
}
if (connection != null) {
connection.close();
}
}
return affectRowCount;
}
@Override
public int updateBySql(String sql) throws SQLException{
return executeUpdate(sql, null);
}
@Override
public int updateBySql(String sql,List
return executeUpdate(sql, params);
}
@Override
public int updateByTable(String tableName, Map
throws SQLException {
/** 获取数据库插入的Map的键值对的值 **/
Set
Iterator
/** 开始拼插入的sql语句 **/
StringBuilder sql = new StringBuilder();
sql.append("UPDATE ");
sql.append(tableName);
sql.append(" SET ");
/** 要更改的的字段sql,其实就是用key拼起来的 **/
StringBuilder columnSql = new StringBuilder();
int i = 0;
List
while (iterator.hasNext()) {
String key = iterator.next();
columnSql.append(i == 0 ? "" : ",");
columnSql.append(key + " = ? ");
params.add(valueMap.get(key));
i++;
}
sql.append(columnSql);
/** 更新的条件:要更改的的字段sql,其实就是用key拼起来的 **/
StringBuilder whereSql = new StringBuilder();
int j = 0;
if (whereMap != null && whereMap.size() > 0) {
whereSql.append(" WHERE ");
iterator = whereMap.keySet().iterator();
while (iterator.hasNext()) {
String key = iterator.next();
whereSql.append(j == 0 ? "" : " AND ");
whereSql.append(key + " = ? ");
params.add(whereMap.get(key));
j++;
}
sql.append(whereSql);
}
return executeUpdate(sql.toString(), params);
}
@Override
public int deleteBySql(String sql) throws SQLException{
return executeUpdate(sql, null);
}
@Override
public int deleteBySql(String sql,Object params) throws SQLException{
return executeUpdate(sql, (ArrayList)params);
}
@Override
public int deleteBySql(String sql,List
return executeUpdate(sql, params);
}
@Override
public int deleteByTable(String tableName, Map
/** 准备删除的sql语句 **/
StringBuilder sql = new StringBuilder();
sql.append("DELETE FROM ");
sql.append(tableName);
/** 更新的条件:要更改的的字段sql,其实就是用key拼起来的 **/
StringBuilder whereSql = new StringBuilder();
List
if (whereMap != null && whereMap.size() > 0) {
whereSql.append(" WHERE ");
/** 获取数据库插入的Map的键值对的值 **/
Set
Iterator
int i = 0;
while (iterator.hasNext()) {
String key = iterator.next();
whereSql.append(i == 0 ? "" : " AND ");
whereSql.append(key + " = ? ");
bindArgs.add(whereMap.get(key));
i++;
}
sql.append(whereSql);
}
return executeUpdate(sql.toString(), bindArgs);
}
public Boolean execute(String sql) {
Connection conn = jdbcConn.getConnection();
PreparedStatement pst = null;
ResultSet rs = null;
try {
String newSQL = JDBCConnection.convertSqlByType(sql);
info_log.info(newSQL);
pst = conn.prepareStatement(newSQL);
return pst.executeUpdate() >= 1 ? true : false;
} catch (SQLException | SqlParseException | ValidationException | RelConversionException e) {
error_log.error(e.getStackTrace());
e.printStackTrace();
return false;
} finally {
JDBCConnection.close(rs, pst, conn);
}
}
private static boolean isEmpty(String s) {
if (null != s && s.trim().length() > 0) {
return false;
}
return true;
}
private static String buildQueryString(boolean distinct, String tables, String[] columns, String where,
String groupBy, String having, String orderBy, String limit) {
if (isEmpty(groupBy) && !isEmpty(having)) {
throw new IllegalArgumentException("HAVING clauses are only permitted when using a groupBy clause");
}
if (!isEmpty(limit) && !sLimitPattern.matcher(limit).matches()) {
throw new IllegalArgumentException("invalid LIMIT clauses:" + limit);
}
StringBuilder query = new StringBuilder(120);
query.append("SELECT ");
if (distinct) {
query.append("DISTINCT ");
}
if (columns != null && columns.length != 0) {
appendColumns(query, columns);
} else {
query.append(" * ");
}
query.append("FROM ");
query.append(tables);
appendClause(query, " WHERE ", where);
appendClause(query, " GROUP BY ", groupBy);
appendClause(query, " HAVING ", having);
appendClause(query, " ORDER BY ", orderBy);
appendClause(query, " LIMIT ", limit);
return query.toString();
}
private List
List
Connection connection = jdbcConn.getConnection();
PreparedStatement preparedStatement = null;
ResultSet resultSet = null;
try {
/** 获取数据库连接池中的连接 **/
// connection = DBConnectionPool.getInstance().getConnection();
String newSQL = JDBCConnection.convertSqlByType(sql);
preparedStatement = connection.prepareStatement(newSQL);
if (bindArgs != null && bindArgs.size()>0) {
int i = 0;
for(Object object:bindArgs){
preparedStatement.setObject(i + 1, object);
}
}
System.out.println(getExecSQL(sql, bindArgs));
resultSet = preparedStatement.executeQuery();
datas = getDatas(resultSet);
} catch (Exception e) {
e.printStackTrace();
} finally {
if (resultSet != null) {
resultSet.close();
}
if (preparedStatement != null) {
preparedStatement.close();
}
if (connection != null) {
connection.close();
}
}
return datas;
}
private List
List
/** 获取结果集的数据结构对象 **/
ResultSetMetaData metaData = resultSet.getMetaData();
while (resultSet.next()) {
Map
for (int i = 1; i <= metaData.getColumnCount(); i++) {
rowMap.put(metaData.getColumnName(i), resultSet.getObject(i));
}
datas.add(rowMap);
}
System.out.println("成功查询到了" + datas.size() + "行数据");
for (int i = 0; i < datas.size(); i++) {
Map
System.out.println("第" + (i + 1) + "行:" + map);
}
return datas;
}
/**
* 所有的更新处理
* @Title: executeUpdate
* @Description: TODO(方法简要描述,必须以句号为结束)
* @since: (开始使用的版本)
* @param sql
* @param bindArgs
* @return
* @throws SQLException
*/
public int executeUpdate(String sql, List
int affectRowCount = -1;
Connection connection = jdbcConn.getConnection();
PreparedStatement preparedStatement = null;
try {
/** 从数据库连接池中获取数据库连接 **/
// connection = DBConnectionPool.getInstance().getConnection();
String newSQL = JDBCConnection.convertSqlByType(sql);
preparedStatement = connection.prepareStatement(newSQL);
/** 设置不自动提交,以便于在出现异常的时候数据库回滚 **/
connection.setAutoCommit(false);
System.out.println(getExecSQL(sql, bindArgs));
if (bindArgs != null) {
/** 绑定参数设置sql占位符中的值 **/
int i = 0;
for(Object object : bindArgs){
preparedStatement.setObject(i + 1, object);
}
}
affectRowCount = preparedStatement.executeUpdate();
connection.commit();
String operate;
if (sql.toUpperCase().indexOf("DELETE FROM") != -1) {
operate = "删除";
} else if (sql.toUpperCase().indexOf("INSERT INTO") != -1) {
operate = "新增";
} else {
operate = "修改";
}
System.out.println("成功" + operate + "了" + affectRowCount + "行");
} catch (Exception e) {
if (connection != null) {
connection.rollback();
}
e.printStackTrace();
} finally {
if (preparedStatement != null) {
preparedStatement.close();
}
if (connection != null) {
connection.close();
}
}
return affectRowCount;
}
/**
* 转化SQL
* @Title: getExecSQL
* @Description: TODO(方法简要描述,必须以句号为结束)
* @since: (开始使用的版本)
* @param sql
* @param bindArgs
* @return
*/
private static String getExecSQL(String sql, List
StringBuilder sb = new StringBuilder(sql);
if (bindArgs != null && bindArgs.size() > 0) {
int index = 0;
for(Object object:bindArgs){
index = sb.indexOf("?", index);
sb.replace(index, index + 1, String.valueOf(object));
}
}
return sb.toString();
}
/**
* 拼接SQL
* @Title: appendClause
* @Description: TODO(方法简要描述,必须以句号为结束)
* @since: (开始使用的版本)
* @param s
* @param name
* @param clause
*/
private static void appendClause(StringBuilder s, String name, String clause) {
if (!isEmpty(clause)) {
s.append(name);
s.append(clause);
}
}
/**
* 添加Cloumn查询项
* @Title: appendColumns
* @Description: TODO(方法简要描述,必须以句号为结束)
* @since: (开始使用的版本)
* @param s
* @param columns
*/
private static void appendColumns(StringBuilder s, String[] columns) {
int n = columns.length;
for (int i = 0; i < n; i++) {
String column = columns[i];
if (column != null) {
if (i > 0) {
s.append(", ");
}
s.append(column);
}
}
s.append(' ');
}
}
到此为止,主要工具类已经完事,开始测试:
public class MysqlTest {
public static Connection conn = JDBCConnection.getInstance().getConnection();
public static void query() {
BaseDao dao = new BaseDaoImpl();
String sql = "select * from test where id=?";
List
//params.add(101);
List
for (Map
System.out.println(map.toString());
}
}
public static void insert() {
BaseDao dao = new BaseDaoImpl();
String sql = "insert into spark_caa.test (id,name) values (101,'caozqa'),(100,'weilpa')";
boolean list = dao.execute(sql);
System.out.println(list);
}
public static void updata() {
BaseDao dao = new BaseDaoImpl();
String sql = "update spark_caa.test set name='888' where id=2";
boolean list = dao.execute(sql);
System.out.println(list);
}
public static void delete() {
BaseDao dao = new BaseDaoImpl();
String sql = "delete from spark_caa.test where id=1";
boolean list = dao.execute(sql);
System.out.println(list);
}
public static void main(String[] args) throws SqlParseException {
query();
insert()