MongoDB 数据库版本:3.0.6
jar 包:mongo-java-driver-3.1.0.jar (见附件)。
数据源配置:
mongodb.soTimeOut=30000 mongodb.connectionsPerHost=500 mongodb.threadsBlock=30 mongodb.host=10.0.0.11 #mongodb.host=localhost mongodb.port=27017 mongodb.database=dkpt mongodb.collectionUrl=dkcollect mongodb.batchSize=3000 mongodb.pool=100
MongodbUtils.java
import com.mongodb.BasicDBObject; import com.mongodb.MongoClient; import com.mongodb.MongoClientOptions; import com.mongodb.MongoWriteException; import com.mongodb.ServerAddress; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; import com.mongodb.client.result.DeleteResult; import com.mongodb.client.result.UpdateResult; import com.techstar.plat.alarm.entity.DeviceWarn; import com.techstar.plat.model.RealTimeData; import org.bson.Document; import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; /** * * @ClassName: MongodbUtils.java * @description * * <pre> * Mongodb 工具类 * 封装基础的增删改查业务 * </pre> * @author * @company * @date 2015-8-20 * @version V1.0 */ public class MongodbUtils { private static Map<String, String> propMap = CommonsTkywConstants.propMap; private static final Integer soTimeOut = Integer.parseInt(propMap .get("mongodb.soTimeOut")); private static final Integer connectionsPerHost = Integer.parseInt(propMap .get("mongodb.connectionsPerHost")); private static final Integer threadsBlock = Integer.parseInt(propMap .get("mongodb.threadsBlock")); private String host = propMap.get("mongodb.host"); private int port = Integer.parseInt(propMap.get("mongodb.port")); private ExecutorService executor = Executors.newFixedThreadPool(100); private String database = propMap.get("mongodb.database"); private String collectionUrl = propMap.get("mongodb.collectionUrl"); /** * 通过 ip,端口号,数据库,连接初始化mongodb实例 * * @param host * @param port * @param database * @param collectionUrl */ public MongodbUtils(String host, int port, String database, String collectionUrl) { this.host = host; this.port = port; this.database = database; this.collectionUrl = collectionUrl; } /** * 默认构造函数,使用配置文件中的参数进行初始化mongodb 实例 */ public MongodbUtils() { } /** * 通过配置文件和传入的collectionName初始化mongodb实例 * * @param collectionName */ public MongodbUtils(String collectionName) { this.collectionUrl = collectionName; } /** * 获取mongodb连接实例 * * @return */ private MongoClient getMongoClient() { MongoClient mongoClient = new MongoClient( new ServerAddress(host, port), new MongoClientOptions.Builder() .socketTimeout(soTimeOut) .connectionsPerHost(connectionsPerHost) .threadsAllowedToBlockForConnectionMultiplier( threadsBlock).socketKeepAlive(true).build()); return mongoClient; } /** * * @return */ private MongoClient getSingleClient() { if (mongoClient == null) { mongoClient = new MongoClient(new ServerAddress(host, port), new MongoClientOptions.Builder() .socketTimeout(soTimeOut) .connectionsPerHost(connectionsPerHost) .threadsAllowedToBlockForConnectionMultiplier( threadsBlock).socketKeepAlive(true).build()); } return mongoClient; } private static final int batchSize = Integer.parseInt(propMap .get("mongodb.batchSize")) * 3; // private static final int batchDeleteSize = batchSize * 10; /** * 只存入key-value的简单数据,如果key相同不删除 * * @param map */ public void addMapString(Map<String, String> map) { if (map == null || map.size() == 0) { return; } List<Document> docs = new ArrayList<Document>(); Document doc = new Document(); for (String key : map.keySet()) { doc.put(key, map.get(key)); } docs.add(doc); executor.execute(new MongodbTask(getMongoClient(), database, collectionUrl, docs)); } /** * 关闭mongodb 连接 */ public void close() { if (mongoClient != null) { mongoClient.close(); mongoClient = null; } } /** * 只存入key-value的简单数据,如果key相同不删除 * * @param map */ public void addMapObject(Map<String, Object> map) { if (map == null || map.size() == 0) { return; } Document doc = new Document(); doc.putAll(map); mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); dbCollection.insertOne(doc); } /** * <pre> * 存入一条复杂键值都是String的数据,如果key相同删除以前的数据,并重新插入新数据 * * 其中Map中一个键固定为“key” 含义是cimId或cimid+量测类型 * </pre> * * @param listMap */ public void addOrUpdateString(Map<String, String> argMap) { if (argMap == null || argMap.keySet() == null || argMap.keySet().size() == 0) { return; } mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); Document doc = new Document(); Object keyV = argMap.get("key"); if (keyV != null) { doc.put("key", keyV); try { dbCollection.deleteMany(doc); } catch (MongoWriteException e) { } } doc.putAll(argMap); dbCollection.insertOne(doc); } /** * <pre> * 存入一条复杂键值都是String的数据,如果key相同删除以前的数据,并重新插入新数据 * * 其中Map中一个键固定为“key” 含义是cimId或cimid+量测类型 * </pre> * * @param listMap * @param deleteKey * 需要根据哪个key删除以前的数据 */ public void addOrUpdateString(Map<String, String> argMap, String deleteKey) { if (argMap == null || argMap.keySet() == null || argMap.keySet().size() == 0) { return; } mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); Document doc = new Document(); Object keyV = argMap.get(deleteKey); if (keyV != null) { doc.put(deleteKey, keyV); try { dbCollection.deleteMany(doc); } catch (MongoWriteException e) { } } doc.putAll(argMap); dbCollection.insertOne(doc); } /** * <pre> * 存入复杂对象,如果key相同删除以前的数据,并重新插入新数据 * * 其中Map中一个键固定为“key” 含义是cimId或cimid+量测类型 * </pre> * * @param listMap * @param deleteKey * 需要根据哪个key删除以前的数据 */ public void batchAddOrUpdateString(List<Map<String, String>> listMap, String deleteKey) { List<Document> docs = new ArrayList<Document>(); int size = listMap.size(); int index = 0; mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); List<String> deleteList = new LinkedList<String>(); for (Map<String, String> map : listMap) { Document doc = new Document(); doc.putAll(map); index++; docs.add(doc); deleteList.add(map.get(deleteKey)); if ((index != 0 && index % batchSize == 0) || index == size) { executor.execute(new MongodbTask(getMongoClient(), database, collectionUrl, docs, deleteList)); docs = new ArrayList<Document>(); deleteList = new LinkedList<String>(); } } } /** * <pre> * 存入复杂对象,如果key相同删除以前的数据,并重新插入新数据 * * 其中Map中一个键固定为“key” 含义是cimId或cimid+量测类型 * </pre> * * @param listMap */ public void batchAddOrUpdateString(List<Map<String, String>> listMap) { // long start = System.currentTimeMillis(); // sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS"); // System.out.println("sssssssss:" + sdf.format(new Date())); List<Document> docs = new LinkedList<Document>(); int size = listMap.size(); int index = 0; List<String> deleteList = new LinkedList<String>(); index = 0; for (Map<String, String> map : listMap) { Document doc = new Document(); doc.putAll(map); index++; docs.add(doc); deleteList.add(map.get("key")); if ((index != 0 && index % batchSize == 0) || index == size) { executor.execute(new MongodbTask(getMongoClient(), database, collectionUrl, docs, deleteList)); docs = new ArrayList<Document>(); deleteList = new LinkedList<String>(); } } // mongoClient.close(); } /** * <pre> * 存入复杂对象,如果key相同删除以前的数据,并重新插入新数据, * * 其中Map中一个键固定为“key” 含义是cimId或cimid+量测类型 * </pre> * * @param listMap */ public void batchAddOrUpdateObject(List<Map<String, Object>> listMap) { List<Document> docs = new ArrayList<Document>(); int size = listMap.size(); int index = 0; // mongoClient = getSingleClient(); // mongodb = mongoClient.getDatabase(database); // dbCollection = mongodb.getCollection(collectionUrl); // Document deleteDoc = new Document(); List<String> deleteList = new LinkedList<String>(); // for (Map<String, Object> map : listMap) { // deleteList.add((String) map.get("key")); // index++; // if ((index != 0 && index % batchDeleteSize == 0) || index == size) { // deleteDoc = new Document(); // deleteDoc.put("key", new Document("$in", deleteList)); // dbCollection.deleteMany(deleteDoc); // deleteList = new LinkedList<String>(); // } // } for (Map<String, Object> map : listMap) { Document doc = new Document(); doc.putAll(map); index++; docs.add(doc); deleteList.add((String) map.get("key")); if ((index != 0 && index % batchSize == 0) || index == size) { executor.execute(new MongodbTask(getMongoClient(), database, collectionUrl, docs, deleteList, "key")); docs = new ArrayList<Document>(); deleteList = new LinkedList<String>(); } } // mongoClient.close(); } /** * <pre> * 存入复杂对象,如果key相同删除以前的数据,并重新插入新数据, * * 其中Map中一个键固定为“key” 含义是cimId或cimid+量测类型 * </pre> * * @param listMap * @param deleteKey * 需要根据哪个key删除以前的数据 */ public void addOrUpdateObject(Map<String, Object> argMap, String deleteKey) { List<Document> docs = new ArrayList<Document>(); if (argMap == null || argMap.keySet() == null || argMap.keySet().size() == 0) { return; } mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); Document doc = new Document(); Object keyV = argMap.get(deleteKey); if (keyV != null) { doc.put(deleteKey, keyV); try { dbCollection.deleteMany(doc); } catch (MongoWriteException e) { } } doc.putAll(argMap); executor.execute(new MongodbTask(getMongoClient(), database, collectionUrl, docs)); } /** * <pre> * 存入复杂对象,如果key相同删除以前的数据,并重新插入新数据, * * 其中Map中一个键固定为“key” 含义是cimId或cimid+量测类型 * </pre> * * @param listMap * @param deleteKey * 需要根据哪个key删除以前的数据 */ public void batchAddOrUpdateObject(List<Map<String, Object>> listMap, String deleteKey) { List<Document> docs = new ArrayList<Document>(); int size = listMap.size(); int index = 0; // MongoClient mongoClient = getSingleClient(); // MongoDatabase mongodb = mongoClient.getDatabase(database); // MongoCollection<Document> dbCollection = // mongodb.getCollection(collectionUrl); // Document deleteDoc = new Document(); List<String> deleteList = new LinkedList<String>(); // for (Map<String, Object> map : listMap) { // deleteList.add((String) map.get(deleteKey)); // index++; // if ((index != 0 && index % batchDeleteSize == 0) || index == size) { // deleteDoc = new Document(); // deleteDoc.put("key", new Document("$in", deleteList)); // dbCollection.deleteMany(deleteDoc); // deleteList = new LinkedList<String>(); // } // } for (Map<String, Object> map : listMap) { Document doc = new Document(); doc.putAll(map); index++; docs.add(doc); deleteList.add((String) map.get(deleteKey)); if ((index != 0 && index % batchSize == 0) || index == size) { executor.execute(new MongodbTask(getMongoClient(), database, collectionUrl, docs, deleteList, deleteKey)); docs = new ArrayList<Document>(); deleteList = new LinkedList<String>(); } } } /** * <pre> * 存入复杂对象,如果key相同删除以前的数据,并重新插入新数据, * * 其中Map中一个键固定为“key” 含义是cimId或cimid+量测类型 * </pre> * * @param listMap * @param deleteKey * 需要根据哪个key删除以前的数据 */ public void addOrUpdateObject(Map<String, Object> argMap) { List<Document> docs = new ArrayList<Document>(); if (argMap == null || argMap.keySet() == null || argMap.keySet().size() == 0) { return; } mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); Document doc = new Document(); Object keyV = argMap.get("key"); if (keyV != null) { doc.put("key", keyV); try { dbCollection.deleteMany(doc); } catch (MongoWriteException e) { } } for (String key : argMap.keySet()) { doc.put(key, argMap.get(key)); } docs.add(doc); executor.execute(new MongodbTask(getMongoClient(), database, collectionUrl, docs)); } /** * 批量存入复杂对象,如果key相同不删除 * * @param listMap */ public void batchAddMapString(List<Map<String, String>> listMap) { List<Document> docs = new ArrayList<Document>(); int size = listMap.size(); int index = 0; for (Map<String, String> map : listMap) { Document doc = new Document(); for (String key : map.keySet()) { doc.put(key, map.get(key)); } index++; docs.add(doc); if ((index != 0 && index % batchSize == 0) || index == size) { executor.execute(new MongodbTask(getMongoClient(), database, collectionUrl, docs)); docs = new ArrayList<Document>(); } } } /** * 批量存入复杂对象,如果key相同不删除 * * @param listMap */ public void batchAddObject(List<Map<String, Object>> listMap) { List<Document> docs = new ArrayList<Document>(); int size = listMap.size(); int index = 0; for (Map<String, Object> map : listMap) { Document doc = new Document(); for (String key : map.keySet()) { doc.put(key, map.get(key)); } index++; docs.add(doc); if ((index != 0 && index % batchSize == 0) || index == size) { executor.execute(new MongodbTask(getMongoClient(), database, collectionUrl, docs)); docs = new ArrayList<Document>(); } } } /** * 根据key获取值 供实时数据使用 * * 如果key是cimeId+量测类型:则返回值该量测类型的值 * * 量测类型含义: * * <pre> * P: 有功 * Q :无功 * U :电压 * I: 电流 * S:开关、刀闸开合状态 * </pre> * * @param key * @return */ public String findByCimeIdType(String key) { Document doc = new Document(); doc.append("key", key); mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); FindIterable<Document> finds = dbCollection.find(doc); MongoCursor<Document> cursor = finds.iterator(); try { if (cursor.hasNext()) { Document findDoc = cursor.next(); String value = (String) findDoc.get("value"); return value; } } finally { if (cursor != null) { cursor.close(); cursor = null; } } return null; } /** * <pre> * 根据key获取值,如果有值则获取长度为2的数组, * 其中array[0]是具体值,array[1]返回是否有效,0-无效,1-有效 * 如果查询不到值则返回为空 * * 如果key是cimeId+量测类型:则返回值该量测类型的值 * * 量测类型含义: * * * P: 有功 * Q :无功 * U :电压 * I: 电流 * S:开关、刀闸开合状态 * </pre> * * @param key * @return */ public String[] findValueValidByCimeIdType(String key) { Document doc = new Document(); doc.append("key", key); mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); FindIterable<Document> finds = dbCollection.find(doc); MongoCursor<Document> cursor = finds.iterator(); try { if (cursor.hasNext()) { Document findDoc = cursor.next(); String[] values = new String[2]; String value = (String) findDoc.get("value"); values[0] = value; Object validO = (Object) findDoc.get("valid"); if (validO != null) { String valid = (String) validO; values[1] = valid; } return values; } } finally { if (cursor != null) { cursor.close(); cursor = null; } } return null; } public MongoCollection<Document> getCollection() { mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); return dbCollection; } /** * <pre> * 根据key获取值,如果无效返回0000 * * 如果key是cimeId+量测类型:则返回值该量测类型的值 * * 量测类型含义: * * P: 有功 * Q :无功 * U :电压 * I: 电流 * S:开关、刀闸开合状态 * </pre> * * @param key * @return */ public String findValueByCimeIdType(String key) { Document doc = new Document(); doc.append("key", key); mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); FindIterable<Document> finds = dbCollection.find(doc); MongoCursor<Document> cursor = finds.iterator(); try { if (cursor.hasNext()) { Document findDoc = cursor.next(); String value = (String) findDoc.get("value"); Object validO = (Object) findDoc.get("valid"); if (validO != null) { String valid = (String) validO; if (valid.equals("0")) { return "0000"; } } return value; } } finally { if (cursor != null) { cursor.close(); cursor = null; } } return null; } /** * 删除集合数据 */ public void drop() { mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); dbCollection.drop(); } private static SimpleDateFormat sdf = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); /** * 根据key活动告警对象 * * @param key * @return */ public DeviceWarn getWarn(String key) { Document doc = new Document(); doc.append("key", key); mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); FindIterable<Document> finds = dbCollection.find(doc); MongoCursor<Document> cursor = finds.iterator(); try { if (cursor.hasNext()) { Document findDoc = cursor.next(); String value = String.valueOf(findDoc.get("value")); DeviceWarn warn = new DeviceWarn(); String methodid = (String) findDoc.get("methodid"); warn.setModelId(methodid); String name = (String) findDoc.get("name"); warn.setName(name); warn.setCimId(key); Object timeO = findDoc.get("time"); if (timeO != null) { String time = (String) timeO; warn.setValue(Double.parseDouble(value)); try { warn.setAlarmDate(new Timestamp(sdf.parse(time) .getTime())); } catch (ParseException e) { } } return warn; } } finally { if (cursor != null) { cursor.close(); cursor = null; } } return null; } /** * 根据cime+量测类型查询当前值及其对应时间, * * @param key * @return */ public RealTimeData findByCimeIdType4Alarm(String key) { Document doc = new Document(); doc.append("key", key); mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); FindIterable<Document> finds = dbCollection.find(doc); MongoCursor<Document> cursor = finds.iterator(); try { if (cursor.hasNext()) { Document findDoc = cursor.next(); String value = String.valueOf(findDoc.get("value")); RealTimeData realTimeData = new RealTimeData(); Object timeO = findDoc.get("time"); if (timeO != null) { String time = (String) timeO; realTimeData.setValue(Double.parseDouble(value)); SimpleDateFormat sdf = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); try { realTimeData.setTime(new Timestamp(sdf.parse(time) .getTime())); } catch (ParseException e) { } } return realTimeData; } } finally { if (cursor != null) { cursor.close(); cursor = null; } } return null; } // /** // * 根据cimeID获取设备对象相关信息 // * // * 数据长度是2:obj[0]是类型,obj[1]是一体化模型对象 // * // * 如:obj[0]是AcLineDot,obj[1]一定时AcLineDotPO对象,对用用可以对obj[1]强制转换成AcLineDotPO // * // * @param key // * cime id // * @return // */ // public Object[] findByCimeId(String key) { // Document doc = new Document(); // doc.append("key", key); // FindIterable<Document> finds = dbCollection.find(doc); // MongoCursor<Document> cursor = finds.iterator(); // if (cursor.hasNext()) { // Document findDoc = cursor.next(); // String type = (String) findDoc.get("type"); // Object obj = findDoc.get("obj"); // Object[] objs = new Object[2]; // objs[0] = type; // objs[1] = obj; // return objs; // } // return null; // } /** * 根据条件查找数据 * * @param findArgs * @return */ public List<Document> find(Map<String, Object> findArgs) { Document doc = new Document(); for (String key : findArgs.keySet()) { doc.append(key, findArgs.get(key)); } mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); FindIterable<Document> finds = dbCollection.find(doc); List<Document> docs = new ArrayList<Document>(); MongoCursor<Document> cursor = finds.iterator(); try { while (cursor.hasNext()) { Document findDoc = cursor.next(); docs.add(findDoc); } } finally { if (cursor != null) { cursor.close(); cursor = null; } } return docs.size() > 0 ? docs : null; } /** * <pre> * 根据key在历史表比较最近的2个版本数据值是否相同, * 如果相同返回null, * 如果不相同返回key * </pre> * * @param key * @return */ public String compareWithLastValue(String key) { Document doc = new Document(); doc.append("key", key); mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); FindIterable<Document> finds = dbCollection.find(doc).limit(2) .sort(new BasicDBObject("time", -1)); MongoCursor<Document> cursor = finds.iterator(); String currentValue = null;// 获取当前值 String lastTimeValue = null;// 获取上次值 try { if (cursor.hasNext()) { Document findDoc = cursor.next(); currentValue = (String) findDoc.get("value"); } if (cursor.hasNext()) { Document findDoc = cursor.next(); lastTimeValue = (String) findDoc.get("value"); } if (currentValue != null && !currentValue.trim().equals("")) { if (lastTimeValue != null && !lastTimeValue.trim().equals("")) { if (currentValue.equals(lastTimeValue)) { return null; } else { return key; } } } } finally { if (cursor != null) { cursor.close(); cursor = null; } } return null; } /** * 根据key获取最新一条数据 * * @param key * @return */ public String findLastValue(String key) { Document doc = new Document(); doc.append("key", key); mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); FindIterable<Document> finds = dbCollection.find(doc); MongoCursor<Document> cursor = finds.iterator(); try { if (cursor.hasNext()) { Document findDoc = cursor.next(); String value = (String) findDoc.get("value"); return value; } } finally { if (cursor != null) { cursor.close(); cursor = null; } } return null; } /** * 更新对象 * * @param filter * @param update * @return */ public long update(Map<String, Object> filter, Map<String, Object> update) { Document doc = new Document(); for (String key : filter.keySet()) { doc.append(key, filter.get(key)); } BasicDBObject updateDocument = new BasicDBObject(); BasicDBObject newDocument = new BasicDBObject(); for (String key : update.keySet()) { newDocument.append(key, update.get(key)); } updateDocument.append("$set", newDocument); mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); UpdateResult result = dbCollection.updateMany(doc, updateDocument); long size = result.getMatchedCount(); return size; } private MongoClient mongoClient = null; private MongoDatabase mongodb = null; private MongoCollection<Document> dbCollection = null; /** * 删除对象 * * @param deleteArgs * @return */ public long delete(Map<String, Object> deleteArgs) { Document doc = new Document(); for (String key : deleteArgs.keySet()) { doc.put(key, deleteArgs.get(key)); } DeleteResult result = null; try { mongoClient = getSingleClient(); mongodb = mongoClient.getDatabase(database); dbCollection = mongodb.getCollection(collectionUrl); result = dbCollection.deleteMany(doc); long size = result.getDeletedCount(); return size; } catch (MongoWriteException e) { } return 0; } }