记录清理功能,在数据量很小的情况下,用不上,如果数据量大了的话,长年累月存储的,那就显得极其重要了,好比视频监控中的NVR存储的视频一样,一般来说存储个60天,那超过60天怎办呢,擦除早期的数据用来存储最近的数据即可。在这个气体安全管理系统中,数据量长年累月也是很大的,一般来说一个节点默认每分钟存储一个数据,如果100个节点(这个应该是保守的数量,据说应用的好多个现场有500个左右的节点,至于如何突破的modbus255个节点的限制,后面的文章会单独讲解),一天下来就是1006024=144000,100天都1440万条数据了,千万级别,这个如果不做早期数据的清理的话,慢慢下去各种性能影响会越来越突出,要么加大存储间隔,要么设定只保存最近的多少条记录,这几个参数都是可以调节的,存储间隔在探测器设置中进行设置,每个探测器节点都可以设置不同的存储间隔;保留最大的记录数在系统设置中设置,默认100W条,超过的会定时清理早期数据。
为此特定封装了一个类DbCleanThread专门用来清理数据和文件夹,为什么还有个清理文件夹的功能呢,这也是现实中需求来的,比如有时候存储一些图片和视频文件等,如果存储器容量不够大,尤其是嵌入式linux,板子上一般存储器不会很大,目前8G居多,随着时间的推移,文件数量会越来越多,本身存储的剩余容量会越来越小,这也需要定期清理早期的文件,保证在现有容量允许的情况下,循环存储文件,编写这个类尽量做成了通用的类,考虑了很多个参数,比如可以设定条件字段、排序字段、清理的间隔、文件夹路径、最大大小等。
皮肤开源:https://gitee.com/feiyangqingyun/QWidgetDemo https://github.com/feiyangqingyun/QWidgetDemo
文件名称:styledemo
体验地址:https://gitee.com/feiyangqingyun/QWidgetExe https://github.com/feiyangqingyun/QWidgetExe
文件名称:bin_sams.zip
#pragma execution_character_set("utf-8")
#include "dbcleanthread.h"
QScopedPointer DbCleanThread::self;
DbCleanThread *DbCleanThread::Instance()
{
if (self.isNull()) {
static QMutex mutex;
QMutexLocker locker(&mutex);
if (self.isNull()) {
self.reset(new DbCleanThread);
}
}
return self.data();
}
DbCleanThread::DbCleanThread(QObject *parent) : QThread(parent)
{
stopped = false;
isBusy = false;
lastTime = QDateTime::currentDateTime();
tableName = "DeviceLog";
countName = "*";
whereColumnName = "SaveTime";
orderSql = "SaveTime asc";
maxCount = 100000;
interval = 30 * 60;
dirPath = "";
dirFileFilter = QStringList("*.jpg");
dirMaxSize = 1024;
dbType = DbType_MySql;
connName = "qt_sql_default_connection";
hostName = "127.0.0.1";
port = 3306;
dbName = "db_mysql";
userName = "root";
userPwd = "root";
}
DbCleanThread::~DbCleanThread()
{
this->stop();
this->wait();
}
void DbCleanThread::run()
{
//在这里打开数据库,Qt5.10以后增加了安全性,不能用主线程创建的数据库连接
QSqlDatabase dbConn;
if (dbType == DbType_Sqlite) {
dbConn = QSqlDatabase::addDatabase("QSQLITE", connName);
dbConn.setDatabaseName(dbName);
} else if (dbType == DbType_MySql) {
dbConn = QSqlDatabase::addDatabase("QMYSQL", connName);
dbConn.setConnectOptions("MYSQL_OPT_RECONNECT=1;MYSQL_OPT_CONNECT_TIMEOUT=1;MYSQL_OPT_READ_TIMEOUT=1;MYSQL_OPT_WRITE_TIMEOUT=1");
dbConn.setHostName(hostName);
dbConn.setPort(port);
dbConn.setDatabaseName(dbName);
dbConn.setUserName(userName);
dbConn.setPassword(userPwd);
}
//数据库打开失败则不用处理
if (!dbConn.open()) {
qDebug() << this->objectName() << dbConn.lastError();
return;
}
while (!stopped) {
//定时执行一次校验时间是否到了该清理的时候
QDateTime now = QDateTime::currentDateTime();
if (lastTime.secsTo(now) >= interval) {
if (isBusy) {
qDebug() << this->objectName() << "DbCleanThread isBusy";
lastTime = now;
continue;
}
isBusy = true;
clean();
isBusy = false;
lastTime = now;
msleep(1);
}
msleep(100);
}
stopped = false;
}
void DbCleanThread::setTableName(const QString &tableName)
{
this->tableName = tableName;
}
void DbCleanThread::setWhereColumnName(const QString &whereColumnName)
{
this->whereColumnName = whereColumnName;
}
void DbCleanThread::setOrderSql(const QString &orderSql)
{
this->orderSql = orderSql;
}
void DbCleanThread::setMaxCount(int maxCount)
{
this->maxCount = maxCount;
}
void DbCleanThread::setInterval(int interval)
{
this->interval = interval;
}
void DbCleanThread::setPar(const QString &tableName, const QString &countName,
const QString &whereColumnName, const QString &orderSql,
int maxCount, int interval)
{
this->tableName = tableName;
this->countName = countName;
this->whereColumnName = whereColumnName;
this->orderSql = orderSql;
this->maxCount = maxCount;
this->interval = interval;
}
void DbCleanThread::setDirPath(const QString &dirPath)
{
this->dirPath = dirPath;
}
void DbCleanThread::setDirFileFilter(const QStringList &dirFileFilter)
{
this->dirFileFilter = dirFileFilter;
}
void DbCleanThread::setDirMaxSize(int dirMaxSize)
{
this->dirMaxSize = dirMaxSize;
}
void DbCleanThread::setConnInfo(DbCleanThread::DbType dbType, const QString &connName, const QString &hostName, int port,
const QString &dbName, const QString &userName, const QString &userPwd)
{
this->dbType = dbType;
this->connName = connName;
this->hostName = hostName;
this->port = port;
this->dbName = dbName;
this->userName = userName;
this->userPwd = userPwd;
}
void DbCleanThread::deleteDirectory(const QString &path)
{
QDir dir(path);
if (!dir.exists()) {
return;
}
dir.setFilter(QDir::AllEntries | QDir::NoDotAndDotDot);
QFileInfoList fileList = dir.entryInfoList();
foreach (QFileInfo fi, fileList) {
if (fi.isFile()) {
fi.dir().remove(fi.fileName());
} else {
deleteDirectory(fi.absoluteFilePath());
dir.rmdir(fi.absoluteFilePath());
}
}
}
void DbCleanThread::stop()
{
stopped = true;
}
void DbCleanThread::clean()
{
//首先查找总记录数,如果总记录数超过限制,则将超出的部分按照字段排序进行删除
QString sql = QString("select count(%1) from %2").arg(countName).arg(tableName);
QSqlQuery query(QSqlDatabase::database(connName));
query.exec(sql);
query.next();
int count = query.value(0).toInt();
int cleanCount = (count - maxCount);
if (cleanCount >= 100) {
QDateTime dtStart = QDateTime::currentDateTime();
//每次最大清理1000条数据
cleanCount = cleanCount > 1000 ? 1000 : cleanCount;
//将要删除的数据指定字段集合查询出来
query.clear();
sql = QString("select %2 from %1 order by %3 limit %4").arg(tableName).arg(whereColumnName).arg(orderSql).arg(cleanCount);
query.exec(sql);
QStringList list;
while(query.next()) {
list << query.value(0).toString();
}
//删除数据
query.clear();
sql = QString("delete from %1 where %2 in(%3)").arg(tableName).arg(whereColumnName).arg(list.join(","));
bool ok = query.exec(sql);
QDateTime dtEnd = QDateTime::currentDateTime();
double msec = dtStart.msecsTo(dtEnd);
emit cleanFinsh(ok, cleanCount, msec);
}
//自动清理文件夹
if (!dirPath.isEmpty()) {
//找出该文件夹下的所有文件夹
QDir dir(dirPath);
if (!dir.exists()) {
return;
}
//按照目录查找,过滤文件夹,按照文件名称排序
dir.setFilter(QDir::Dirs | QDir::NoSymLinks | QDir::NoDotAndDotDot);
dir.setSorting(QDir::Name);
QStringList list = dir.entryList();
//遍历所有目录,对所有文件大小相加得到总大小,文件就在文件夹下,不会再有子目录
qint64 size = 0;
foreach (QString path, list) {
QDir d(dirPath + "/" + path);
QFileInfoList infos = d.entryInfoList(dirFileFilter);
foreach (QFileInfo info, infos) {
size += info.size();
}
//转化成MB,超过预定大小自动删除第一个文件夹,跳出循环无需继续判断
int sizeMB = size / (1024 * 1024);
if (sizeMB >= dirMaxSize) {
//删除该目录下的所有文件
QString firstDir = dirPath + "/" + list.at(0);
QDir dir(firstDir);
dir.setFilter(QDir::AllEntries | QDir::NoDotAndDotDot);
QStringList list = dir.entryList();
foreach (QString file, list) {
dir.remove(file);
qDebug() << "remove file" << firstDir << file;
}
//删除文件夹本身
dir.rmdir(firstDir);
qDebug() << "remove dir" << firstDir;
emit cleanDir(firstDir);
break;
}
}
}
}