springboot+mongodb+python+echarts数据分析与预测
实验室检测数据展示与预测
- 零：概要
- 一、前端输出
- 二、构建过程
零：概要
目标：数据监测与数据预测
系统：centos7 docker发布
展示方式：使用echarts展示，数据实时监测制作模型，指标分析模型，回归模型
方法：1、从mongodb取值，制作数据模型，
2、调用python算法进行数据的计算及展示
一、前端输出
1、实时展示
2、指标模型
3、标准化OLS
二、构建过程
1、pom.xml配置，回载模块
<dependencies>
<dependency>
<groupId>org.springframework.bootgroupId>
<artifactId>spring-boot-starter-data-mongodbartifactId>
dependency>
<dependency>
<groupId>org.springframework.bootgroupId>
<artifactId>spring-boot-starter-webartifactId>
dependency>
<dependency>
<groupId>org.springframework.bootgroupId>
<artifactId>spring-boot-starter-thymeleafartifactId>
dependency>
<dependency>
<groupId>org.springframework.bootgroupId>
<artifactId>spring-boot-starter-testartifactId>
<scope>testscope>
dependency>
dependencies>
2、配置数据库
spring:
data:
mongodb:
uri: mongodb://username:password@localhost:27017/databases
thymeleaf:
cache: false
server:
port: 8081
3、构建接口调用mongodb数据
public class MongoRealTime {
@Autowired
MongoTemplate mongoTemplate;
public AggregateIterable<Document> getdata(String collection, String order,long timeval) {
String collectionName=collection;
String orderNum=order;
Integer depth=24; //查看数据量24小时
MongoCollection<Document> coll = null;
String col=null;
Date start_time=new Date(timeval+1000*60*60*8);
Date end_time=new Date(timeval+1000*60*60*(depth+8));
Document match = new Document();
match.put("MachineNum", new Document("$exists", true));
match.put("JobNum", new Document("$exists", true));
Document match_time=new Document();
match_time.put("$exists", true);
match_time.put("$gt", start_time);
match_time.put("$lt",end_time);
match.put("CreateTime",match_time);
Document project = new Document();
project.put("_id", 0);
project.put("MachineNum", 1);
project.put("JobNum", 1);
project.put("CreateTime", 1);
if (collectionName.equals( "OutWater")) {
match.put("FullDrain", new Document("$exists", true));
match.put("HalfDrain", new Document("$exists", true));
match.put("DrainOrder", orderNum);
project.put("FullDrain", 1);
project.put("HalfDrain", 1);
col="WaterValve";
} else if (collectionName.equals("SmartCover")) {
match.put("SeatTime", new Document("$exists", true));
match.put("CoverTime", new Document("$exists", true));
match.put("Order", orderNum);
project.put("SeatTime", 1);
project.put("CoverTime", 1);
col="SmartCover";
} else if (collectionName.equals("InterWater")) {
match.put("RealWaterLevel", new Document("$exists", true));
match.put("InletOrder", orderNum);
project.put("RealWaterLevel", 1);
col="WaterValve";
}
Document group_match = new Document("$match", match);
Document group_project = new Document("$project", project);
List<Document> aggregateList = new ArrayList<Document>();
aggregateList.add(group_match);
aggregateList.add(group_project);
AggregateIterable<Document> docs=mongoTemplate.getCollection(col).aggregate(aggregateList);
return docs;
}
}
4、api
public class MongoContoller {
@Autowired
MongoRealTime mongoRealTime;
@ResponseBody
@RequestMapping("/realtimelist")
public HashMap realtimelist(
String conn
, String order
,long timeval
){
return mongoRealTime.getrealtime(conn,order,timeval); }
}
5、html展示
<div class="data_bodey">
<div th:replace="commons/reportheadbar::#reportheadMenu(activerUri='r3')">div>
<div class="inner" style="height: 90%;">
<div class="dataAllBorder01">
<div class="dataAllBorder01">
<p class="data_chartall" id="realtimeid" >数据加载中。。。。。p>
div>
div>
div>
div>
6、模型渲染
function labrealtime(conn, order, date) {
var myChart = echarts.init(document.getElementById('realtimeid'));
var option;
$.getJSON('/realtimelist', {
"conn": conn, "order": order, "timeval": date.getTime()}, function (_rawData) {
var MachineNums = _rawData['key'][0];
var datasetWithFilters = [];
var seriesList = [];
var yname = null;
var ychange = null; //转换/100
var yname100 = null;
echarts.util.each(MachineNums, function (MachineNum) {
var ylist = [];
if (conn == "SmartCover") {
ylist.push("SeatTime");
ylist.push("CoverTime");
yname = '单位：秒';
yname100 = 'ms';
ychange = 100;
} else if (conn == "OutWater") {
ylist.push("FullDrain");
ylist.push("HalfDrain");
yname = '单位：mm';
ychange = 10;
yname100 = 'mm';
} else {
ylist.push("RealWaterLevel");
yname = '单位：mm';
ychange = 10;
yname100 = 'mm';
}
for (let i = 0; i < ylist.length; i++) {
datasetWithFilters.push({
id: 'dataset_' + MachineNum + '_' + ylist[i],
fromDatasetId: 'dataset_raw',
transform: {
type: 'filter',
config: {
and: [
{
dimension: ylist[i], gt: 0},
{
dimension: 'MachineNum', '=': MachineNum}
]
}
}
},
);
seriesList.push({
type: 'line',
datasetId: 'dataset_' + MachineNum + '_' + ylist[i],
showSymbol: false,
name: MachineNum,
endLabel: {
show: true,
formatter: function (params) {
return ylist[i] + params.value[0] + ': ' + params.value[2 + i] + yname100;
},
color: '#fbff24',
},
labelLayout: {
moveOverlap: 'shiftY'
},
emphasis: {
focus: 'series'
},
encode: {
x: 'CreateTime',
y: ylist[i],
label: ['MachineNum', ylist[i]],
tooltip: ['MachineNum', ylist[i]],
}
},
);
}
});
option = {
animationDuration: 5000,
dataset: [{
id: 'dataset_raw',
source: _rawData['data']
}].concat(datasetWithFilters),
title: {
text: '机台数据实时监测',
textStyle: {
color: '#38f3ff'
}
},
tooltip: {
order: 'valueDesc',
trigger: 'axis'
},
xAxis: {
// type: 'category',
type: 'time',
nameLocation: 'middle',
axisLabel: {
color: '#38f3ff',
fontWeigth: 'bold',
fontSize: 20,
}
},
yAxis: {
name: yname,
nameTextStyle: {
fontSize: 12,
color: 'rgb(255,255,255,0.8)',
},
axisLabel: {
fontSize: 20,
fontWeight: 'bold',
color: function (value, index) {
return ychange === 100 && value <= 2000 && value >= 300 ? 'green' : 'red';
},
formatter: function (value, index) {
return value / ychange;
},
},
splitLine: {
show: true,
lineStyle: {
color: 'rgb(18,100,254,0.3)'
}
},
},
grid: {
right: 180,
},
series: seriesList,
};
myChart.clear();
myChart.setOption(option);
});
option && myChart.setOption(option);
}
7、python建模（使用java调用python进行建模，结果集用于数据的展示）
class labModel:
def __init__(self):
labmongodb = MongoClient("mongodb://localhost:27017")
self.servercoll = labmongodb["machine"]
self.anaysiscoll = self.servercoll['AnalysisDB']
def get_data(self, docslist):
obj = docslist.get('ObjectName')
if obj == "CoverTime" or obj == "SeatTime":
databasename = "SmartCover"
ordername = "Order"
elif obj == "FullDrain" or obj == "HalfDrain":
databasename = "WaterValve"
ordername = "DrainOrder"
else:
databasename = "WaterValve"
ordername = "InletOrder"
coll = self.servercoll[databasename]
docs = coll.aggregate([
{
'$match': {
ordername: docslist.get('Order'),
'MachineNum': docslist.get('MachineNum'),
'JobNum': int(docslist.get('JobNum')),
docslist.get('ObjectName'):{
'$gt':0}
}
},
{
'$project': {
'CreateTime': 1, '_id': 0, obj: 1}},
{
'$sort': {
'CreateTime': 1}}
])
df = pd.DataFrame(list(docs))
return df
def df_OLS(self, df, obj):
# 抽样
df_train = df.sample(300)
features = df_train[obj].index.values.reshape(-1, 1)
target = df_train[obj]
train_x, test_x, train_y, test_y = train_test_split(features, target, test_size=0.3, random_state=42)
test_x.sort(axis=0)
train_x = sm.add_constant(train_x)
test_x1 = sm.add_constant(test_x)
est = sm.OLS(train_y, train_x)
est2 = est.fit()
predict_est = est2.predict(test_x1) - test_y # 残差
predict_est_mean = np.mean(predict_est) # 残差均值
predict_mean_std = (predict_est - predict_est_mean) / np.std(predict_est) # 标准化残差
fit_v=pd.DataFrame(train_x, columns=['b', 'x'])
fit_v.drop('b',axis=1,inplace=True)
fit_v['y']=est2.fittedvalues.values.tolist()
fit_v['resid_pearson']=est2.resid_pearson.tolist()
fittedvalues_v=np.append((fit_v.columns.to_numpy()).reshape((1,-1)),fit_v.to_numpy(),axis=0)
test_v=pd.DataFrame(test_x,columns=['x'])
test_v['y']=test_y.tolist()
test_v['predict_y']=est2.predict(test_x1).tolist()
test_v['ZRESID']=predict_mean_std.tolist()
# predict_v=np.insert(test_v.to_numpy(),0,['x','y','predict_y','ZRESID'],axis=0)
predict_v=np.append((test_v.columns.to_numpy()).reshape((1,-1)),test_v.to_numpy(),axis=0)
ols_document = {
'fittedvalues':fittedvalues_v.tolist(),
'predict':predict_v.tolist(),
'model': {
'r2': est2.rsquared, # 方程R方
'p': est2.f_pvalue, # 方程p
'params': est2.params.to_dict(), # 参数
'stdbse': est2.bse.to_dict(), # 参数估计标准误
'pvalues': est2.pvalues.to_dict(), # 参数p值
}
}
return ols_document
def df_save(self, docs):
# coll=self.colltestdb #localhost mongodb
coll = self.anaysiscoll # serverhost mongodb
document_exists = {
'Type': docs.get('Type'), 'Order': docs.get('Order'), 'MachineNum': docs.get('MachineNum'),
'JobNum': docs.get('JobNum'),
'ObjectName': docs.get('ObjectName')}
coll.replace_one(document_exists, docs, upsert=True)
if __name__ == '__main__':
try:
docs = {
}
docs['Type'] = 2
docs['ObjectName'] = sys.argv[1] # 项目进水，排水，慢落
docs['Order'] = sys.argv[2] # 订单号
docs['MachineNum'] = sys.argv[3] # 机台
docs['JobNum'] = sys.argv[4] # 工位号
docs['RunDateTime'] = datetime.now()
try:
lab = labModel()
data = lab.get_data(docs)
if len(data) > 100:
olsdocs = lab.df_OLS(data, docs.get('ObjectName'))
docs.update(olsdocs)
lab.df_save(docs)
print('ok')
except:
print('err')
except IndexError:
print('err')