一维卷积神经网络在近红外光谱分析中的应用

尝试1维卷积网络运用于光谱近红外分析，可能是样本数太少，目前测试结果不是很理想。样本数据：https://pan.baidu.com/s/1IuMSPOVmSD26IFgf2pCDqg 第一列是要拟合的水分含量， 后50列为光谱数据。

import pathlib
import sys
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import tensorflow as tf
import numpy as np
from tensorflow import keras
from tensorflow.keras import layers
from keras.layers import Reshape
from keras.layers import Embedding, Conv1D,MaxPooling1D,GlobalAveragePooling1D,Dense
from keras.models import Sequential
from keras.layers import Dropout
from matplotlib import rcParams
from keras.layers.normalization import BatchNormalization

column_names = ['water','1','2','3','4','5', '6', '7','8','9','10','11','12','13','14','15', '16', '17','18','19','20','21','22','23','24','25', '26', '27','28','29','30',
               '31','32','33','34','35', '36', '37','38','39','40','41','42','43','44','45', '46', '47','48','49','50']                      
raw_dataset = pd.read_csv('./water.csv',names=column_names, sep=',',header = None, encoding='gkb')                      
dataset = raw_dataset.copy()
# 查看前3条数据
print(dataset.head(3))

dataset.isna().sum()

dataset = dataset.dropna()

train_dataset = dataset.sample(frac=0.8, random_state=22)
print("train:")
print(train_dataset)
test_dataset = dataset.drop(train_dataset.index)
print("test:")
print(test_dataset)

# 解决中文乱码问题
plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus']=False

train_stats = train_dataset.describe()
print(train_stats)
train_stats.pop("water")
train_stats = train_stats.transpose()
print(train_stats)

train_labels = train_dataset.pop('water')
print(train_labels)
test_labels = test_dataset.pop('water')
print("water:")
print(test_labels)

def norm(x):
  return (x - train_stats['mean']) / train_stats['std']
  
  
normed_train_data = norm(train_dataset)
normed_test_data = norm(test_dataset)

#def build_model():        全连接神经网络
#  input_dim = len(train_dataset.keys())
  
#  model = keras.Sequential([
#    layers.Dense(64, activation='relu', input_shape=[input_dim,]),
#    layers.Dense(64, activation='relu'),
#    layers.Dense(1)
#  ])
#  model.compile(loss='mse', metrics=['mae', 'mse'],
#                optimizer=tf.keras.optimizers.RMSprop(0.001))
#  return model

def build_model():          //1D卷积神经网络
    input_dim = len(train_dataset.keys())
    model = Sequential()
    model.add(Conv1D(10, 7, activation='relu',input_shape=[input_dim,1]))
    model.add(MaxPooling1D(2))
    model.add(Conv1D(6, 7, activation='relu'))
    model.add(MaxPooling1D(2))

    model.add(GlobalAveragePooling1D())
    model.add(Dense(1,activation='linear'))
    model.compile(loss='mse', metrics=['mae', 'mse'],
                optimizer='adam')
    return model

model = build_model()
# 打印模型的描述信息，每一层的大小、参数个数等
model.summary()

EPOCHS = 100 

class ProgressBar(keras.callbacks.Callback):
  def on_epoch_end(self, epoch, logs):
    # 显示进度条
    self.draw_progress_bar(epoch + 1, EPOCHS)

  def draw_progress_bar(self, cur, total, bar_len=50):
    cur_len = int(cur / total * bar_len)
    sys.stdout.write("\r")
    sys.stdout.write("[{:<{}}] {}/{}".format("=" * cur_len, bar_len, cur, total))
    sys.stdout.flush()

    
#early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)    
    
history = model.fit(
  np.expand_dims(normed_train_data,2), train_labels,
  epochs=EPOCHS, validation_split=0.1, verbose=0,batch_size=4,shuffle=True)
  
hist = pd.DataFrame(history.history)
hist['epoch'] = history.epoch
hist.tail(3)
print(hist)

def plot_history(history):
  hist = pd.DataFrame(history.history)
  hist['epoch'] = history.epoch
  plt.figure()
  plt.xlabel('epoch')
  plt.ylabel('metric - MSE')
  plt.plot(hist['epoch'], hist['mse'], label='train')
  plt.plot(hist['epoch'], hist['val_mse'], label = 'test')
  plt.ylim([0, 500])
  plt.legend()

  
  plt.figure()
  plt.xlabel('epoch')
  plt.ylabel('metric - MAE')
  plt.plot(hist['epoch'], hist['mae'], label='train')
  plt.plot(hist['epoch'], hist['val_mae'], label = 'test')
  plt.ylim([0, 50])
  plt.legend()

  
plot_history(history)

loss, mae, mse = model.evaluate(np.expand_dims(normed_test_data,2), test_labels, verbose=0)
print("平均绝对误差(MAE): {:5.2f} ".format(mae))

test_pred = model.predict(np.expand_dims(normed_test_data,2)).flatten()

print("!!!!!!!!!!!!!!!!!!")
print(test_labels)
print("~~~~~~~~~~~~~~~~~~")
print(test_pred)

plt.figure()
plt.scatter(test_labels, test_pred)
plt.xlabel('真实值')
plt.ylabel('预测值')
plt.axis('equal')
plt.axis('square')
plt.xlim([50,70])
plt.ylim([50,70])
plt.plot([-100, 100], [-100, 100])

rcParams['font.sans-serif'] = 'SimHei'
fig = plt.figure(figsize=(10,6))
plt.plot(range(test_labels.shape[0]),test_labels,color="blue", linewidth=1.5, linestyle="-")
plt.plot(range(test_labels.shape[0]),test_pred,color="red", linewidth=1.5, linestyle="-")
plt.xlim((0,200))
plt.ylim((50,70))
plt.legend(['real','pred'])

plt.show()
 

运行结果如下：