高光谱图像分类

1. Paper解读

下面代码原理来自2019年IEEE论文：
HybridSN: Exploring 3-D–2-D CNN Feature
Hierarchy for Hyperspectral Image Classification
该论文认为二维卷积不能提取光谱维度的特征，只能提取空间维度的特征；而三维卷积计算过于复杂；基于这两种方法各有优缺点，因此作者提出一种三维和二维卷积混合的方法来进行高光谱图像分类。
主要网络结构如下图所示：

原始输入为M×N×D，M为长度，N为宽度，D为光谱数量。
由于高光谱图像会显示出混合的土地覆盖类别，因此引入高类内变异性和类间相似性。为了消除光谱冗余，对原始的HSI数据进行principal component analysis(PCA)，如图所示，此时光谱数D变为了B，空间域M和N的值不变。
接下来，将HSI数据块划分为小的重叠的3-D-patches，小块的真值标签由中间像素的标签决定。3-D-patches总数为(M-S+1)×(N-S+1)。
接下来为3个3-D-CNN，1个2-D-CNN，3个全连接层。

上面的结构用表格显示如下：

2. 代码

1. 引入数据和基本函数库

! wget http://www.ehu.eus/ccwintco/uploads/6/67/Indian_pines_corrected.mat
! wget http://www.ehu.eus/ccwintco/uploads/c/c4/Indian_pines_gt.mat
! pip install spectral

import numpy as np
import matplotlib.pyplot as plt
import scipy.io as sio
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, accuracy_score, classification_report, cohen_kappa_score
import spectral
import torch
import torchvision
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

2. 定义 HybridSN 类

三维卷积部分：

conv1：（1, 30, 25, 25）， 8个 7x3x3 的卷积核 ==>（8, 24, 23, 23）
conv2：（8, 24, 23, 23）， 16个 5x3x3 的卷积核 ==>（16, 20, 21, 21）
conv3：（16, 20, 21, 21），32个 3x3x3 的卷积核 ==>（32, 18, 19, 19）
接下来要进行二维卷积，因此把前面的 32*18 reshape 一下，得到 （576, 19, 19）

二维卷积：（576, 19, 19） 64个 3x3 的卷积核，得到 （64, 17, 17）

接下来是一个 flatten 操作，变为 18496 维的向量，

接下来依次为256，128节点的全连接层，都使用比例为0.4的 Dropout，

最后输出为 16 个节点，是最终的分类类别数。

下面是 HybridSN 类的代码：

class HybridSN(nn.Module):
  def __init__(self):
    super(HybridSN, self).__init__()
    # 3D卷积层
    self.conv1 = nn.Conv3d(1, 8, (7, 3, 3))
    self.conv2 = nn.Conv3d(8, 16, (5, 3, 3))
    self.conv3 = nn.Conv3d(16, 32, (3, 3, 3))
    # 2D卷积层
    self.conv4 = nn.Conv2d(576, 64, (3, 3))
    # 全连接层
    self.liner1 = nn.Linear(18496, 256)
    self.liner2 = nn.Linear(256, 128)
    self.liner3 = nn.Linear(128, 16)
    # Dropout
    self.dropout = nn.Dropout(0.4)
    # 激活函数
    self.relu = nn.LeakyReLU()
    self.logsoftmax = nn.LogSoftmax(dim=1)

  def forward(self, x):
    out = self.conv1(x)
    out = self.relu(out)
    out = self.conv2(out)
    out = self.relu(out)
    out = self.conv3(out)
    out = self.relu(out)
    
    out = out.view(-1, out.shape[1]*out.shape[2], out.shape[3], out.shape[4])
    out = self.conv4(out)
    out = self.relu(out)
    
    out = out.view(x.size(0), -1)
    out = self.liner1(out)
    out = self.relu(out)
    out = self.dropout(out)
    out = self.liner2(out)
    out = self.relu(out)
    out = self.dropout(out)
    out = self.liner3(out)
    out = self.logsoftmax(out)
    return out

# 随机输入，测试网络结构是否通
x = torch.randn(1, 1, 30, 25, 25)
net = HybridSN()
y = net(x)
print(y.shape)

网络结构测试可以得到下面结果，说明这里是ok的

3. 创建数据集

首先对高光谱数据实施PCA降维；然后创建 keras 方便处理的数据格式；然后随机抽取 10% 数据做为训练集，剩余的做为测试集。
首先定义基本函数：

# 对高光谱数据 X 应用 PCA 变换
def applyPCA(X, numComponents):
    newX = np.reshape(X, (-1, X.shape[2]))
    pca = PCA(n_components=numComponents, whiten=True)
    newX = pca.fit_transform(newX)
    newX = np.reshape(newX, (X.shape[0], X.shape[1], numComponents))
    return newX

# 对单个像素周围提取 patch 时，边缘像素就无法取了，因此，给这部分像素进行 padding 操作
def padWithZeros(X, margin=2):
    newX = np.zeros((X.shape[0] + 2 * margin, X.shape[1] + 2* margin, X.shape[2]))
    x_offset = margin
    y_offset = margin
    newX[x_offset:X.shape[0] + x_offset, y_offset:X.shape[1] + y_offset, :] = X
    return newX

# 在每个像素周围提取 patch ，然后创建成符合 keras 处理的格式
def createImageCubes(X, y, windowSize=5, removeZeroLabels = True):
    # 给 X 做 padding
    margin = int((windowSize - 1) / 2)
    zeroPaddedX = padWithZeros(X, margin=margin)
    # split patches
    patchesData = np.zeros((X.shape[0] * X.shape[1], windowSize, windowSize, X.shape[2]))
    patchesLabels = np.zeros((X.shape[0] * X.shape[1]))
    patchIndex = 0
    for r in range(margin, zeroPaddedX.shape[0] - margin):
        for c in range(margin, zeroPaddedX.shape[1] - margin):
            patch = zeroPaddedX[r - margin:r + margin + 1, c - margin:c + margin + 1]   
            patchesData[patchIndex, :, :, :] = patch
            patchesLabels[patchIndex] = y[r-margin, c-margin]
            patchIndex = patchIndex + 1
    if removeZeroLabels:
        patchesData = patchesData[patchesLabels>0,:,:,:]
        patchesLabels = patchesLabels[patchesLabels>0]
        patchesLabels -= 1
    return patchesData, patchesLabels

def splitTrainTestSet(X, y, testRatio, randomState=345):
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=testRatio, random_state=randomState, stratify=y)
    return X_train, X_test, y_train, y_test

读取并创建数据集：

# 地物类别
class_num = 16
X = sio.loadmat('Indian_pines_corrected.mat')['indian_pines_corrected']
y = sio.loadmat('Indian_pines_gt.mat')['indian_pines_gt']

# 用于测试样本的比例
test_ratio = 0.90
# 每个像素周围提取 patch 的尺寸
patch_size = 25
# 使用 PCA 降维，得到主成分的数量
pca_components = 30

print('Hyperspectral data shape: ', X.shape)
print('Label shape: ', y.shape)

print('\n... ... PCA tranformation ... ...')
X_pca = applyPCA(X, numComponents=pca_components)
print('Data shape after PCA: ', X_pca.shape)

print('\n... ... create data cubes ... ...')
X_pca, y = createImageCubes(X_pca, y, windowSize=patch_size)
print('Data cube X shape: ', X_pca.shape)
print('Data cube y shape: ', y.shape)

print('\n... ... create train & test data ... ...')
Xtrain, Xtest, ytrain, ytest = splitTrainTestSet(X_pca, y, test_ratio)
print('Xtrain shape: ', Xtrain.shape)
print('Xtest  shape: ', Xtest.shape)

# 改变 Xtrain, Ytrain 的形状，以符合 keras 的要求
Xtrain = Xtrain.reshape(-1, patch_size, patch_size, pca_components, 1)
Xtest  = Xtest.reshape(-1, patch_size, patch_size, pca_components, 1)
print('before transpose: Xtrain shape: ', Xtrain.shape) 
print('before transpose: Xtest  shape: ', Xtest.shape) 

# 为了适应 pytorch 结构，数据要做 transpose
Xtrain = Xtrain.transpose(0, 4, 3, 1, 2)
Xtest  = Xtest.transpose(0, 4, 3, 1, 2)
print('after transpose: Xtrain shape: ', Xtrain.shape) 
print('after transpose: Xtest  shape: ', Xtest.shape) 


""" Training dataset"""
class TrainDS(torch.utils.data.Dataset): 
    def __init__(self):
        self.len = Xtrain.shape[0]
        self.x_data = torch.FloatTensor(Xtrain)
        self.y_data = torch.LongTensor(ytrain)        
    def __getitem__(self, index):
        # 根据索引返回数据和对应的标签
        return self.x_data[index], self.y_data[index]
    def __len__(self): 
        # 返回文件数据的数目
        return self.len

""" Testing dataset"""
class TestDS(torch.utils.data.Dataset): 
    def __init__(self):
        self.len = Xtest.shape[0]
        self.x_data = torch.FloatTensor(Xtest)
        self.y_data = torch.LongTensor(ytest)
    def __getitem__(self, index):
        # 根据索引返回数据和对应的标签
        return self.x_data[index], self.y_data[index]
    def __len__(self): 
        # 返回文件数据的数目
        return self.len

# 创建 trainloader 和 testloader
trainset = TrainDS()
testset  = TestDS()
train_loader = torch.utils.data.DataLoader(dataset=trainset, batch_size=128, shuffle=True, num_workers=2)
test_loader  = torch.utils.data.DataLoader(dataset=testset,  batch_size=128, shuffle=False, num_workers=2)

4. 训练

# 使用GPU训练，可以在菜单 "代码执行工具" -> "更改运行时类型" 里进行设置
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

# 网络放到GPU上
net = HybridSN().to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(), lr=0.001)

# 开始训练
total_loss = 0
for epoch in range(100):
    for i, (inputs, labels) in enumerate(train_loader):
        inputs = inputs.to(device)
        labels = labels.to(device)
        # 优化器梯度归零
        optimizer.zero_grad()
        # 正向传播 +　反向传播 + 优化 
        outputs = net(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        total_loss += loss.item()
    print('[Epoch: %d]   [loss avg: %.4f]   [current loss: %.4f]' %(epoch + 1, total_loss/(epoch+1), loss.item()))

print('Finished Training')

这里epoch取的较小，只有100。由于HybridSN 类中的激活函数选取的是LeakyReLU()和LogSoftmax()，因此只要100次就可以将loss收敛到1.3111，如下图所示。如果用ReLU()的话，epoch=100的情况下一般训练结果loss在2~4之间的小数，效果相对更差。当然，增加epoch必会导致Loss更小，这里就暂时还用epoch=100。

5. 模型测试

count = 0
# 模型测试
for inputs, _ in test_loader:
    inputs = inputs.to(device)
    outputs = net(inputs)
    outputs = np.argmax(outputs.detach().cpu().numpy(), axis=1)
    if count == 0:
        y_pred_test =  outputs
        count = 1
    else:
        y_pred_test = np.concatenate( (y_pred_test, outputs) )

# 生成分类报告
classification = classification_report(ytest, y_pred_test, digits=4)
print(classification)

准确率97.31%，这个结果还一般。

6. 显示分类结果

# load the original image
X = sio.loadmat('Indian_pines_corrected.mat')['indian_pines_corrected']
y = sio.loadmat('Indian_pines_gt.mat')['indian_pines_gt']

height = y.shape[0]
width = y.shape[1]

X = applyPCA(X, numComponents= pca_components)
X = padWithZeros(X, patch_size//2)

# 逐像素预测类别
outputs = np.zeros((height,width))
for i in range(height):
    for j in range(width):
        if int(y[i,j]) == 0:
            continue
        else :
            image_patch = X[i:i+patch_size, j:j+patch_size, :]
            image_patch = image_patch.reshape(1,image_patch.shape[0],image_patch.shape[1], image_patch.shape[2], 1)
            X_test_image = torch.FloatTensor(image_patch.transpose(0, 4, 3, 1, 2)).to(device)                                   
            prediction = net(X_test_image)
            prediction = np.argmax(prediction.detach().cpu().numpy(), axis=1)
            outputs[i][j] = prediction+1
    if i % 20 == 0:
        print('... ... row ', i, ' handling ... ...')

predict_image = spectral.imshow(classes = outputs.astype(int),figsize =(5,5))

这结果图跟她论文里比还差的远嘛，论文的结果对比图如下图所示。不过这也正常，作者肯定是训练不知道多少次之后选取的最好结果，我这里只训练了几次取得结果其实也还可以了。

3. 思考

1）思考3D卷积和2D卷积的区别？
2D卷积只能提取二维特征，就如这篇文章中所说的，只提取了空间特征而不能提取光谱特征；三维卷积多一个维度自然可以提取更多的特征，但是多一个维度就意味着更复杂的计算量。由于很多深度学习框架本身就有极大的深度，每一层再都运用三维卷积，那计算量就太大了。因此2D卷积相对3D卷积更常见一些。

2）训练网络，然后多测试几次，会发现每次分类的结果都不一样，请思考为什么？
我认为可能是两方面原因：
一方面是初始阈值和权值不同，导致每次训练结果不同。
二是训练时，我们根据设置的dropout值，每次dropout掉不同的隐藏神经元来进行网络训练，因此会产生结果不同。

3）如果想要进一步提升高光谱图像的分类性能，可以如何使用注意力机制？
一般可以从空间注意力和通道注意力这两个方面入手，增加分类性能。无论是对图像分类还是对图像识别和图像分割来说，将注意力机制加入到网络当中，一般都会显著提高性能。对于常用的SEnet，需要添加两个全连接层，网络会更加复杂一些，参数也会相应增加，可采用一些新颖的轻量级的注意力网络，比如最近我看的一篇CVPR2020的论文介绍的一个新颖的ECAnet，不需要降维操作就可以跨通道交互，只要一维卷积即可。
下篇博客我会试着讲述该论文。
ECA-Net: Efficient Channel Attention for Deep Convolutional Neural Networks
下面是ECAnet核心代码：

class eca_layer(nn.Module):
    """Constructs a ECA module.
    Args:
        channel: Number of channels of the input feature map
        k_size: Adaptive selection of kernel size
    """
    def __init__(self, channel, k_size=3):
        super(eca_layer, self).__init__()
        self.avg_pool = nn.AdaptiveAvgPool2d(1)
        self.conv = nn.Conv1d(1, 1, kernel_size=k_size, padding=(k_size - 1) // 2, bias=False) 
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        # x: input features with shape [b, c, h, w]
        b, c, h, w = x.size()

        # feature descriptor on the global spatial information
        y = self.avg_pool(x)

        # Two different branches of ECA module
        y = self.conv(y.squeeze(-1).transpose(-1, -2)).transpose(-1, -2).unsqueeze(-1)

        # Multi-scale information fusion
        y = self.sigmoid(y)

        return x * y.expand_as(x)

添加到HybridSN中：

class HybridSN(nn.Module):
  def __init__(self):
    super(HybridSN, self).__init__()
    # 3D卷积层
    self.conv1 = nn.Conv3d(1, 8, (7, 3, 3))
    self.conv2 = nn.Conv3d(8, 16, (5, 3, 3))
    self.conv3 = nn.Conv3d(16, 32, (3, 3, 3))
    # 2D卷积层
    self.conv4 = nn.Conv2d(576, 64, (3, 3))
    # 全连接层
    self.liner1 = nn.Linear(18496, 256)
    self.liner2 = nn.Linear(256, 128)
    self.liner3 = nn.Linear(128, 16)
    # Dropout
    self.dropout = nn.Dropout(0.4)
    # 激活函数
    self.relu = nn.LeakyReLU()
    self.logsoftmax = nn.LogSoftmax(dim=1)
    # ECAnet
    self.eca = eca_layer(64, 3)

  def forward(self, x):
    out = self.conv1(x)
    out = self.relu(out)
    out = self.conv2(out)
    out = self.relu(out)
    out = self.conv3(out)
    out = self.relu(out)
    
    out = out.view(-1, out.shape[1]*out.shape[2], out.shape[3], out.shape[4])
    out = self.conv4(out)
    out = self.relu(out)

    out = self.eca(out)
    
    out = out.view(x.size(0), -1)
    out = self.liner1(out)
    out = self.relu(out)
    out = self.dropout(out)
    out = self.liner2(out)
    out = self.relu(out)
    out = self.dropout(out)
    out = self.liner3(out)
    out = self.logsoftmax(out)
    return out

几次测试后发现出现结果都在0.977以上，比之前的测试结果要好：

参考文献
[1] Roy, Swalpa Kumar, et al. “HybridSN: Exploring 3-D–2-D CNN feature hierarchy for hyperspectral image classification.” IEEE Geoscience and Remote Sensing Letters 17.2 (2019): 277-281.