Deep Convolutional and LSTM Recurrent Neural Networks for Multimodal Wearable Activity Recognition

keywords: 活动识别; 时间序列分类; 传感器网络; 深度学习; keras

文章发表在2016年的Sensors杂志上，提出结合使用卷积和LSTM的网络结构DeepConvLSTM，处理Opportunity和Skoda两个复杂活动数据集。在Opportunity数据集上的Locomotion和gesture两个子任务上，分别实现了0.93和0.866的F1-score；在Skoda数据集上，实现了0.958的F1-score。在活动识别领域，首次提出卷积和LSTM的结合，并达到state-of-art。
本文使用Python3，numpy，keras复现该论文，最终实现接近于论文提到的F1-score。

Opportunity数据集

Opportunity数据集，在部署了大量传感器的封闭环境内采集五个实验对象（S1-S5）连续5天（ADL1-5，Drill）的活动数据。

• 论文中使用了人体穿戴的加速度计和IMU传感器数据，仅采用加速度、陀螺仪、磁力计三类共110个传感器信号。

# col number of sensor signals
acc_map = {
    'RKN^': range(1, 4),
    'HIP': range(4, 7),
    'LUA^': range(7, 10),
    'RUA_': range(10, 13),
    'LH': range(13, 16),
    'BACK-A': range(16, 19),
    'RKN_': range(19, 22),
    'RWR': range(22, 25),
    'RUA^': range(25, 28),
    'LUA_': range(28, 31),
    'LWR': range(31, 34),
    'RH': range(34, 37),
}
imu_map = {
    'BACK-I': range(37, 46),
    'RUA': range(50, 59),
    'RLA': range(63, 72),
    'LUA': range(76, 85),
    'LLA': range(89, 98),
    'LSHOE': range(102, 118),
    'RSHOE': range(118, 134),
}

• 论文中使用了Opportunity数据集的两种识别任务locomotion和gesture，对应数据文件中的第244列和第250列。

label_maps = {'locomotion': 243, 'gesture': 249}  # label col number of OPP dataset

• 实验对象S1的全部数据，S2与S3的ADL1-3，Drill数据作为训练集，S2与S3的ADL4-5作为测试集。

train_file_opp = ['S1-Drill', 'S1-ADL1', 'S1-ADL2', 'S1-ADL3', 'S1-ADL4', 'S1-ADL5', 'S3-Drill', 'S3-ADL1', 'S3-ADL2', 'S3-ADL3', 'S2-Drill', 'S2-ADL1', 'S2-ADL2', 'S2-ADL3']
test_file_opp = ['S2-ADL4', 'S2-ADL5', 'S3-ADL4', 'S3-ADL5']

数据处理

在活动识别中的原始数据，是按时间排列的传感器数据，每个时刻对应一个活动标签。每个实验对象每天的数据，单独存放为一个数据文件。通过指定的文件名，需要用到的传感器标号，以及对应的活动识别任务，读取feature和label。

def load_feature_label_OPP(file_names, acc_sensors, imu_sensors, label_col):
    """
    choose column by the specific sensor lists, load the choosen cols in files and return data in format (feature, label).
    @param file_names: list, the file name list of data to be loaded;
    @param acc_sensors: list, the sensor list of accelerometers
    @param imu_sensors: list, the sensor list of IMU
    @return: tuple(ndarray, ndarray), feature and label
    """
    pathes = []

    for f in file_names:
        pathes.append(f+'.dat')

    feature, label = None, None
    used_col = []
    for sensor in acc_sensors:
        if sensor in acc_map:
            used_col.extend(acc_map[sensor])
    for sensor in imu_sensors:
        if sensor in imu_map:
            used_col.extend(imu_map[sensor])
    # count the number of columns used
    global cols_used
    cols_used = len(used_col)

    used_col.append(label_maps[label_col])  # label
    print('%d cols we used' % len(used_col))

    for p in pathes:
        path = os.path.join(base_path, p)
        temp = np.loadtxt(path, usecols=used_col, dtype=np.float32)
        f, l = temp[:, :-1], temp[:, -1].reshape(-1, 1)
        f = linear_interpolation(f)

        if label is not None:
            feature = np.vstack((feature, f))
            label = np.vstack((label, l))

        else:
            feature, label = f, l
    return feature, label

使用滑动窗口对数据进行处理，将一段时间内的所有传感器数据作为特征，最后一个时刻的活动类别作为标签。

def data_segment(data, window_size, stride_size):
    """
    divide the sequence into time window.
    @param data: the input sequence.
    @param window_size: the size of time window.
    @param stride_size: the size of stride step.
    @return: the time sequences in format of time window.
    """
    result = data[window_size-1::stride_size]
    length = result.shape[0]
    for i in range(window_size-2, -1, -1):
        _data = data[i::stride_size][:length]
        result = np.hstack((_data, result))
    return result


def get_label(label, window_size, stride_size):
    """
    get the last label in time window.
    """
    label = label[window_size-1:: stride_size]
    return label

在将数据输入到神经网络之前，对经过滑动窗口处理的数据进行标准化。在对时间窗口形式的输入数据标准化时，要保持对不同时刻的相同传感器信号进行相同的处理，避免破坏时间窗口内数据的时间依赖关系。

def normalization_window(train, test):
    """
    normalize the dataset by the mean value and std value after slide window. 
    Data format - (sample size, time axis, sensor signals, chunnel).
    @param train: ndarray, the mean and std value will be computed and used to normalize train and test set;
    @param test: ndarray, normalize by the mean and std values of train set.
    """
    if train.shape[1:] != test.shape[1:]:
        print("data shape error")
    else:
        for i in range(train.shape[2]):
            std = train[:, :, i, :].std()
            mean = train[:, :, i, :].mean()
            train[:, :, i, :] = (train[:, :, i, :] - mean) / std
            test[:, :, i, :] = (test[:, :, i, :] - mean) / std
    return train, test

网络结构设计

使用5*1的一维卷积核在时间轴上进行卷积，为了使用LSTM层，要在卷积之后调整数据的维度，从（sample size，time axis，sensor signal， chunnel）调整为（sample size，time axis，sensor signal* chunnel），time axis的值就作为LSTM的时间步，经过两个LSTM层之后，将学习到的特征展开为一维，使用softmax层获得活动类别。

def DeepLSTMConv(input_shape,):
    """
    structure of neural network.
    @param input_shape: tuple, in format (time axis, sensor signals, chunnel).
    """
    input_X = Input(shape=input_shape)
    x = Conv2D(padding="valid", kernel_size=(5, 1), filters=64, activation='relu')(input_X)
    x = Conv2D(padding="valid", kernel_size=(5, 1), filters=64, activation='relu')(x)
    x = Conv2D(padding="valid", kernel_size=(5, 1), filters=64, activation='relu')(x)
    x = Conv2D(padding="valid", kernel_size=(5, 1), filters=64, activation='relu')(x)
    # x = MaxPooling2D(pool_size=(3, 1))(x)
    x = Reshape((x.shape[1].value, -1))(x)
    x = LSTM(128 ,return_sequences=True, activation='relu')(x)
    x = LSTM(128, return_sequences=True, activation='relu')(x)
    x = Flatten()(x)
    x = Dense(class_num, activation='softmax')(x)

    ColConv = Model(inputs=input_X, outputs=x)
    ColConv.summary()
    ColConv.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=['accuracy'])
    return ColConv

运行模型并保存结果

def main(file_path, train_info, epochs=64):
    col_names = ['dataset', 'label col', 'info', 'epoch', 'window size', 'stride step', 'coverage rate', 'train accuracy', 
                 'train loss', 'test accuracy', 'test loss', 'test f1score', 'run second', 'sensor_id', 'f1scores', 
                 'precisions', 'recalls']
    cols = ','.join(col_names) + '\r\n'

    if os.path.exists(file_path) == False:
        f = open(file_path, 'w')
        f.write(cols)
        f.close()

    conv_shape = f_train.shape[1:]
    print(conv_shape)
    print('Train set: %d, Test set: %d' % (la_tr.shape[0], la_te.shape[0]))
    model = DeepLSTMConv(conv_shape)
    tr_acc, tr_loss, te_acc, te_loss, te_fscore, run_sec = [0.0] * 6

    run_sec = time_spare_test(model.fit, f_train, la_tr, batch_size=256, epochs=epochs, verbose=1)

    tr_loss, tr_acc = model.evaluate(f_train, la_tr, verbose=0)
    print('[%d] Train loss: %f' % (0, tr_loss))
    print('[%d] Train accuracy: %.2f%%' % (0, 100*tr_acc))
    te_loss, te_acc = model.evaluate(f_test, la_te, verbose=0)
    print('[%d] Test loss: %f' % (0, te_loss))
    print('[%d] Test accuracy: %.2f%%' % (0, 100*te_acc))
    pred = model.predict(f_test)
    te_fscore = get_f1_score(pred, la_te)
    print('[%d] Test f1-score: %.2f%%' % (0, 100 * te_fscore))
    print(get_confusion_matrix(pred, la_te))
    fscore_list, precision_list, recall_list = get_prf_mat(pred, la_te)

    tr_acc, tr_loss, te_acc, te_loss, te_fscore = ["%.4f" % i for i in [tr_acc, tr_loss, te_acc, te_loss, te_fscore]]
    run_sec = str(run_sec)
    sensors = 'all'
    if dataset == "OPP":
        sensors = '"%s"' % (';'.join(acc_sen+imu_sen))
    elif dataset == "skoda":
        sensors = '"%s"' % (';'.join(map(lambda x: str(x), skoda_sensors)))

    fscores = '"%s"' % (';'.join(["%.4f" % x for x in fscore_list]))
    pvalues = '"%s"' % (';'.join(["%.4f" % x for x in precision_list]))
    rvalues = '"%s"' % (';'.join(["%.4f" % x for x in recall_list]))
    result = '%s,'*7 % (dataset, opp_col, train_info, epochs, ws, ss, 1-ss/ws)
    result += ','.join([tr_acc, tr_loss, te_acc, te_loss, te_fscore, run_sec, sensors, fscores, pvalues, rvalues])
    result += '\r\n'
    f = open(file_path, 'a+')
    f.write(result)
    f.close()