自动驾驶项目——方向盘角度和油门预测
自动驾驶项目——方向盘角度和油门预测
1、数据生成
数据来源是unity公司的模拟器。
2、数据增强
由于数据的不均匀性，有的角度多半是0，因此对数据进行删除操作，同时应用数据增强，水平翻转，图像剪切，亮度调整以及随机打乱顺序。
数据增强的代码：
def horizontal_flip(img, degree):
'''
按照50%的概率水平翻转图像
img: 输入图像
degree: 输入图像对于的转动角度
'''
choice = np.random.choice([0, 1])
if choice == 1:
img, degree = cv2.flip(img, 1), -degree
return (img, degree)
def random_brightness(img, degree):
'''
随机调整输入图像的亮度， 调整强度于 0.1(变黑)和1(无变化)之间
img: 输入图像
degree: 输入图像对于的转动角度
'''
hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)
# 调整亮度V: alpha * V
alpha = np.random.uniform(low=0.1, high=1.0, size=None)
v = hsv[:, :, 2]
v = v * alpha
hsv[:, :, 2] = v.astype('uint8')
rgb = cv2.cvtColor(hsv.astype('uint8'), cv2.COLOR_HSV2RGB)
return (rgb, degree)
def left_right_random_swap(img_address, degree, degree_corr=1.0 / 4):
'''
随机从左，中，右图像中选择一张图像， 并相应调整转动的角度
img_address: 中间图像的的文件路径
degree: 中间图像对于的方向盘转动角度
degree_corr: 方向盘转动角度调整的值
'''
swap = np.random.choice(['L', 'R', 'C'])
if swap == 'L':
img_address = img_address.replace('center', 'left')
corrected_label = np.arctan(np.tan(degree) + degree_corr)
return (img_address, corrected_label)
elif swap == 'R':
img_address = img_address.replace('center', 'right')
corrected_label = np.arctan(np.tan(degree) - degree_corr)
return (img_address, corrected_label)
else:
return (img_address, degree)
def discard_zero_steering(degrees, rate):
'''
从角度为零的index中随机选择部分index返回
degrees: 输入的角度值
rate: 丢弃率， 如果rate=0.8， 意味着 80% 的index会被返回，用于丢弃
'''
steering_zero_idx = np.where(degrees == 0)
steering_zero_idx = steering_zero_idx[0]
size_del = int(len(steering_zero_idx) * rate)
return np.random.choice(steering_zero_idx, size=size_del, replace=False)3、深度网络模型的设计
网络设计思路来源于yolo模型，基于keras实现，两个输出的设计。
因为方向盘角度有正也有负，故方向盘的输出激活函数是‘linear’，而油门throttle在行驶过程中多半是加速，所以采用的是变形的relu激活函数‘LeakyReLU(alpha=0.05)’。然后根据方向盘与油门在行驶时的重要性，设置他们的误差权重分别为1.0，0.8。
def get_model(shape):
'''
预测方向盘角度: 以图像为输入, 预测方向盘的转动角度和油门throttle
shape: 输入图像的尺寸, 例如(128, 128, 3)
'''
input_1 = Input(shape, name='input_1')
# input_2 = Input(shape, name='input_2')
conv_1 = Conv2D(16, (3, 3), strides=(1, 1), padding="same", activation='relu')(input_1)
maxpool_1 = MaxPooling2D(pool_size=(2, 2))(conv_1)
conv_2 = Conv2D(32, (3, 3), strides=(1, 1), padding="same", activation='relu')(maxpool_1)
maxpool_2 = MaxPooling2D(pool_size=(2, 2))(conv_2)
conv_3 = Conv2D(64, (3, 3), strides=(1, 1), padding="same", activation='relu')(maxpool_2)
maxpool_3 = MaxPooling2D(pool_size=(2, 2))(conv_3)
conv_4 = Conv2D(128, (3, 3), strides=(1, 1), padding="same", activation='relu')(maxpool_3)
maxpool_4 = MaxPooling2D(pool_size=(2, 2))(conv_4)
conv_5 = Conv2D(256, (3, 3), strides=(1, 1), padding="same", activation='relu')(maxpool_4)
maxpool_5 = MaxPooling2D(pool_size=(2, 2))(conv_5)
conv_6 = Conv2D(512, (3, 3), strides=(1, 1), padding="same", activation='relu')(maxpool_5)
maxpool_6 = MaxPooling2D(pool_size=(2, 2))(conv_6)
conv_7 = Conv2D(1024, (3, 3), strides=(1, 1), padding="same", activation='relu')(maxpool_6)
maxpool_7 = MaxPooling2D(pool_size=(2, 2))(conv_7)
flat = Flatten()(maxpool_7)
dense_1 = Dense(256, activation='relu')(flat)
dense_2 = Dense(512, activation='relu')(dense_1)
dense_3 = Dense(10, activation='relu')(dense_2)
output_1 = Dense(1, activation='linear', name='output_1')(dense_3)
output_2 = Dense(1, name='output_2')(dense_3)
output_2_ = LeakyReLU(alpha=0.05)(output_2)
model = Model(inputs=[input_1], outputs=[output_1, output_2_])
sgd = Adam(lr=0.0001)
# model.compile(optimizer=sgd, loss={'output_1':'mean_squared_error', 'output_2':'mean_squared_error'},
# loss_weights={'output_1': 1., 'output_2': 0.8})
model.compile(optimizer=sgd, loss=['mean_squared_error', 'mean_squared_error'],
loss_weights=[1.0, 0.8])
return model4、整体代码：
# -*- coding: utf-8 -*-
# @Time : 2019/5/24 8:10
# @Author : Chaucer_Gxm
# @Email : [email protected]
# @File : multi_output_train.py
# @GitHub : https://github.com/Chaucergit/Code-and-Algorithm
# @blog : https://blog.csdn.net/qq_24819773
# @Software: PyCharm
# -*- coding: utf-8 -*-
import numpy as np
from keras.optimizers import SGD, Adam
from keras.layers.core import Dense, Dropout, Activation
from keras.layers import Conv2D, MaxPooling2D, Flatten, PReLU
from keras.layers.advanced_activations import LeakyReLU
from keras.models import Sequential, Model
from keras import backend as K
from keras.regularizers import l2
import os.path
from keras import Input, Model
from keras.utils import plot_model
import csv
import cv2
import glob
import pickle
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
import json
from keras import callbacks
import math
from matplotlib import pyplot
from keras.utils.vis_utils import plot_model
import os
os.environ["PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin/'
SEED = 13
def horizontal_flip(img, degree):
'''
按照50%的概率水平翻转图像
img: 输入图像
degree: 输入图像对于的转动角度
'''
choice = np.random.choice([0, 1])
if choice == 1:
img, degree = cv2.flip(img, 1), -degree
return (img, degree)
def random_brightness(img, degree):
'''
随机调整输入图像的亮度， 调整强度于 0.1(变黑)和1(无变化)之间
img: 输入图像
degree: 输入图像对于的转动角度
'''
hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)
# 调整亮度V: alpha * V
alpha = np.random.uniform(low=0.1, high=1.0, size=None)
v = hsv[:, :, 2]
v = v * alpha
hsv[:, :, 2] = v.astype('uint8')
rgb = cv2.cvtColor(hsv.astype('uint8'), cv2.COLOR_HSV2RGB)
return (rgb, degree)
def left_right_random_swap(img_address, degree, degree_corr=1.0 / 4):
'''
随机从左，中，右图像中选择一张图像， 并相应调整转动的角度
img_address: 中间图像的的文件路径
degree: 中间图像对于的方向盘转动角度
degree_corr: 方向盘转动角度调整的值
'''
swap = np.random.choice(['L', 'R', 'C'])
if swap == 'L':
img_address = img_address.replace('center', 'left')
corrected_label = np.arctan(np.tan(degree) + degree_corr)
return (img_address, corrected_label)
elif swap == 'R':
img_address = img_address.replace('center', 'right')
corrected_label = np.arctan(np.tan(degree) - degree_corr)
return (img_address, corrected_label)
else:
return (img_address, degree)
def discard_zero_steering(degrees, rate):
'''
从角度为零的index中随机选择部分index返回
degrees: 输入的角度值
rate: 丢弃率， 如果rate=0.8， 意味着 80% 的index会被返回，用于丢弃
'''
steering_zero_idx = np.where(degrees == 0)
steering_zero_idx = steering_zero_idx[0]
size_del = int(len(steering_zero_idx) * rate)
return np.random.choice(steering_zero_idx, size=size_del, replace=False)
def get_model(shape):
'''
预测方向盘角度: 以图像为输入, 预测方向盘的转动角度和油门throttle
shape: 输入图像的尺寸, 例如(128, 128, 3)
'''
input_1 = Input(shape, name='input_1')
# input_2 = Input(shape, name='input_2')
conv_1 = Conv2D(16, (3, 3), strides=(1, 1), padding="same", activation='relu')(input_1)
maxpool_1 = MaxPooling2D(pool_size=(2, 2))(conv_1)
conv_2 = Conv2D(32, (3, 3), strides=(1, 1), padding="same", activation='relu')(maxpool_1)
maxpool_2 = MaxPooling2D(pool_size=(2, 2))(conv_2)
conv_3 = Conv2D(64, (3, 3), strides=(1, 1), padding="same", activation='relu')(maxpool_2)
maxpool_3 = MaxPooling2D(pool_size=(2, 2))(conv_3)
conv_4 = Conv2D(128, (3, 3), strides=(1, 1), padding="same", activation='relu')(maxpool_3)
maxpool_4 = MaxPooling2D(pool_size=(2, 2))(conv_4)
conv_5 = Conv2D(256, (3, 3), strides=(1, 1), padding="same", activation='relu')(maxpool_4)
maxpool_5 = MaxPooling2D(pool_size=(2, 2))(conv_5)
conv_6 = Conv2D(512, (3, 3), strides=(1, 1), padding="same", activation='relu')(maxpool_5)
maxpool_6 = MaxPooling2D(pool_size=(2, 2))(conv_6)
conv_7 = Conv2D(1024, (3, 3), strides=(1, 1), padding="same", activation='relu')(maxpool_6)
maxpool_7 = MaxPooling2D(pool_size=(2, 2))(conv_7)
flat = Flatten()(maxpool_7)
dense_1 = Dense(256, activation='relu')(flat)
dense_2 = Dense(512, activation='relu')(dense_1)
dense_3 = Dense(10, activation='relu')(dense_2)
output_1 = Dense(1, activation='linear', name='output_1')(dense_3)
output_2 = Dense(1, name='output_2')(dense_3)
output_2_ = LeakyReLU(alpha=0.05)(output_2)
model = Model(inputs=[input_1], outputs=[output_1, output_2_])
sgd = Adam(lr=0.0001)
# model.compile(optimizer=sgd, loss={'output_1':'mean_squared_error', 'output_2':'mean_squared_error'},
# loss_weights={'output_1': 1., 'output_2': 0.8})
model.compile(optimizer=sgd, loss=['mean_squared_error', 'mean_squared_error'],
loss_weights=[1.0, 0.8])
return model
# 图像数据增强
def image_transformation(img_address, degree, data_dir):
img_address, degree = left_right_random_swap(img_address, degree) # 图像的左右翻转
img = cv2.imread(data_dir + img_address)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img, degree = random_brightness(img, degree) # 图像亮度调整
img, degree = horizontal_flip(img, degree) # 图像水平翻转
return (img, degree)
def generator_data(img, degree, throttle, batch_size, shape, data_dir='data/',discard_rate=0.65):
y_bag = []
z_bag = []
x, y, z = shuffle(img, degree, throttle)
rand_zero_idx = discard_zero_steering(y, rate=discard_rate)
new_x = np.delete(x, rand_zero_idx, axis=0)
new_y = np.delete(y, rand_zero_idx, axis=0)
new_z = np.delete(z, rand_zero_idx, axis=0)
offset = 0
while True:
X = np.empty((batch_size, *shape))
Y = np.empty((batch_size, 1))
Z = np.empty((batch_size, 1))
for example in range(batch_size):
img_address, img_steering, img_throttle = new_x[example + offset], new_y[example + offset], new_z[example + offset]
img, img_steering = image_transformation(img_address, img_steering, data_dir)
X[example, :, :, :] = cv2.resize(img[80:140, 0:320], (shape[0], shape[1])) / 255 - 0.5
Y[example] = img_steering
Z[example] = img_throttle
y_bag.append(img_steering)
z_bag.append(img_throttle)
'''
到达原来数据的结尾, 从头开始
'''
if (example + 1) + offset > len(new_y) - 1:
x, y, z = shuffle(x, y, z)
rand_zero_idx = discard_zero_steering(y, rate=discard_rate)
new_x = x
new_y = y
new_z = z
new_x = np.delete(new_x, rand_zero_idx, axis=0)
new_y = np.delete(new_y, rand_zero_idx, axis=0)
new_z = np.delete(new_z, rand_zero_idx, axis=0)
offset = 0
yield (X, [Y, Z])
offset = offset + batch_size
np.save('y_bag.npy', np.array(y_bag))
np.save('z_bag.npy', np.array(z_bag))
np.save('Xbatch_sample.npy', X)
if __name__ == "__main__":
data_path = 'data/'
with open(data_path + 'driving_log.csv', 'r') as csvfile:
file_reader = csv.reader(csvfile, delimiter=',')
log = []
for row in file_reader:
log.append(row)
log = np.array(log)
# 去掉文件第一行
log = log[0:, :]
# 判断图像文件数量是否等于csv日志文件中记录的数量
ls_imgs = glob.glob(data_path + 'IMG/*.jpg')
assert len(ls_imgs) == len(log) * 3, 'number of images does not match'
# 使用20%的数据作为测试数据
validation_ratio = 0.2
shape = (128, 128, 3)
batch_size = 32
nb_epoch = 200
x_ = log[:, 0]
y_ = log[:, 3].astype(float)
z_ = log[:, 4].astype(float)
print(x_.shape)
print(y_.shape)
print(z_.shape)
x_, y_, z_ = shuffle(x_, y_, z_)
X_train, X_val, y_train, y_val, z_train, z_val = train_test_split(x_, y_, z_, test_size=validation_ratio, random_state=SEED)
print('batch size: {}'.format(batch_size))
print('Train set size: {} | Validation set size: {}'.format(len(X_train), len(X_val)))
samples_per_epoch = batch_size
# 使得validation数据量大小为batch_size的整数陪
nb_val_samples = len(y_val) - len(y_val) % batch_size
model = get_model(shape)
print(model.summary())
# 根据validation loss保存最优模型
save_best = callbacks.ModelCheckpoint('best_model_self.h5', monitor='val_loss', verbose=1,
save_best_only=True, mode='min')
# 如果训练持续没有validation loss的提升, 提前结束训练
early_stop = callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=50,
verbose=0, mode='auto')
callbacks_list = [early_stop, save_best]
history = model.fit_generator(generator_data(X_train, y_train, z_train, batch_size, shape),
steps_per_epoch=samples_per_epoch,
validation_steps=nb_val_samples // batch_size,
validation_data=generator_data(X_val, y_val, z_val, batch_size, shape),
epochs=nb_epoch, verbose=1, callbacks=callbacks_list)
with open('./trainHistoryDict_self.p', 'wb') as file_pi:
pickle.dump(history.history, file_pi)
# pyplot.plot(history.history['loss'])
# pyplot.plot(history.history['val_loss'])
# pyplot.title('model train vs validation loss')
# pyplot.ylabel('loss')
# pyplot.xlabel('epoch')
# pyplot.legend(['train', 'validation'], loc='upper right')
# pyplot.savefig('train_val_self.jpg')
# 保存模型
with open('model_self.json', 'w') as f:
f.write(model.to_json())
model.save('model_self.h5')
print('Done!')
训练过程：
待续。。。。。