paddle2.2.0：DQN算法训练cartpole游戏
DQN，基于Q-learning，结合了神经网络，不再使用Q表格来存储Q值，而是用神经网络拟合的方式，可以大大减少内存的占用，同时也更加省时。
DQN在Q-learning的基础上，又添加了经验池回放和固定Q网络两个新的技巧
LEARN_FREQ = 5
MEMORY_SIZE = 20000
MEMORY_WARMUP_SIZE = 200
BATCH_SIZE = 32
LEARNING_RATE = 0.001
GAMMA = 0.99
class Model(parl.Model):
def __init__(self, obs_dim, act_dim):
super().__init__()
self.fc1 = nn.Linear(obs_dim, 128)
self.fc2 = nn.Linear(128, 128)
self.fc3 = nn.Linear(128, act_dim)
def forward(self, x):
h1 = F.relu(self.fc1(x))
h2 = F.relu(self.fc2(h1))
Q = self.fc3(h2)
return Q
class Agent(parl.Agent):
def __init__(self, algthrim, act_dim, e_greed=None, e_greed_decrement=None):
super().__init__(algorithm)
self.e_greed = e_greed
self.e_greed_decrement = e_greed_decrement
self.act_dim = act_dim
self.global_steps = 0
self.update_target_steps = 200
def sample(self, obs):
sample1 = np.random.random()
if sample1 > self.e_greed:
act = self.predict(obs)
else:
act = np.random.randint(self.act_dim)
self.e_greed = max(0.01, self.e_greed - self.e_greed_decrement)
return act
def predict(self, obs):
obs = paddle.to_tensor(obs, dtype='float32')
predQ = self.alg.predict(obs)
act = paddle.argmax(predQ).numpy()[0]
return act
def learn(self, obs, act, reward, next_obs, terminal):
if self.global_steps % self.update_target_steps == 0:
self.alg.sync_target()
self.global_steps += 1
act = np.expand_dims(act, axis=-1)
reward = np.expand_dims(act, axis=-1)
terminal = np.expand_dims(terminal, axis=-1)
obs = paddle.to_tensor(obs, dtype='float32')
act = paddle.to_tensor(act, dtype='int32')
reward = paddle.to_tensor(reward, dtype='float32')
next_obs = paddle.to_tensor(next_obs, dtype='float32')
terminal = paddle.to_tensor(terminal, dtype='float32')
loss = self.alg.learn(obs, act, reward, next_obs, terminal)
return loss.numpy()[0]
class DQN(parl.Algorithm):
def __init__(self, model, gamma=None, lr=None):
self.gamma = gamma
self.model = model
self.target_model = copy.deepcopy(model)
self.optimizer = paddle.optimizer.Adam(learning_rate=lr, parameters=model.parameters())
self.mse_loss = paddle.nn.MSELoss(reduction='mean')
def predict(self, obs):
return self.model(obs)
def learn(self, obs, act, reward, next_obs, terminal):
pred_value = self.model(obs)
act_dim = pred_value.shape[-1]
act = paddle.squeeze(act, axis=-1)
act = F.one_hot(act, num_classes=act_dim)
pred_value = paddle.multiply(pred_value, act)
pred_value = paddle.sum(pred_value, axis=1, keepdim=True)
with paddle.no_grad():
max_v = self.target_model(next_obs).max(1, keepdim=True)
target = reward + (1 - terminal) * self.gamma * max_v
loss = self.mse_loss(pred_value, target)
self.optimizer.clear_grad()
loss.backward()
self.optimizer.step()
return loss
def sync_target(self):
self.model.sync_weights_to(self.target_model)
class ReplayMemory(object):
def __init__(self, max_size):
self.buffer1 = collections.deque(maxlen=max_size)
def append(self, exp):
self.buffer1.append(exp)
def sample(self, batch_size):
mini_batch = random.sample(self.buffer1, batch_size)
obs_batch, action_batch, reward_batch, next_obs_batch, done_batch = [], [], [], [], []
for experience in mini_batch:
s, a, r, s_p, done = experience
obs_batch.append(s)
action_batch.append(a)
reward_batch.append(r)
next_obs_batch.append(s_p)
done_batch.append(done)
return np.array(obs_batch, dtype='float32'), \
np.array(action_batch, dtype='float32'),\
np.array(reward_batch, dtype='float32'),\
np.array(next_obs_batch, dtype='float32'),\
np.array(done_batch, dtype='float32')
def __len__(self):
return len(self.buffer1)
def run_episode(env, agent, rpm):
total_reward = 0
obs = env.reset()
step = 0
while True:
step += 1
action = agent.sample(obs)
next_obs, reward, done, _ = env.step(action)
rpm.append((obs, action, reward, next_obs, done))
if(len(rpm) > MEMORY_WARMUP_SIZE) and (step % LEARN_FREQ == 0):
(batch_obs, batch_action, batch_reward, batch_next_obs, batch_done) = rpm.sample(BATCH_SIZE)
train_loss = agent.learn(batch_obs, batch_action, batch_reward, batch_next_obs, batch_done)
total_reward += reward
obs = next_obs
if done:
break
return total_reward
def evaluate(env, agent, render=False):
eval_reward = []
for i in range(5):
obs = env.reset()
episode_reward = 0
while True:
action = agent.predict(obs)
obs, reward, done, _ = env.step(action)
episode_reward += reward
if render:
env.render()
# img.set_data(env.render(mode='rgb_array'))
# display.display(plt.gcf())
# display.clear_output(wait=True)
if done:
break
eval_reward.append(episode_reward)
return np.mean(eval_reward)
env = gym.make('CartPole-v0')
# img = plt.imshow(env.render(mode='rgb_array'))
action_dim = env.action_space.n
obs_shape = env.observation_space.shape
rpm = ReplayMemory(MEMORY_SIZE)
model = Model(obs_shape[0], action_dim)
algorithm = DQN(model, gamma=GAMMA, lr=LEARNING_RATE)
agent = Agent(
algorithm,
act_dim=action_dim,
e_greed=0.1,
e_greed_decrement=1e-6
)
while len(rpm) < MEMORY_WARMUP_SIZE:
run_episode(env, agent, rpm)
max_episode = 2000
episode = 0
while episode < max_episode:
for i in range(0, 50):
total_reward = run_episode(env, agent, rpm)
episode += 1
eval_reward = evaluate(env, agent, render=False)
logger.info('episode{}, e_greed:{}, test_reward:{}'.format(episode, agent.e_greed, eval_reward))
save_path = './dqn_model.ckpt'
agent.save(save_path)输出：
训练完后，小车就可以愉快的冲上山坡了
[11-29 23:19:36 MainThread @machine_info.py:88] nvidia-smi -L found gpu count: 1
[11-29 23:19:37 MainThread @3050295955.py:180] episode50, e_greed:0.09900199999999901, test_reward:9.8
[11-29 23:19:38 MainThread @3050295955.py:180] episode100, e_greed:0.09847399999999848, test_reward:9.2
[11-29 23:19:38 MainThread @3050295955.py:180] episode150, e_greed:0.09797599999999798, test_reward:9.0
[11-29 23:19:39 MainThread @3050295955.py:180] episode200, e_greed:0.09748699999999749, test_reward:9.4
[11-29 23:19:39 MainThread @3050295955.py:180] episode250, e_greed:0.09692799999999693, test_reward:10.8
[11-29 23:19:40 MainThread @3050295955.py:180] episode300, e_greed:0.09634299999999635, test_reward:10.0
[11-29 23:19:41 MainThread @3050295955.py:180] episode350, e_greed:0.09570499999999571, test_reward:46.0
[11-29 23:19:43 MainThread @3050295955.py:180] episode400, e_greed:0.09323999999999324, test_reward:10.6
[11-29 23:19:53 MainThread @3050295955.py:180] episode450, e_greed:0.08461999999998462, test_reward:200.0
[11-29 23:20:04 MainThread @3050295955.py:180] episode500, e_greed:0.07530999999997531, test_reward:197.8
[11-29 23:20:15 MainThread @3050295955.py:180] episode550, e_greed:0.06622799999996623, test_reward:145.4
[11-29 23:20:26 MainThread @3050295955.py:180] episode600, e_greed:0.05703799999995704, test_reward:135.2
[11-29 23:20:36 MainThread @3050295955.py:180] episode650, e_greed:0.048734999999948736, test_reward:155.6
[11-29 23:20:47 MainThread @3050295955.py:180] episode700, e_greed:0.0401009999999401, test_reward:147.6
[11-29 23:20:56 MainThread @3050295955.py:180] episode750, e_greed:0.03288899999993289, test_reward:200.0
[11-29 23:21:05 MainThread @3050295955.py:180] episode800, e_greed:0.025805999999925805, test_reward:180.0
[11-29 23:21:14 MainThread @3050295955.py:180] episode850, e_greed:0.018424999999918423, test_reward:121.6
[11-29 23:21:24 MainThread @3050295955.py:180] episode900, e_greed:0.01, test_reward:135.0
[11-29 23:21:35 MainThread @3050295955.py:180] episode950, e_greed:0.01, test_reward:162.2
[11-29 23:21:46 MainThread @3050295955.py:180] episode1000, e_greed:0.01, test_reward:158.2
[11-29 23:21:55 MainThread @3050295955.py:180] episode1050, e_greed:0.01, test_reward:200.0
[11-29 23:22:07 MainThread @3050295955.py:180] episode1100, e_greed:0.01, test_reward:200.0
[11-29 23:22:19 MainThread @3050295955.py:180] episode1150, e_greed:0.01, test_reward:117.0
[11-29 23:22:31 MainThread @3050295955.py:180] episode1200, e_greed:0.01, test_reward:200.0
[11-29 23:22:43 MainThread @3050295955.py:180] episode1250, e_greed:0.01, test_reward:200.0
[11-29 23:22:55 MainThread @3050295955.py:180] episode1300, e_greed:0.01, test_reward:112.8
[11-29 23:23:07 MainThread @3050295955.py:180] episode1350, e_greed:0.01, test_reward:200.0
[11-29 23:23:19 MainThread @3050295955.py:180] episode1400, e_greed:0.01, test_reward:200.0
[11-29 23:23:31 MainThread @3050295955.py:180] episode1450, e_greed:0.01, test_reward:200.0
[11-29 23:23:43 MainThread @3050295955.py:180] episode1500, e_greed:0.01, test_reward:200.0
[11-29 23:23:53 MainThread @3050295955.py:180] episode1550, e_greed:0.01, test_reward:200.0
[11-29 23:24:04 MainThread @3050295955.py:180] episode1600, e_greed:0.01, test_reward:155.6
[11-29 23:24:15 MainThread @3050295955.py:180] episode1650, e_greed:0.01, test_reward:102.6
[11-29 23:24:23 MainThread @3050295955.py:180] episode1700, e_greed:0.01, test_reward:121.6
[11-29 23:24:34 MainThread @3050295955.py:180] episode1750, e_greed:0.01, test_reward:120.6
[11-29 23:24:44 MainThread @3050295955.py:180] episode1800, e_greed:0.01, test_reward:200.0
[11-29 23:24:54 MainThread @3050295955.py:180] episode1850, e_greed:0.01, test_reward:116.4
[11-29 23:25:07 MainThread @3050295955.py:180] episode1900, e_greed:0.01, test_reward:200.0
[11-29 23:25:18 MainThread @3050295955.py:180] episode1950, e_greed:0.01, test_reward:200.0
[11-29 23:25:31 MainThread @3050295955.py:180] episode2000, e_greed:0.01, test_reward:200.0