pytorch DQN实战

Deep Q-Network实战

示例一立杆子

示例一源自莫烦pytorch教学4.5,
原文链接：https://www.bilibili.com/video/av15997678?p=27

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import gym

# Hyper Parameters
BATCH_SIZE = 32
LR = 0.01                   # learning rate
EPSILON = 0.9               # greedy policy
GAMMA = 0.9                 # reward discount
TARGET_REPLACE_ITER = 100   # target update frequency
MEMORY_CAPACITY = 2000

env = gym.make('CartPole-v0')
env = env.unwrapped

N_ACTIONS = env.action_space.n
N_STATES = env.observation_space.shape[0]

ENV_A_SHAPE = 0


class Net(nn.Module):
    """
    获得Q-learning值
    很简单的两层的网络
    这只是个网络，不会更新自己的参数，只有当optimizer更新之后才会有变化
    """
    def __init__(self, ):
        """
        两层神经网络，但是现在还不知道要不要这个初始化
        等以后学了在修改
        目前按照莫烦源码定义
        input: N_STATES
        output: N_ACTIONS_value
        """
        super(Net, self).__init__()
        self.fc1 = nn.Linear(N_STATES, 50)
        self.fc1.weight.data.normal_(0, 0.1)   # initialization
        self.out = nn.Linear(50, N_ACTIONS)
        self.out.weight.data.normal_(0, 0.1)   # initialization

    def forward(self, x):
        x = self.fc1(x)
        x = F.relu(x)
        actions_value = self.out(x)
        return actions_value


class DQN(object):
    def __init__(self):
        """
        两个网络，分别是
        eval_net
        target_net
        """
        self.eval_net, self.target_net = Net(), Net()
        self.learn_step_counter = 0
        # for target updating
        self.memory_counter = 0
        # for storing memory
        self.memory = np.zeros((MEMORY_CAPACITY, N_STATES * 2 + 2))     # initialize memory
        self.optimizer = torch.optim.Adam(self.eval_net.parameters(), lr=LR)
        # eval_net 每次都更新参数
        self.loss_func = nn.MSELoss()

    def choose_action(self, x):
        """
        :param x: state 环境
        :return: action
        """
        x = torch.unsqueeze(torch.FloatTensor(x), 0)
        # input only one sample
        if np.random.uniform() < EPSILON:   # greedy 根据value，当小于概率，就采取价值最高的动作
            actions_value = self.eval_net.forward(x)
            action = torch.max(actions_value, 1)[1].data.numpy()
            action = action[0] if ENV_A_SHAPE == 0 else action.reshape(ENV_A_SHAPE)  # return the argmax index
        else:   # random
            action = np.random.randint(0, N_ACTIONS)
            action = action if ENV_A_SHAPE == 0 else action.reshape(ENV_A_SHAPE)
        return action

    def store_transition(self, s, a, r, s_):
        """
        记忆库
        :param s:state
        :param a:action
        :param r:reward
        :param s_:next state
        :return:
        """
        transition = np.hstack((s, [a, r], s_))
        # replace the old memory with new memory
        index = self.memory_counter % MEMORY_CAPACITY
        self.memory[index, :] = transition
        self.memory_counter += 1

    def learn(self):
        # target parameter update
        if self.learn_step_counter % TARGET_REPLACE_ITER == 0:
            self.target_net.load_state_dict(self.eval_net.state_dict())
        self.learn_step_counter += 1

        # sample batch transitions
        sample_index = np.random.choice(MEMORY_CAPACITY, BATCH_SIZE)
        b_memory = self.memory[sample_index, :]

        b_s = torch.FloatTensor(b_memory[:, :N_STATES])
        b_a = torch.LongTensor(b_memory[:, N_STATES:N_STATES+1].astype(int))
        b_r = torch.FloatTensor(b_memory[:, N_STATES+1:N_STATES+2])
        b_s_ = torch.FloatTensor(b_memory[:, -N_STATES:])
        # 取出记忆库里的四个参数
        # q_eval w.r.t the action in experience
        q_eval = self.eval_net(b_s).gather(1, b_a)  # shape (batch, 1)
        q_next = self.target_net(b_s_).detach()     # detach from graph, don't backpropagate
        q_target = b_r + GAMMA * q_next.max(1)[0].view(BATCH_SIZE, 1)   # shape (batch, 1)
        loss = self.loss_func(q_eval, q_target)
        # loss 的计算方式为，从经验池里的现实值，从下一步动作得出的估计值（由当前reward加GAMMA*下一步reward）
        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()

dqn = DQN()
print('\nCollecting experience...')
for i_episode in range(400):
    s = env.reset()
    # 400次，每次都重新reset环境
    ep_r = 0
    while True:
        env.render()
        a = dqn.choose_action(s)

        # take action
        s_, r, done, info = env.step(a)

        # modify the reward
        x, x_dot, theta, theta_dot = s_
        r1 = (env.x_threshold - abs(x)) / env.x_threshold - 0.8
        r2 = (env.theta_threshold_radians - abs(theta)) / env.theta_threshold_radians - 0.5
        r = r1 + r2
        # 这里把reward改变了一下，要不然学习太慢

        dqn.store_transition(s, a, r, s_)

        ep_r += r
        if dqn.memory_counter > MEMORY_CAPACITY:
            dqn.learn()
            if done:
                print('Ep: ', i_episode,
                      '| Ep_r: ', round(ep_r, 2))
        if done:
            break
            #  杆倒了失败，从新开始
        s = s_

示例二爬坡实验：