强化学习:A2C求解倒立摆问题代码
1.问题背景
倒立摆问题的问题背景就不再赘述了，在实现过程中用到了python的gym库。导入该环境的过程代码如下:
# 倒立摆网络
env = gym.make("CartPole-v0")
env.reset()
print("env_state:{}".format(env.state))
print("env_step(0):{}".format(env.step(0)))
在此之前需要导入的库为:
import numpy as np
import torch
# 导入torch的各种模块
import torch.nn as nn
from torch.nn import functional as F
from torch.distributions import Categorical
import gym
import warnings
warnings.filterwarnings('ignore')
import random
from collections import deque
import matplotlib.pyplot as plt
2.A2C训练原理
class policyNet(nn.Module):
def __init__(self,state_num,action_num,hidden_num=40,lr=0.01):
super(policyNet,self).__init__()
self.fc1 = nn.Linear(state_num,hidden_num)
self.action_fc = nn.Linear(hidden_num,action_num)
self.state_fc = nn.Linear(hidden_num,1)
self.optimzer = torch.optim.Adam(self.parameters(),lr=lr)
# # 动作与回报的缓存
# self.saved_states = []
# self.saved_log_pi = []
# self.saved_values = []
# self.saved_actions = []
# self.saved_rewards = []
# 前馈
def forward(self,state):
hidden_output = self.fc1(state)
x = F.relu(hidden_output)
action_prob = self.action_fc(x)
action_prob = F.softmax(action_prob,dim=-1)
state_values = self.state_fc(x)
return action_prob,state_values
# 选择动作
def select_action(self,state):
# 输入state为numpy数据类型，要先进行数据转化
# 输出lnpi(a|s),v(s),r
state = torch.from_numpy(state).float()
probs,state_value = self.forward(state)
m = Categorical(probs)
action = m.sample()
# 存储数据
log_pi_as = m.log_prob(action)
# self.saved_states.append(state)
# self.saved_log_pi.append(log_pi_as)
# self.saved_values.append(state_value)
# self.saved_actions.append(action)
# 返回action
return action.item(),log_pi_as
# 对每个回合执行以下动作
# 更新策略
def update_policy(self,env,state,gamma=0.9):
# 更新策略
# state:numpy类型数据
# next_state:numpy类型数据
action,ln_pi_a_s = self.select_action(state)
next_state,reward,done,_ = env.step(action)
# self.saved_rewards.append(reward)
_,next_state_values = self.forward(torch.from_numpy(next_state).float())
_,state_values = self.forward(torch.from_numpy(state).float())
U = reward + gamma*next_state_values*(1 - done)
loss_theta = -(U - state_values)*ln_pi_a_s
loss_w = 0.5*(U - state_values)**2
self.optimzer.zero_grad()
loss = loss_theta + loss_w
loss.backward()
self.optimzer.step()
# # 更新完后重置缓存
# del self.saved_actions[:]
# del self.saved_log_pi[:]
# del self.saved_rewards[:]
# del self.saved_states[:]
# del self.saved_values[:]
return next_state,reward,done
def train_network(self,env,epsiodes=100,gamma=0.9):
epsiode_array = []
mean_array = []
for epsiode in range(epsiodes):
epsiode_reward = 0.0
state = env.reset()
while True:
next_state,reward,done = self.update_policy(env,state,gamma=gamma)
epsiode_reward += reward
if done:
break
state = next_state
epsiode_array.append(epsiode_reward)
mean_array.append(np.mean(epsiode_array))
print("第{}次迭代的奖励为{},平均奖励为{}".format(epsiode,epsiode_reward,int(mean_array[-1])))
return epsiode_array,mean_array
3.主函数训练网络
policynet = policyNet(state_num=4,action_num=2)
epsiode_array,mean_array = policynet.train_network(env,epsiodes=3000,gamma=0.9)
plt.plot(epsiode_array)
plt.plot(mean_array)
结果为:
给出epsiodes=3000时的结果，可以看到训练到2000多次时网络在每个epsiode已经能达到最高的奖励效果:
作为对比,初期奖励如下:
收敛曲线如下:
