【强化学习】Q-learning与SARSAS算法比较与SARSA算法实现
一、Q-learning与SARSA区别
Q-learning为offpolicy(通过之前的历史，也可以是别人的历史，学习者和决策者不一定相同)，target使用greedy，action用ε-greedy。行动策略和评估策略不是一个策略。
SARSA是on-policy的更新方式（边决策边学习，学习者也是决策者），它的行动策略和评估策略都是ε-greedy策略。与Q-learning相比更保守。
二、SARSA算法代码实现-test
from maze_env import Maze
from RL_brain import SarsaTable
def update():
for episode in range(100):
observation = env.reset()
action = RL.choose_action(str(observation))#基于观测值进行动作的选择
while True:
env.render()#更新环境
observation_, reward, done = env.step(action)#执行动作得到下一个观测值、获得的奖励以及是否结束
action_ = RL.choose_action(str(observation_))#下一个action就是下个回合的action
RL.learn(str(observation), action, reward, str(observation_), action_)#状态转移，考虑下一个action
observation = observation_
action = action_
if done:
break
# end of game
print('game over')
env.destroy()
if __name__ == "__main__":
env = Maze()
RL = SarsaTable(actions=list(range(env.n_actions)))
env.after(100, update)
env.mainloop()
三、SARSA算法代码实现-brain
import numpy as np
import pandas as pd
class RL(object):#这是一个父类，SARSA table可以继承他，分为三部分1.__init__的这个条件2.检查Q表中是否有state没有就补上3.选择动作，90%10%
def __init__(self, action_space, learning_rate=0.01, reward_decay=0.9, e_greedy=0.9):
self.actions = action_space # a list
self.lr = learning_rate
self.gamma = reward_decay
self.epsilon = e_greedy
self.q_table = pd.DataFrame(columns=self.actions, dtype=np.float64)
def check_state_exist(self, state):
if state not in self.q_table.index:
# append new state to q table
self.q_table = self.q_table.append(
pd.Series(
[0]*len(self.actions),
index=self.q_table.columns,
name=state,
)
)
def choose_action(self, observation):
self.check_state_exist(observation)
# 动作选择
if np.random.rand() < self.epsilon:
# 选择最好的动作
state_action = self.q_table.loc[observation, :]
action = np.random.choice(state_action[state_action == np.max(state_action)].index)
else:
# 随机选择动作
action = np.random.choice(self.actions)
return action
def learn(self, *args):#这部分Qlearning和SARSA是不同的，需要分开编写
pass
# off-policy
class QLearningTable(RL):
def __init__(self, actions, learning_rate=0.01, reward_decay=0.9, e_greedy=0.9):
super(QLearningTable, self).__init__(actions, learning_rate, reward_decay, e_greedy)
def learn(self, s, a, r, s_):
self.check_state_exist(s_)
q_predict = self.q_table.loc[s, a]
if s_ != 'terminal':
q_target = r + self.gamma * self.q_table.loc[s_, :].max()
else:
q_target = r
self.q_table.loc[s, a] += self.lr * (q_target - q_predict) # update
# on-policy
class SarsaTable(RL):
def __init__(self, actions, learning_rate=0.01, reward_decay=0.9, e_greedy=0.9):
super(SarsaTable, self).__init__(actions, learning_rate, reward_decay, e_greedy)#通过父类传入参数
def learn(self, s, a, r, s_, a_):#与Q学习相比，多了一个下一个action也要被学习
self.check_state_exist(s_)
q_predict = self.q_table.loc[s, a]
if s_ != 'terminal':
q_target = r + self.gamma * self.q_table.loc[s_, a_] # next state is not terminal，这一步不同于Q学习，这里不是最大的值，而是采取下一个行动的下一个值
else:
q_target = r # next state is terminal
self.q_table.loc[s, a] += self.lr * (q_target - q_predict) # update