【深入浅出强化学习-原理入门】1 基于gym的MDP

windows版本强化学习gym找金币游戏

• 第一步：grid_mdp.py代码展示

import logging #日志模块
import numpy
import random
from gym import spaces
import gym

logging = logging.getLogger(__name__)


# Set this in SOME subclasses
class GridEnv(gym.Env):
    metadata = {
        'render.modes':['human','rgb_array'],
        'video.frames_per_second':2
    }

    def _seed(self, seed=None):
        self.np_random, seed = seeding.np_random(seed)
        return [seed]

    def __init__(self):
        # 状态空间
        self.states = [1,2,3,4,5,6,7,8]
        # 设置机器人的位置，机器人的位置根据所处状态不同位置也不同。
        # 事先计算出每个状态点机器人位置的中心坐标，并存储到两个向量中，在类初始化中给出
        self.x = [140, 220, 300, 380, 460, 140, 300, 460]
        self.y = [250, 250, 250, 250, 250, 150, 150, 150]
        self.terminate_states = dict() #终止状态为字典格式
        self.terminate_states[6] = 1
        self.terminate_states[7] = 1
        self.terminate_states[8] = 1

        # 动作空间
        self.actions = ['n','e','s','w']

        # 回报函数
        self.rewards = dict(); #回报的数据结构为字典
        self.rewards['1_s'] = -1.0
        self.rewards['3_s'] = 1.0
        self.rewards['5_s'] = -1.0

        # 状态转移概率
        self.t = dict();
        self.t['1_s'] = 6
        self.t['1_e'] = 2
        self.t['2_w'] = 1
        self.t['2_e'] = 3
        self.t['3_s'] = 7
        self.t['3_w'] = 2
        self.t['3_e'] = 4
        self.t['4_w'] = 3
        self.t['4_e'] = 5
        self.t['5_w'] = 8
        self.t['5_w'] = 4

        #折扣因子
        self.gamma = 0.8
        self.viewer = None
        self.state = None

    def getTerminal(self):
        return self.terminate_states

    def getGamma(self):
        return self.gamma

    def getStates(self):
        return self.states

    def getAction(self):
        return self.actions

    def getTerminate_states(self):
        return self.terminate_states

    def setAction(self,s):
        self.state = s

    # step()函数输入是动作，
    # 输出是下一时刻的动作、回报、是否终止、调试信息
    # 没有的用{}表示
    def _step(self,action):
        # 系统当前状态
        state = self.state
        # 判断系统当前状态是否为终止状态
        if state in self.terminate_states:
            return state,0,True,{}
        #将状态与动作组成字典的键值
        key = "%d_%s"(state,action)

        #状态转移
        if key in self.t:
            next_state = self.t[key]
        else:
            next_state = state
        self.state = next_state

        is_terminal = False

        if next_state in self.terminate_states:
            is_terminal = True

        if key not in self.rewards:
            r = 0.0
        else:
            r = self.rewards[key]

        return next_state,r,is_terminal,{}

    # reset函数建立
    # reset常常用随机的方法初始机器人状态
    def _reset(self):
        self.state = self.states[int(random.random()*len(self.states))]
        return self.state


        # render函数建立
    def _render(self, mode='human', close=False):
        if close:
            if self.viewer is not None:
                self.viewer.close()
                self.viewer = None
            return
        screen_width = 600
        screen_height = 400

        if self.viewer is None:
            # 调用rendering的画图函数
            from gym.envs.classic_control import rendering
            self.viewer = rendering.Viewer(screen_width, screen_height)

            # 创建网格世界，一共11条直线
            self.line1 = rendering.Line((100, 300), (500, 300))
            self.line2 = rendering.Line((100, 200), (500, 200))
            self.line3 = rendering.Line((100, 300), (100, 100))
            self.line4 = rendering.Line((180, 300), (180, 100))
            self.line5 = rendering.Line((260, 300), (260, 100))
            self.line6 = rendering.Line((340, 300), (340, 100))
            self.line7 = rendering.Line((420, 300), (420, 100))
            self.line8 = rendering.Line((500, 300), (500, 100))
            self.line9 = rendering.Line((100, 100), (180, 100))
            self.line10 = rendering.Line((260, 100), (340, 100))
            self.line11 = rendering.Line((420, 100), (500, 100))

            # 接下来写死亡区域，黑色实心圆代表死亡区域
            # 创建第一个骷髅
            self.kulo1 = rendering.make_circle(40)
            self.circletrans = rendering.Transform(translation=(140, 150))  # 圆心坐标
            self.kulo1.add_attr(self.circletrans)
            self.kulo1.set_color(0, 0, 0)

            # 创建第二个骷髅
            self.kulo2 = rendering.make_circle(40)
            self.circletrans = rendering.Transform(translation=(460, 150))
            self.kulo2.add_attr(self.circletrans)
            self.kulo2.set_color(0, 0, 0)

            # 创建金币区域，用浅色的圆表示
            self.gold = rendering.make_circle(40)
            self.circletrans = rendering.Transform(translation=(300, 150))
            self.gold.add_attr(self.circletrans)
            self.gold.set_color(1, 0.9, 0)

            # 创建机器人，用不同颜色的圆表示
            self.robot = rendering.make_circle(30)
            self.robotrans = rendering.Transform()
            self.robot.add_attr(self.robotrans)
            self.robot.set_color(0.8, 0.6, 0.4)

            # 给11条直线设置颜色，并将这些创建的对象添加到几何中
            self.line1.set_color(0, 0, 0)
            self.line2.set_color(0, 0, 0)
            self.line3.set_color(0, 0, 0)
            self.line4.set_color(0, 0, 0)
            self.line5.set_color(0, 0, 0)
            self.line6.set_color(0, 0, 0)
            self.line7.set_color(0, 0, 0)
            self.line8.set_color(0, 0, 0)
            self.line9.set_color(0, 0, 0)
            self.line10.set_color(0, 0, 0)
            self.line11.set_color(0, 0, 0)

            self.viewer.add_geom(self.line1)
            self.viewer.add_geom(self.line2)
            self.viewer.add_geom(self.line3)
            self.viewer.add_geom(self.line4)
            self.viewer.add_geom(self.line5)
            self.viewer.add_geom(self.line6)
            self.viewer.add_geom(self.line7)
            self.viewer.add_geom(self.line8)
            self.viewer.add_geom(self.line9)
            self.viewer.add_geom(self.line10)
            self.viewer.add_geom(self.line11)
            self.viewer.add_geom(self.kulo1)
            self.viewer.add_geom(self.kulo2)
            self.viewer.add_geom(self.gold)
            self.viewer.add_geom(self.robot)


        # 根据这两个向量和机器人当前状态，就可以设置机器人当前的圆心坐标
        if self.state is None: return None
        self.robotrans.set_translation(self.x[self.state - 1], self.y[self.state - 1])


        return self.viewer.render(return_rgb_array=mode =='rgb_array')

• 第二步：将grid_mdp.py放到gym安装的目录处，如图
  
• 第三步：修改当前目录的_init_.py(我的位置是：F:\Deeplearning\Anaconda\envs\tensorflow\Lib\site-packages\gym\envs\classic_control)
  在文尾添加：

from gym.envs.classic_control.grid_mdp import GridEnv

• 第四步：修改上一目录的_init_.py(我的位置是：F:\Deeplearning\Anaconda\envs\tensorflow\Lib\site-packages\gym\envs)
  添加：

register(
    id='GridWorld-v0',
    entry_point = 'gym.envs.classic_control:GridEnv',
    max_episode_steps=200,
    reward_threshold=100.0,
)

• 第五步：打开Anaconda Prompt

(base) C:\Users\Willing>activate tensorflow

(tensorflow) C:\Users\Willing>python
Python 3.6.12 |Anaconda, Inc.| (default, Sep  9 2020, 00:29:25) [MSC v.1916 64 bit (AMD64)] on win32
Type "help", "copyright", "credits" or "license" for more information.
>>> import gym
>>> env=gym.make('GridWorld-v0')
F:\Deeplearning\Anaconda\envs\tensorflow\lib\site-packages\gym\logger.py:30: UserWarning: [33mWARN: Environment 'gym.envs.classic_control.grid_mdp.GridEnv'>' has deprecated methods '_step' and '_reset' rather than 'step' and 'reset'. Compatibility code invoked. Set _gym_disable_underscore_compat = True to disable this behavior.[0m
  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))
>>> env.reset()
7
>>> env.render()
True

注： tensorflow这里是我创建的虚拟环境名（安装gym的地方）

报错处理：

1. 之前一直报错在def reset(self, **kwargs): ，后来发现是因为reset()和render()函数的缩进出了问题
2. 报错AttributeError: 'GridEnv' object has no attribute '_seed'，解决方案在grid_mdp.py文件中添加

def _seed(self, seed=None):
        self.np_random, seed = seeding.np_random(seed)
        return [seed]

3. 报错在AttributeError: 'GridEnv' object has no attribute '_render'
   解决方案：将grid_mdp.py文件中的reset和step函数定义改为def _step => def step 这种形式就可以了。