强化学习-cs294-hw2-demo

本来在做Berkeley的cs294的hw2，但是由于对gym环境，比如observation和action的数据形式，和对TensorFlow的不熟悉，所以针对gym的CartPole-v0环境做一个演员-评论家的demo。这样可以清楚的观察到所做的努力，在没有加入优势函数的时候，网络训练的效果较差，几乎没有，加入后有较大的改善，但是还是没有破百，加入后破百了但是没有上两百，在正则化之后分数可以随意飙高。
明确如下：

1. CartPole-v0的环境下，目标是向一个方向连续滑动且保持杆子的平衡
2. 对优势函数做正则化可以有效防止过拟合，使得最后获得的reward分数更高
3. 该环境的数据是离散的

#!/usr/bin/env python
# -*- coding: utf8 -*-

import gym
import numpy as np
import tensorflow as tf
import cPickle as pickle
import matplotlib.pyplot as plt
import math


def buildNet(input_layer, output_shape=[None,2], scope='test', layer_size=4, size=10, output_activation=None):
    layer = input_layer
    with tf.variable_scope(scope):
        for i in range(0, layer_size):
            layer = tf.layers.dense(layer, size, activation=tf.tanh)
        output_layer = tf.layers.dense(layer, output_shape, activation=output_activation, name="ac_logits")
        return output_layer



env = gym.make('CartPole-v0')
discrete = isinstance(env.action_space, gym.spaces.Discrete)
obs_dim = env.observation_space.shape[0]
ac_dim = env.action_space.n if discrete else env.action_space.shape[0]

input_layer = tf.placeholder(tf.float32, [None, obs_dim], name="observation")
ac_logits = buildNet(input_layer, ac_dim, 'ac_test')
ac = tf.placeholder(tf.int32, [None], name="action")
ac_sample = tf.reshape(tf.multinomial(ac_logits,1),[-1]) #multinomial->从多项式分布中抽取样本，抽取的样本数是1;[-1]表示shape是缺省值，这样可以从中选出该选择哪类动作
logprob = -tf.nn.sparse_softmax_cross_entropy_with_logits(labels=ac,logits=ac_logits) #把概率分布转换成softmax形式，则如此所有概率之和为1

baseline = tf.squeeze(buildNet(input_layer, 1, 'baseline'))
base_target = tf.placeholder(tf.float32, [None], name='base_target')
base_loss = tf.nn.l2_loss(baseline-base_target) #对baseline的损失函数只要是普通的l2范数即可
base_train = tf.train.AdamOptimizer(0.001).minimize(base_loss)

adv = tf.placeholder(tf.float32, [None], name='adv')

loss = tf.reduce_mean(-logprob*adv)
train = tf.train.AdamOptimizer(0.001).minimize(loss)



# buildNet()
step = 200

i = 0
batch_size = 50
min_timesteps_per_batch = 500
gamma = 0.99

init = tf.global_variables_initializer()

with tf.Session() as sess:
    sess.run(init)
    # print(sess.run(ac_logits, feed_dict={input_layer: ob[None]})[0])

    for i in range(i, step):
        paths = []
        batch = 0
        reward = 0
        while True:
            obs, acs, rews = [],[],[]
            ob = env.reset()
            while True:
                # env.render()
                obs.append(ob)
                action = sess.run(ac_sample, {input_layer: ob[None]})
                # print(action)
                action = action[0]
                acs.append(action)
                # ob, rew, done, _ = env.step()
                ob, rew, done, _ = env.step(action)
                reward += rew
                rews.append(rew)
                if done:
                    break

            path = {
                'observation': obs,
                'action': acs,
                'reward': rews
            }
            # print(np.sum(rews))
            paths.append(path)
            batch+=len(path['reward'])
            if batch >=  min_timesteps_per_batch:
                break
        ob_no = np.concatenate([path["observation"] for path in paths])
        ac_na = np.concatenate([path["action"] for path in paths])
        q_n = []
        for path in paths:
            r = path['reward']
            max_len = len(r)
            q = [np.sum(np.power(gamma, np.arange(max_len-t)) * r[t:]) for t in range(max_len)]
            q_n.extend(q)
        b_n = sess.run(baseline, {input_layer:ob_no})
        b_n = (b_n - np.mean(q_n)) / (np.std(q_n))
        # q_n = q_n - b_n
        adv_n = q_n - b_n
        adv_n = (adv_n - np.mean(adv_n))/(np.std(adv_n))
        print("step:",i,"reward:",reward/len(paths))
        print(sess.run(train, {input_layer:ob_no, ac:ac_na, adv:adv_n}))
        q_n = (q_n - np.mean(q_n)) / (np.std(q_n))
        sess.run(base_train, {input_layer: ob_no, base_target:q_n})

#最终想要达到的效果，应该是能稳定平滑的向一边移动