强化学习(Sarsa与q-learning)代码示例
一、Sarsa算法
Sarsa算法是on-policy的，行为策略是什么，目标策略就是什么，即优化的是他实际执行的策略，所以使用Sarsa算法的agent在探索时显得有点“胆小”。
上Sarsa的代码：
import time
import numpy as np
import gym
class SarsaAgent(object):
def __init__(self, obs_n, act_n, e_greed=0.1, gamma=0.9, learning_rate=0.01):
self.act_n = act_n
self.Q = np.zeros((obs_n, act_n))
self.lr = learning_rate
self.gamma = gamma
self.epsilon = e_greed
def sample(self, obs):
if np.random.uniform(0,1) > (1.0 - self.epsilon):
action = np.random.choice(self.act_n)
else:
action = self.predict(obs)
return action
def predict(self, obs):
Q_list = self.Q[obs, :]
Qmax = np.max(Q_list)
action_list = np.where(Q_list == Qmax)[0]
action = np.random.choice(action_list)
return action
def learn(self, obs, action, next_obs, next_action, reward, done):
predict_Q = self.Q[obs, action]
if done:
target_Q = reward
else:
target_Q = reward + self.gamma * self.Q[next_obs, next_action]
self.Q[obs, action] += self.lr * (target_Q - predict_Q)
def run_episode(env, agent, render=False):
total_reward = 0
total_steps = 0
obs = env.reset()
action = agent.sample(obs)
while True:
next_obs, reward, done, _ = env.step(action)
next_action = agent.sample(next_obs)
agent.learn(obs, action, next_obs, next_action, reward, done)
total_reward += reward
total_steps += 1
obs = next_obs
action = next_action
if render:
env.render()
if done:
break
return total_reward, total_steps
def test_episode(env, agent):
total_reward = 0
# total_steps = 0
obs = env.reset()
while True:
action = agent.predict(obs)
next_obs, reward, done, _ = env.step(action)
next_action = agent.predict(next_obs)
total_reward += reward
obs = next_obs
time.sleep(0.5)
env.render()
if done:
break
return total_reward
env = gym.make('CliffWalking-v0')
agent = SarsaAgent(obs_n=env.observation_space.n,
act_n=env.action_space.n,
learning_rate=0.1,
gamma=0.9,
e_greed=0.1)
for episode in range(500):
ep_reward, ep_step = run_episode(env, agent, False)
print("episode:{}, step:{}, reward:{}".format(episode, ep_step, ep_reward))
test_reward = test_episode(env, agent)
print("test_reward:{}".format(test_reward))结果：agent在前往终点的路途上，会远远的绕过陷阱
episode:0, step:63, reward:-261
episode:1, step:837, reward:-2124
episode:2, step:225, reward:-324
episode:3, step:133, reward:-133
episode:4, step:283, reward:-481
episode:5, step:213, reward:-312
episode:6, step:74, reward:-74
episode:7, step:290, reward:-389
episode:8, step:185, reward:-284
episode:9, step:221, reward:-419
episode:10, step:196, reward:-295
episode:11, step:36, reward:-36
episode:12, step:212, reward:-410
episode:13, step:117, reward:-216
episode:14, step:79, reward:-79
episode:15, step:96, reward:-96
episode:16, step:47, reward:-47
episode:17, step:75, reward:-75
episode:18, step:79, reward:-79
episode:19, step:170, reward:-170
episode:20, step:55, reward:-55
episode:21, step:75, reward:-273
episode:22, step:99, reward:-198
episode:23, step:53, reward:-152
episode:24, step:53, reward:-53
episode:25, step:108, reward:-207
episode:26, step:136, reward:-334
episode:27, step:74, reward:-74
episode:28, step:61, reward:-61
episode:29, step:76, reward:-76
episode:30, step:81, reward:-81
episode:31, step:72, reward:-171
episode:32, step:104, reward:-104
episode:33, step:58, reward:-58
episode:34, step:48, reward:-48
episode:35, step:31, reward:-31
episode:36, step:74, reward:-74
episode:37, step:117, reward:-216
episode:38, step:38, reward:-38
episode:39, step:68, reward:-167
episode:40, step:46, reward:-46
episode:41, step:61, reward:-61
episode:42, step:93, reward:-291
episode:43, step:170, reward:-368
episode:44, step:67, reward:-67
episode:45, step:139, reward:-238
episode:46, step:174, reward:-372
episode:47, step:37, reward:-37
episode:48, step:174, reward:-273
episode:49, step:39, reward:-39
episode:50, step:58, reward:-58
episode:51, step:79, reward:-178
episode:52, step:55, reward:-55
episode:53, step:86, reward:-86
episode:54, step:74, reward:-74
episode:55, step:67, reward:-67
episode:56, step:67, reward:-67
episode:57, step:57, reward:-57
episode:58, step:71, reward:-71
episode:59, step:123, reward:-123
episode:60, step:66, reward:-66
episode:61, step:85, reward:-85
episode:62, step:50, reward:-50
episode:63, step:80, reward:-80
episode:64, step:72, reward:-72
episode:65, step:50, reward:-50
episode:66, step:73, reward:-73
episode:67, step:38, reward:-38
episode:68, step:72, reward:-72
episode:69, step:41, reward:-41
episode:70, step:107, reward:-206
episode:71, step:129, reward:-228
episode:72, step:27, reward:-27
episode:73, step:67, reward:-67
episode:74, step:73, reward:-172
episode:75, step:56, reward:-56
episode:76, step:47, reward:-47
episode:77, step:67, reward:-67
episode:78, step:34, reward:-34
episode:79, step:51, reward:-150
episode:80, step:68, reward:-68
episode:81, step:74, reward:-173
episode:82, step:38, reward:-38
episode:83, step:35, reward:-35
episode:84, step:61, reward:-61
episode:85, step:64, reward:-163
episode:86, step:49, reward:-49
episode:87, step:63, reward:-63
episode:88, step:49, reward:-49
episode:89, step:51, reward:-51
episode:90, step:49, reward:-49
episode:91, step:46, reward:-46
episode:92, step:50, reward:-50
episode:93, step:31, reward:-31
episode:94, step:63, reward:-162
episode:95, step:40, reward:-40
episode:96, step:53, reward:-53
episode:97, step:33, reward:-33
episode:98, step:58, reward:-157
episode:99, step:43, reward:-43
episode:100, step:44, reward:-44
episode:101, step:38, reward:-38
episode:102, step:27, reward:-27
episode:103, step:33, reward:-33
episode:104, step:29, reward:-29
episode:105, step:24, reward:-24
episode:106, step:38, reward:-38
episode:107, step:35, reward:-35
episode:108, step:65, reward:-65
episode:109, step:25, reward:-25
episode:110, step:21, reward:-120
episode:111, step:25, reward:-25
episode:112, step:44, reward:-143
episode:113, step:31, reward:-130
episode:114, step:33, reward:-33
episode:115, step:24, reward:-24
episode:116, step:23, reward:-23
episode:117, step:47, reward:-47
episode:118, step:40, reward:-40
episode:119, step:31, reward:-31
episode:120, step:65, reward:-164
episode:121, step:21, reward:-21
episode:122, step:29, reward:-29
episode:123, step:19, reward:-19
episode:124, step:31, reward:-31
episode:125, step:50, reward:-149
episode:126, step:35, reward:-35
episode:127, step:33, reward:-33
episode:128, step:32, reward:-32
episode:129, step:58, reward:-58
episode:130, step:31, reward:-31
episode:131, step:21, reward:-21
episode:132, step:53, reward:-53
episode:133, step:34, reward:-34
episode:134, step:24, reward:-24
episode:135, step:24, reward:-24
episode:136, step:40, reward:-40
episode:137, step:25, reward:-25
episode:138, step:41, reward:-41
episode:139, step:21, reward:-21
episode:140, step:23, reward:-23
episode:141, step:42, reward:-42
episode:142, step:23, reward:-23
episode:143, step:34, reward:-34
episode:144, step:29, reward:-29
episode:145, step:31, reward:-31
episode:146, step:23, reward:-23
episode:147, step:27, reward:-27
episode:148, step:25, reward:-25
episode:149, step:31, reward:-130
episode:150, step:22, reward:-22
episode:151, step:21, reward:-21
episode:152, step:25, reward:-25
episode:153, step:26, reward:-26
episode:154, step:27, reward:-27
episode:155, step:21, reward:-21
episode:156, step:26, reward:-26
episode:157, step:17, reward:-17
episode:158, step:30, reward:-129
episode:159, step:21, reward:-21
episode:160, step:29, reward:-128
episode:161, step:21, reward:-21
episode:162, step:21, reward:-21
episode:163, step:21, reward:-21
episode:164, step:24, reward:-24
episode:165, step:19, reward:-19
episode:166, step:15, reward:-15
episode:167, step:21, reward:-21
episode:168, step:33, reward:-33
episode:169, step:23, reward:-23
episode:170, step:23, reward:-23
episode:171, step:29, reward:-29
episode:172, step:22, reward:-22
episode:173, step:27, reward:-27
episode:174, step:27, reward:-27
episode:175, step:24, reward:-24
episode:176, step:30, reward:-30
episode:177, step:31, reward:-31
episode:178, step:24, reward:-321
episode:179, step:20, reward:-20
episode:180, step:28, reward:-28
episode:181, step:29, reward:-29
episode:182, step:37, reward:-37
episode:183, step:23, reward:-23
episode:184, step:30, reward:-30
episode:185, step:21, reward:-21
episode:186, step:17, reward:-17
episode:187, step:19, reward:-19
episode:188, step:23, reward:-23
episode:189, step:17, reward:-17
episode:190, step:21, reward:-21
episode:191, step:23, reward:-23
episode:192, step:30, reward:-30
episode:193, step:29, reward:-29
episode:194, step:17, reward:-17
episode:195, step:21, reward:-21
episode:196, step:27, reward:-27
episode:197, step:21, reward:-21
episode:198, step:26, reward:-26
episode:199, step:27, reward:-27
episode:200, step:24, reward:-24
episode:201, step:20, reward:-20
episode:202, step:36, reward:-36
episode:203, step:29, reward:-29
episode:204, step:23, reward:-23
episode:205, step:29, reward:-29
episode:206, step:23, reward:-23
episode:207, step:26, reward:-26
episode:208, step:19, reward:-19
episode:209, step:21, reward:-21
episode:210, step:19, reward:-19
episode:211, step:25, reward:-25
episode:212, step:28, reward:-28
episode:213, step:33, reward:-132
episode:214, step:17, reward:-17
episode:215, step:23, reward:-23
episode:216, step:25, reward:-25
episode:217, step:22, reward:-22
episode:218, step:20, reward:-20
episode:219, step:22, reward:-22
episode:220, step:20, reward:-20
episode:221, step:21, reward:-21
episode:222, step:19, reward:-19
episode:223, step:23, reward:-23
episode:224, step:21, reward:-21
episode:225, step:29, reward:-128
episode:226, step:25, reward:-25
episode:227, step:17, reward:-17
episode:228, step:26, reward:-26
episode:229, step:15, reward:-15
episode:230, step:24, reward:-24
episode:231, step:17, reward:-17
episode:232, step:18, reward:-18
episode:233, step:25, reward:-25
episode:234, step:17, reward:-17
episode:235, step:19, reward:-19
episode:236, step:17, reward:-17
episode:237, step:19, reward:-19
episode:238, step:23, reward:-23
episode:239, step:15, reward:-15
episode:240, step:19, reward:-19
episode:241, step:19, reward:-19
episode:242, step:19, reward:-19
episode:243, step:18, reward:-18
episode:244, step:28, reward:-28
episode:245, step:15, reward:-15
episode:246, step:15, reward:-15
episode:247, step:22, reward:-22
episode:248, step:17, reward:-17
episode:249, step:17, reward:-17
episode:250, step:19, reward:-19
episode:251, step:17, reward:-17
episode:252, step:15, reward:-15
episode:253, step:17, reward:-17
episode:254, step:15, reward:-15
episode:255, step:25, reward:-25
episode:256, step:25, reward:-25
episode:257, step:18, reward:-18
episode:258, step:22, reward:-121
episode:259, step:19, reward:-19
episode:260, step:36, reward:-36
episode:261, step:18, reward:-18
episode:262, step:27, reward:-27
episode:263, step:17, reward:-17
episode:264, step:20, reward:-20
episode:265, step:22, reward:-22
episode:266, step:17, reward:-17
episode:267, step:23, reward:-23
episode:268, step:17, reward:-17
episode:269, step:24, reward:-24
episode:270, step:21, reward:-21
episode:271, step:17, reward:-17
episode:272, step:22, reward:-22
episode:273, step:27, reward:-27
episode:274, step:18, reward:-18
episode:275, step:20, reward:-20
episode:276, step:20, reward:-119
episode:277, step:25, reward:-25
episode:278, step:24, reward:-24
episode:279, step:19, reward:-19
episode:280, step:21, reward:-21
episode:281, step:20, reward:-20
episode:282, step:21, reward:-21
episode:283, step:19, reward:-19
episode:284, step:21, reward:-21
episode:285, step:24, reward:-24
episode:286, step:21, reward:-21
episode:287, step:19, reward:-19
episode:288, step:17, reward:-17
episode:289, step:21, reward:-21
episode:290, step:21, reward:-21
episode:291, step:17, reward:-17
episode:292, step:19, reward:-19
episode:293, step:20, reward:-20
episode:294, step:18, reward:-18
episode:295, step:19, reward:-19
episode:296, step:19, reward:-19
episode:297, step:16, reward:-16
episode:298, step:17, reward:-17
episode:299, step:19, reward:-19
episode:300, step:17, reward:-17
episode:301, step:15, reward:-15
episode:302, step:18, reward:-117
episode:303, step:21, reward:-21
episode:304, step:15, reward:-15
episode:305, step:20, reward:-20
episode:306, step:15, reward:-15
episode:307, step:17, reward:-17
episode:308, step:15, reward:-15
episode:309, step:17, reward:-17
episode:310, step:17, reward:-17
episode:311, step:21, reward:-21
episode:312, step:23, reward:-23
episode:313, step:19, reward:-19
episode:314, step:22, reward:-22
episode:315, step:17, reward:-17
episode:316, step:21, reward:-21
episode:317, step:21, reward:-21
episode:318, step:18, reward:-18
episode:319, step:19, reward:-19
episode:320, step:18, reward:-18
episode:321, step:17, reward:-17
episode:322, step:17, reward:-17
episode:323, step:20, reward:-20
episode:324, step:18, reward:-117
episode:325, step:20, reward:-20
episode:326, step:19, reward:-19
episode:327, step:21, reward:-21
episode:328, step:21, reward:-21
episode:329, step:29, reward:-29
episode:330, step:21, reward:-21
episode:331, step:19, reward:-19
episode:332, step:18, reward:-18
episode:333, step:22, reward:-22
episode:334, step:18, reward:-18
episode:335, step:20, reward:-20
episode:336, step:25, reward:-25
episode:337, step:21, reward:-21
episode:338, step:17, reward:-17
episode:339, step:22, reward:-22
episode:340, step:15, reward:-15
episode:341, step:21, reward:-21
episode:342, step:16, reward:-16
episode:343, step:19, reward:-19
episode:344, step:15, reward:-15
episode:345, step:15, reward:-15
episode:346, step:15, reward:-15
episode:347, step:21, reward:-21
episode:348, step:15, reward:-15
episode:349, step:15, reward:-15
episode:350, step:16, reward:-16
episode:351, step:15, reward:-15
episode:352, step:19, reward:-19
episode:353, step:17, reward:-17
episode:354, step:17, reward:-17
episode:355, step:17, reward:-17
episode:356, step:17, reward:-116
episode:357, step:15, reward:-15
episode:358, step:19, reward:-19
episode:359, step:16, reward:-16
episode:360, step:19, reward:-19
episode:361, step:19, reward:-19
episode:362, step:17, reward:-17
episode:363, step:17, reward:-17
episode:364, step:19, reward:-19
episode:365, step:15, reward:-15
episode:366, step:15, reward:-15
episode:367, step:15, reward:-15
episode:368, step:17, reward:-17
episode:369, step:15, reward:-15
episode:370, step:17, reward:-17
episode:371, step:17, reward:-17
episode:372, step:16, reward:-16
episode:373, step:23, reward:-23
episode:374, step:15, reward:-15
episode:375, step:19, reward:-19
episode:376, step:17, reward:-17
episode:377, step:17, reward:-17
episode:378, step:25, reward:-25
episode:379, step:19, reward:-19
episode:380, step:15, reward:-15
episode:381, step:17, reward:-17
episode:382, step:23, reward:-23
episode:383, step:17, reward:-17
episode:384, step:15, reward:-15
episode:385, step:19, reward:-19
episode:386, step:21, reward:-21
episode:387, step:25, reward:-25
episode:388, step:17, reward:-17
episode:389, step:18, reward:-18
episode:390, step:21, reward:-21
episode:391, step:17, reward:-17
episode:392, step:17, reward:-17
episode:393, step:19, reward:-19
episode:394, step:15, reward:-15
episode:395, step:17, reward:-17
episode:396, step:21, reward:-21
episode:397, step:19, reward:-19
episode:398, step:15, reward:-15
episode:399, step:21, reward:-21
episode:400, step:21, reward:-21
episode:401, step:19, reward:-19
episode:402, step:25, reward:-25
episode:403, step:21, reward:-21
episode:404, step:17, reward:-17
episode:405, step:19, reward:-19
episode:406, step:18, reward:-18
episode:407, step:17, reward:-17
episode:408, step:22, reward:-22
episode:409, step:20, reward:-20
episode:410, step:19, reward:-19
episode:411, step:20, reward:-20
episode:412, step:23, reward:-23
episode:413, step:17, reward:-17
episode:414, step:17, reward:-17
episode:415, step:18, reward:-18
episode:416, step:25, reward:-25
episode:417, step:21, reward:-21
episode:418, step:19, reward:-19
episode:419, step:21, reward:-21
episode:420, step:17, reward:-17
episode:421, step:17, reward:-17
episode:422, step:17, reward:-17
episode:423, step:17, reward:-17
episode:424, step:17, reward:-17
episode:425, step:17, reward:-17
episode:426, step:17, reward:-17
episode:427, step:17, reward:-17
episode:428, step:39, reward:-138
episode:429, step:23, reward:-23
episode:430, step:21, reward:-21
episode:431, step:18, reward:-18
episode:432, step:17, reward:-17
episode:433, step:17, reward:-17
episode:434, step:24, reward:-24
episode:435, step:18, reward:-18
episode:436, step:17, reward:-17
episode:437, step:28, reward:-28
episode:438, step:18, reward:-18
episode:439, step:18, reward:-18
episode:440, step:24, reward:-24
episode:441, step:17, reward:-17
episode:442, step:20, reward:-20
episode:443, step:19, reward:-19
episode:444, step:21, reward:-21
episode:445, step:17, reward:-17
episode:446, step:18, reward:-18
episode:447, step:22, reward:-22
episode:448, step:19, reward:-19
episode:449, step:45, reward:-144
episode:450, step:21, reward:-21
episode:451, step:23, reward:-23
episode:452, step:19, reward:-19
episode:453, step:17, reward:-17
episode:454, step:20, reward:-20
episode:455, step:17, reward:-17
episode:456, step:20, reward:-20
episode:457, step:17, reward:-17
episode:458, step:22, reward:-22
episode:459, step:19, reward:-19
episode:460, step:18, reward:-18
episode:461, step:23, reward:-23
episode:462, step:15, reward:-15
episode:463, step:25, reward:-25
episode:464, step:20, reward:-20
episode:465, step:17, reward:-17
episode:466, step:17, reward:-17
episode:467, step:19, reward:-19
episode:468, step:17, reward:-17
episode:469, step:17, reward:-17
episode:470, step:15, reward:-15
episode:471, step:19, reward:-19
episode:472, step:23, reward:-23
episode:473, step:23, reward:-23
episode:474, step:17, reward:-17
episode:475, step:19, reward:-19
episode:476, step:17, reward:-17
episode:477, step:15, reward:-15
episode:478, step:19, reward:-19
episode:479, step:16, reward:-16
episode:480, step:18, reward:-18
episode:481, step:23, reward:-23
episode:482, step:20, reward:-20
episode:483, step:19, reward:-19
episode:484, step:15, reward:-15
episode:485, step:16, reward:-115
episode:486, step:18, reward:-18
episode:487, step:19, reward:-19
episode:488, step:19, reward:-19
episode:489, step:15, reward:-15
episode:490, step:19, reward:-19
episode:491, step:15, reward:-15
episode:492, step:21, reward:-21
episode:493, step:17, reward:-17
episode:494, step:18, reward:-18
episode:495, step:18, reward:-18
episode:496, step:17, reward:-17
episode:497, step:17, reward:-17
episode:498, step:19, reward:-19
episode:499, step:15, reward:-15
o o o o o o o o o o o o
o o o o o o o o o o o o
x o o o o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
x o o o o o o o o o o o
o o o o o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o x o o o o o o o o o o
o o o o o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o x o o o o o o o o o
o o o o o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o x o o o o o o o o
o o o o o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o x o o o o o o o
o o o o o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o x o o o o o o
o o o o o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o x o o o o o
o o o o o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o x o o o o
o o o o o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o x o o o
o o o o o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o x o o
o o o o o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o o x o
o o o o o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o o o x
o o o o o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o o o o
o o o o o o o o o o o x
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o o o o
o o o o o o o o o o o o
o C C C C C C C C C C x
test_reward:-15二、Q-learning算法：
off-policy，在探索环境时会显得比Sarsa更加大胆，行为策略与目标策略分离
上代码：
class Qagent():
def __init__(self, act_n, obs_n, learning_rate=0.01, gamma=0.9, e_greed=0.1):
self.act_n = act_n
self.lr = learning_rate
self.gamma = gamma
self.epsilon = e_greed
self.Q = np.zeros((obs_n,act_n))
def sample(self, obs):
if np.random.uniform(0,1) < (1.0 - self.epsilon):
action = self.predict(obs)
else:
action = np.random.choice(self.act_n)
return action
def predict(self, obs):
Q_list = self.Q[obs, :]
maxQ = max(Q_list)
action_list = np.where(Q_list == maxQ)[0]
action = np.random.choice(action_list)
return action
def learn(self, obs, action, reward, next_obs, done):
predict_Q = self.Q[obs, action]
if done:
target_Q = reward
else:
target_Q = reward + self.gamma * np.max(self.Q[next_obs, :])
self.Q[obs, action] += self.lr * (target_Q - predict_Q)
def save(self):
npy_file = './q_table.npy'
np.save(npy_file, self.Q)
print(npy_file + ' saved.')
def restore(self, npy_file='./q_table.npy'):
self.Q = np.load(npy_file)
print(npy_file + ' loaded.')
def run_episode(env, agent, render=False):
total_steps = 0
total_reward = 0
obs = env.reset()
while True:
action = agent.sample(obs)
next_obs, reward, done, _ = env.step(action)
agent.learn(obs, action, reward, next_obs, done)
obs = next_obs
total_reward += reward
total_steps += 1
if render:
env.render()
if done:
break
return total_reward, total_steps
def test_episode(env, agent):
total_reward = 0
obs = env.reset()
while True:
action = agent.predict(obs)
next_obs, reward, done, _ = env.step(action)
total_reward += reward
obs = next_obs
time.sleep(0.5)
env.render()
if done:
break
return total_reward
env = gym.make('CliffWalking-v0')
agent = Qagent(
obs_n=env.observation_space.n,
act_n=env.action_space.n,
learning_rate=0.1,
gamma=0.9,
e_greed=0.1)
for episode in range(500):
ep_reward, ep_steps = run_episode(env, agent, False)
print('Episode %s: step = %s , reward= %.1f' % (episode, ep_steps, ep_reward))
test_reward = test_episode(env, agent)
print('test reward = %.1f' % (test_reward))输出：可以明显的看出，Q-learning的agent是贴着陷阱走的，非常大胆，最后的效果也很好
Episode 0: step = 674 , reward= -2060.0
Episode 1: step = 272 , reward= -371.0
Episode 2: step = 305 , reward= -404.0
Episode 3: step = 199 , reward= -496.0
Episode 4: step = 388 , reward= -685.0
Episode 5: step = 111 , reward= -210.0
Episode 6: step = 207 , reward= -405.0
Episode 7: step = 90 , reward= -189.0
Episode 8: step = 189 , reward= -189.0
Episode 9: step = 177 , reward= -375.0
Episode 10: step = 67 , reward= -67.0
Episode 11: step = 157 , reward= -157.0
Episode 12: step = 148 , reward= -247.0
Episode 13: step = 69 , reward= -168.0
Episode 14: step = 137 , reward= -137.0
Episode 15: step = 149 , reward= -149.0
Episode 16: step = 59 , reward= -59.0
Episode 17: step = 148 , reward= -445.0
Episode 18: step = 110 , reward= -110.0
Episode 19: step = 131 , reward= -230.0
Episode 20: step = 85 , reward= -85.0
Episode 21: step = 70 , reward= -70.0
Episode 22: step = 57 , reward= -57.0
Episode 23: step = 129 , reward= -228.0
Episode 24: step = 89 , reward= -89.0
Episode 25: step = 247 , reward= -643.0
Episode 26: step = 82 , reward= -82.0
Episode 27: step = 91 , reward= -91.0
Episode 28: step = 66 , reward= -66.0
Episode 29: step = 47 , reward= -47.0
Episode 30: step = 55 , reward= -55.0
Episode 31: step = 93 , reward= -93.0
Episode 32: step = 78 , reward= -78.0
Episode 33: step = 54 , reward= -54.0
Episode 34: step = 169 , reward= -367.0
Episode 35: step = 119 , reward= -317.0
Episode 36: step = 33 , reward= -33.0
Episode 37: step = 32 , reward= -32.0
Episode 38: step = 82 , reward= -82.0
Episode 39: step = 123 , reward= -123.0
Episode 40: step = 34 , reward= -34.0
Episode 41: step = 54 , reward= -54.0
Episode 42: step = 55 , reward= -55.0
Episode 43: step = 76 , reward= -175.0
Episode 44: step = 38 , reward= -38.0
Episode 45: step = 81 , reward= -81.0
Episode 46: step = 75 , reward= -273.0
Episode 47: step = 54 , reward= -54.0
Episode 48: step = 56 , reward= -56.0
Episode 49: step = 76 , reward= -373.0
Episode 50: step = 58 , reward= -58.0
Episode 51: step = 57 , reward= -255.0
Episode 52: step = 95 , reward= -95.0
Episode 53: step = 41 , reward= -140.0
Episode 54: step = 44 , reward= -44.0
Episode 55: step = 75 , reward= -75.0
Episode 56: step = 32 , reward= -32.0
Episode 57: step = 59 , reward= -59.0
Episode 58: step = 21 , reward= -21.0
Episode 59: step = 109 , reward= -208.0
Episode 60: step = 54 , reward= -153.0
Episode 61: step = 46 , reward= -145.0
Episode 62: step = 38 , reward= -38.0
Episode 63: step = 57 , reward= -156.0
Episode 64: step = 48 , reward= -48.0
Episode 65: step = 39 , reward= -39.0
Episode 66: step = 50 , reward= -50.0
Episode 67: step = 47 , reward= -47.0
Episode 68: step = 119 , reward= -218.0
Episode 69: step = 42 , reward= -42.0
Episode 70: step = 84 , reward= -282.0
Episode 71: step = 22 , reward= -22.0
Episode 72: step = 36 , reward= -36.0
Episode 73: step = 59 , reward= -59.0
Episode 74: step = 31 , reward= -31.0
Episode 75: step = 57 , reward= -57.0
Episode 76: step = 44 , reward= -143.0
Episode 77: step = 42 , reward= -42.0
Episode 78: step = 43 , reward= -43.0
Episode 79: step = 50 , reward= -50.0
Episode 80: step = 26 , reward= -26.0
Episode 81: step = 46 , reward= -46.0
Episode 82: step = 42 , reward= -42.0
Episode 83: step = 36 , reward= -135.0
Episode 84: step = 41 , reward= -41.0
Episode 85: step = 26 , reward= -26.0
Episode 86: step = 79 , reward= -178.0
Episode 87: step = 42 , reward= -42.0
Episode 88: step = 37 , reward= -136.0
Episode 89: step = 102 , reward= -399.0
Episode 90: step = 41 , reward= -41.0
Episode 91: step = 36 , reward= -135.0
Episode 92: step = 44 , reward= -143.0
Episode 93: step = 46 , reward= -46.0
Episode 94: step = 39 , reward= -39.0
Episode 95: step = 30 , reward= -30.0
Episode 96: step = 36 , reward= -135.0
Episode 97: step = 33 , reward= -33.0
Episode 98: step = 46 , reward= -46.0
Episode 99: step = 55 , reward= -154.0
Episode 100: step = 34 , reward= -34.0
Episode 101: step = 36 , reward= -36.0
Episode 102: step = 43 , reward= -43.0
Episode 103: step = 49 , reward= -49.0
Episode 104: step = 77 , reward= -275.0
Episode 105: step = 62 , reward= -557.0
Episode 106: step = 55 , reward= -154.0
Episode 107: step = 24 , reward= -24.0
Episode 108: step = 36 , reward= -36.0
Episode 109: step = 39 , reward= -39.0
Episode 110: step = 40 , reward= -40.0
Episode 111: step = 34 , reward= -34.0
Episode 112: step = 25 , reward= -25.0
Episode 113: step = 31 , reward= -31.0
Episode 114: step = 34 , reward= -34.0
Episode 115: step = 36 , reward= -36.0
Episode 116: step = 27 , reward= -27.0
Episode 117: step = 32 , reward= -32.0
Episode 118: step = 17 , reward= -17.0
Episode 119: step = 24 , reward= -123.0
Episode 120: step = 48 , reward= -48.0
Episode 121: step = 51 , reward= -249.0
Episode 122: step = 28 , reward= -28.0
Episode 123: step = 38 , reward= -38.0
Episode 124: step = 38 , reward= -137.0
Episode 125: step = 33 , reward= -33.0
Episode 126: step = 19 , reward= -19.0
Episode 127: step = 49 , reward= -148.0
Episode 128: step = 44 , reward= -44.0
Episode 129: step = 62 , reward= -161.0
Episode 130: step = 47 , reward= -146.0
Episode 131: step = 38 , reward= -137.0
Episode 132: step = 22 , reward= -22.0
Episode 133: step = 45 , reward= -243.0
Episode 134: step = 28 , reward= -28.0
Episode 135: step = 56 , reward= -155.0
Episode 136: step = 24 , reward= -24.0
Episode 137: step = 24 , reward= -24.0
Episode 138: step = 23 , reward= -23.0
Episode 139: step = 17 , reward= -17.0
Episode 140: step = 33 , reward= -33.0
Episode 141: step = 22 , reward= -22.0
Episode 142: step = 36 , reward= -135.0
Episode 143: step = 32 , reward= -32.0
Episode 144: step = 32 , reward= -32.0
Episode 145: step = 26 , reward= -26.0
Episode 146: step = 47 , reward= -146.0
Episode 147: step = 49 , reward= -148.0
Episode 148: step = 21 , reward= -21.0
Episode 149: step = 36 , reward= -36.0
Episode 150: step = 42 , reward= -240.0
Episode 151: step = 27 , reward= -126.0
Episode 152: step = 18 , reward= -18.0
Episode 153: step = 39 , reward= -138.0
Episode 154: step = 21 , reward= -120.0
Episode 155: step = 33 , reward= -33.0
Episode 156: step = 34 , reward= -34.0
Episode 157: step = 30 , reward= -30.0
Episode 158: step = 21 , reward= -120.0
Episode 159: step = 29 , reward= -29.0
Episode 160: step = 25 , reward= -25.0
Episode 161: step = 29 , reward= -29.0
Episode 162: step = 22 , reward= -22.0
Episode 163: step = 38 , reward= -38.0
Episode 164: step = 29 , reward= -128.0
Episode 165: step = 27 , reward= -27.0
Episode 166: step = 14 , reward= -14.0
Episode 167: step = 31 , reward= -31.0
Episode 168: step = 27 , reward= -27.0
Episode 169: step = 24 , reward= -24.0
Episode 170: step = 69 , reward= -267.0
Episode 171: step = 19 , reward= -19.0
Episode 172: step = 30 , reward= -30.0
Episode 173: step = 14 , reward= -113.0
Episode 174: step = 58 , reward= -157.0
Episode 175: step = 30 , reward= -129.0
Episode 176: step = 16 , reward= -16.0
Episode 177: step = 25 , reward= -25.0
Episode 178: step = 13 , reward= -13.0
Episode 179: step = 30 , reward= -30.0
Episode 180: step = 23 , reward= -23.0
Episode 181: step = 15 , reward= -15.0
Episode 182: step = 17 , reward= -17.0
Episode 183: step = 33 , reward= -132.0
Episode 184: step = 22 , reward= -22.0
Episode 185: step = 29 , reward= -128.0
Episode 186: step = 13 , reward= -13.0
Episode 187: step = 44 , reward= -143.0
Episode 188: step = 23 , reward= -23.0
Episode 189: step = 15 , reward= -15.0
Episode 190: step = 46 , reward= -145.0
Episode 191: step = 25 , reward= -25.0
Episode 192: step = 15 , reward= -15.0
Episode 193: step = 25 , reward= -124.0
Episode 194: step = 24 , reward= -24.0
Episode 195: step = 22 , reward= -22.0
Episode 196: step = 16 , reward= -16.0
Episode 197: step = 19 , reward= -19.0
Episode 198: step = 23 , reward= -122.0
Episode 199: step = 17 , reward= -17.0
Episode 200: step = 20 , reward= -20.0
Episode 201: step = 22 , reward= -22.0
Episode 202: step = 20 , reward= -20.0
Episode 203: step = 42 , reward= -141.0
Episode 204: step = 47 , reward= -146.0
Episode 205: step = 13 , reward= -13.0
Episode 206: step = 22 , reward= -121.0
Episode 207: step = 29 , reward= -29.0
Episode 208: step = 44 , reward= -242.0
Episode 209: step = 21 , reward= -21.0
Episode 210: step = 19 , reward= -19.0
Episode 211: step = 43 , reward= -142.0
Episode 212: step = 16 , reward= -115.0
Episode 213: step = 17 , reward= -17.0
Episode 214: step = 13 , reward= -13.0
Episode 215: step = 18 , reward= -18.0
Episode 216: step = 17 , reward= -116.0
Episode 217: step = 15 , reward= -15.0
Episode 218: step = 20 , reward= -20.0
Episode 219: step = 24 , reward= -24.0
Episode 220: step = 16 , reward= -16.0
Episode 221: step = 13 , reward= -13.0
Episode 222: step = 25 , reward= -25.0
Episode 223: step = 13 , reward= -13.0
Episode 224: step = 15 , reward= -15.0
Episode 225: step = 21 , reward= -21.0
Episode 226: step = 24 , reward= -24.0
Episode 227: step = 37 , reward= -235.0
Episode 228: step = 13 , reward= -13.0
Episode 229: step = 20 , reward= -20.0
Episode 230: step = 13 , reward= -13.0
Episode 231: step = 31 , reward= -130.0
Episode 232: step = 20 , reward= -20.0
Episode 233: step = 15 , reward= -15.0
Episode 234: step = 15 , reward= -15.0
Episode 235: step = 13 , reward= -13.0
Episode 236: step = 22 , reward= -22.0
Episode 237: step = 18 , reward= -117.0
Episode 238: step = 18 , reward= -18.0
Episode 239: step = 20 , reward= -20.0
Episode 240: step = 17 , reward= -17.0
Episode 241: step = 15 , reward= -15.0
Episode 242: step = 37 , reward= -37.0
Episode 243: step = 15 , reward= -15.0
Episode 244: step = 16 , reward= -16.0
Episode 245: step = 13 , reward= -13.0
Episode 246: step = 49 , reward= -148.0
Episode 247: step = 27 , reward= -27.0
Episode 248: step = 13 , reward= -13.0
Episode 249: step = 31 , reward= -130.0
Episode 250: step = 15 , reward= -15.0
Episode 251: step = 13 , reward= -13.0
Episode 252: step = 19 , reward= -118.0
Episode 253: step = 13 , reward= -13.0
Episode 254: step = 13 , reward= -13.0
Episode 255: step = 30 , reward= -129.0
Episode 256: step = 17 , reward= -17.0
Episode 257: step = 21 , reward= -21.0
Episode 258: step = 27 , reward= -126.0
Episode 259: step = 13 , reward= -13.0
Episode 260: step = 16 , reward= -115.0
Episode 261: step = 18 , reward= -18.0
Episode 262: step = 13 , reward= -13.0
Episode 263: step = 13 , reward= -13.0
Episode 264: step = 21 , reward= -21.0
Episode 265: step = 18 , reward= -18.0
Episode 266: step = 15 , reward= -15.0
Episode 267: step = 13 , reward= -13.0
Episode 268: step = 15 , reward= -15.0
Episode 269: step = 14 , reward= -14.0
Episode 270: step = 21 , reward= -21.0
Episode 271: step = 13 , reward= -13.0
Episode 272: step = 24 , reward= -24.0
Episode 273: step = 17 , reward= -116.0
Episode 274: step = 13 , reward= -13.0
Episode 275: step = 39 , reward= -237.0
Episode 276: step = 15 , reward= -15.0
Episode 277: step = 18 , reward= -117.0
Episode 278: step = 20 , reward= -218.0
Episode 279: step = 13 , reward= -13.0
Episode 280: step = 15 , reward= -15.0
Episode 281: step = 17 , reward= -17.0
Episode 282: step = 13 , reward= -13.0
Episode 283: step = 15 , reward= -15.0
Episode 284: step = 13 , reward= -13.0
Episode 285: step = 24 , reward= -123.0
Episode 286: step = 34 , reward= -34.0
Episode 287: step = 13 , reward= -13.0
Episode 288: step = 20 , reward= -119.0
Episode 289: step = 13 , reward= -13.0
Episode 290: step = 14 , reward= -14.0
Episode 291: step = 15 , reward= -15.0
Episode 292: step = 15 , reward= -15.0
Episode 293: step = 13 , reward= -13.0
Episode 294: step = 13 , reward= -13.0
Episode 295: step = 28 , reward= -28.0
Episode 296: step = 19 , reward= -118.0
Episode 297: step = 24 , reward= -222.0
Episode 298: step = 13 , reward= -13.0
Episode 299: step = 22 , reward= -220.0
Episode 300: step = 22 , reward= -121.0
Episode 301: step = 15 , reward= -15.0
Episode 302: step = 13 , reward= -13.0
Episode 303: step = 40 , reward= -337.0
Episode 304: step = 15 , reward= -15.0
Episode 305: step = 13 , reward= -13.0
Episode 306: step = 21 , reward= -21.0
Episode 307: step = 14 , reward= -14.0
Episode 308: step = 28 , reward= -127.0
Episode 309: step = 16 , reward= -115.0
Episode 310: step = 24 , reward= -123.0
Episode 311: step = 21 , reward= -120.0
Episode 312: step = 15 , reward= -15.0
Episode 313: step = 34 , reward= -232.0
Episode 314: step = 13 , reward= -13.0
Episode 315: step = 23 , reward= -23.0
Episode 316: step = 17 , reward= -116.0
Episode 317: step = 13 , reward= -13.0
Episode 318: step = 31 , reward= -328.0
Episode 319: step = 15 , reward= -15.0
Episode 320: step = 13 , reward= -13.0
Episode 321: step = 30 , reward= -228.0
Episode 322: step = 13 , reward= -13.0
Episode 323: step = 13 , reward= -13.0
Episode 324: step = 13 , reward= -13.0
Episode 325: step = 37 , reward= -334.0
Episode 326: step = 15 , reward= -15.0
Episode 327: step = 15 , reward= -15.0
Episode 328: step = 13 , reward= -13.0
Episode 329: step = 13 , reward= -13.0
Episode 330: step = 15 , reward= -15.0
Episode 331: step = 13 , reward= -13.0
Episode 332: step = 22 , reward= -121.0
Episode 333: step = 13 , reward= -13.0
Episode 334: step = 15 , reward= -15.0
Episode 335: step = 13 , reward= -13.0
Episode 336: step = 13 , reward= -13.0
Episode 337: step = 15 , reward= -15.0
Episode 338: step = 19 , reward= -19.0
Episode 339: step = 26 , reward= -125.0
Episode 340: step = 13 , reward= -13.0
Episode 341: step = 14 , reward= -113.0
Episode 342: step = 15 , reward= -15.0
Episode 343: step = 13 , reward= -13.0
Episode 344: step = 19 , reward= -19.0
Episode 345: step = 32 , reward= -230.0
Episode 346: step = 13 , reward= -13.0
Episode 347: step = 15 , reward= -15.0
Episode 348: step = 15 , reward= -15.0
Episode 349: step = 18 , reward= -117.0
Episode 350: step = 13 , reward= -13.0
Episode 351: step = 13 , reward= -13.0
Episode 352: step = 13 , reward= -13.0
Episode 353: step = 13 , reward= -13.0
Episode 354: step = 13 , reward= -13.0
Episode 355: step = 13 , reward= -13.0
Episode 356: step = 13 , reward= -13.0
Episode 357: step = 20 , reward= -119.0
Episode 358: step = 15 , reward= -15.0
Episode 359: step = 18 , reward= -117.0
Episode 360: step = 15 , reward= -15.0
Episode 361: step = 17 , reward= -17.0
Episode 362: step = 13 , reward= -13.0
Episode 363: step = 18 , reward= -18.0
Episode 364: step = 20 , reward= -119.0
Episode 365: step = 13 , reward= -13.0
Episode 366: step = 18 , reward= -117.0
Episode 367: step = 13 , reward= -13.0
Episode 368: step = 13 , reward= -13.0
Episode 369: step = 30 , reward= -228.0
Episode 370: step = 28 , reward= -226.0
Episode 371: step = 17 , reward= -17.0
Episode 372: step = 13 , reward= -13.0
Episode 373: step = 17 , reward= -17.0
Episode 374: step = 13 , reward= -13.0
Episode 375: step = 13 , reward= -13.0
Episode 376: step = 16 , reward= -115.0
Episode 377: step = 14 , reward= -14.0
Episode 378: step = 14 , reward= -14.0
Episode 379: step = 13 , reward= -13.0
Episode 380: step = 15 , reward= -15.0
Episode 381: step = 14 , reward= -14.0
Episode 382: step = 16 , reward= -16.0
Episode 383: step = 19 , reward= -19.0
Episode 384: step = 33 , reward= -231.0
Episode 385: step = 17 , reward= -17.0
Episode 386: step = 13 , reward= -13.0
Episode 387: step = 21 , reward= -120.0
Episode 388: step = 30 , reward= -129.0
Episode 389: step = 13 , reward= -13.0
Episode 390: step = 19 , reward= -19.0
Episode 391: step = 13 , reward= -13.0
Episode 392: step = 23 , reward= -122.0
Episode 393: step = 13 , reward= -13.0
Episode 394: step = 19 , reward= -19.0
Episode 395: step = 13 , reward= -13.0
Episode 396: step = 13 , reward= -13.0
Episode 397: step = 13 , reward= -13.0
Episode 398: step = 13 , reward= -13.0
Episode 399: step = 16 , reward= -115.0
Episode 400: step = 26 , reward= -125.0
Episode 401: step = 20 , reward= -119.0
Episode 402: step = 13 , reward= -13.0
Episode 403: step = 15 , reward= -15.0
Episode 404: step = 25 , reward= -124.0
Episode 405: step = 19 , reward= -19.0
Episode 406: step = 23 , reward= -221.0
Episode 407: step = 13 , reward= -13.0
Episode 408: step = 17 , reward= -116.0
Episode 409: step = 13 , reward= -13.0
Episode 410: step = 15 , reward= -15.0
Episode 411: step = 17 , reward= -17.0
Episode 412: step = 13 , reward= -13.0
Episode 413: step = 13 , reward= -13.0
Episode 414: step = 16 , reward= -115.0
Episode 415: step = 18 , reward= -117.0
Episode 416: step = 15 , reward= -15.0
Episode 417: step = 15 , reward= -15.0
Episode 418: step = 13 , reward= -13.0
Episode 419: step = 14 , reward= -14.0
Episode 420: step = 16 , reward= -16.0
Episode 421: step = 13 , reward= -13.0
Episode 422: step = 13 , reward= -13.0
Episode 423: step = 13 , reward= -13.0
Episode 424: step = 17 , reward= -17.0
Episode 425: step = 33 , reward= -33.0
Episode 426: step = 14 , reward= -14.0
Episode 427: step = 18 , reward= -117.0
Episode 428: step = 28 , reward= -127.0
Episode 429: step = 15 , reward= -15.0
Episode 430: step = 15 , reward= -15.0
Episode 431: step = 14 , reward= -14.0
Episode 432: step = 15 , reward= -15.0
Episode 433: step = 13 , reward= -13.0
Episode 434: step = 22 , reward= -22.0
Episode 435: step = 14 , reward= -14.0
Episode 436: step = 17 , reward= -17.0
Episode 437: step = 25 , reward= -25.0
Episode 438: step = 17 , reward= -17.0
Episode 439: step = 24 , reward= -123.0
Episode 440: step = 29 , reward= -128.0
Episode 441: step = 21 , reward= -120.0
Episode 442: step = 32 , reward= -131.0
Episode 443: step = 13 , reward= -13.0
Episode 444: step = 18 , reward= -117.0
Episode 445: step = 13 , reward= -13.0
Episode 446: step = 13 , reward= -13.0
Episode 447: step = 13 , reward= -13.0
Episode 448: step = 16 , reward= -16.0
Episode 449: step = 13 , reward= -13.0
Episode 450: step = 19 , reward= -118.0
Episode 451: step = 23 , reward= -23.0
Episode 452: step = 15 , reward= -15.0
Episode 453: step = 21 , reward= -120.0
Episode 454: step = 23 , reward= -23.0
Episode 455: step = 15 , reward= -15.0
Episode 456: step = 25 , reward= -124.0
Episode 457: step = 13 , reward= -13.0
Episode 458: step = 25 , reward= -322.0
Episode 459: step = 25 , reward= -124.0
Episode 460: step = 13 , reward= -13.0
Episode 461: step = 21 , reward= -120.0
Episode 462: step = 17 , reward= -17.0
Episode 463: step = 13 , reward= -13.0
Episode 464: step = 13 , reward= -13.0
Episode 465: step = 17 , reward= -17.0
Episode 466: step = 15 , reward= -15.0
Episode 467: step = 26 , reward= -125.0
Episode 468: step = 17 , reward= -116.0
Episode 469: step = 28 , reward= -28.0
Episode 470: step = 15 , reward= -15.0
Episode 471: step = 13 , reward= -13.0
Episode 472: step = 13 , reward= -13.0
Episode 473: step = 15 , reward= -15.0
Episode 474: step = 13 , reward= -13.0
Episode 475: step = 13 , reward= -13.0
Episode 476: step = 17 , reward= -17.0
Episode 477: step = 13 , reward= -13.0
Episode 478: step = 13 , reward= -13.0
Episode 479: step = 13 , reward= -13.0
Episode 480: step = 22 , reward= -121.0
Episode 481: step = 21 , reward= -120.0
Episode 482: step = 27 , reward= -126.0
Episode 483: step = 19 , reward= -19.0
Episode 484: step = 15 , reward= -15.0
Episode 485: step = 30 , reward= -525.0
Episode 486: step = 15 , reward= -15.0
Episode 487: step = 14 , reward= -14.0
Episode 488: step = 15 , reward= -15.0
Episode 489: step = 16 , reward= -16.0
Episode 490: step = 17 , reward= -17.0
Episode 491: step = 15 , reward= -15.0
Episode 492: step = 13 , reward= -13.0
Episode 493: step = 32 , reward= -131.0
Episode 494: step = 13 , reward= -13.0
Episode 495: step = 15 , reward= -15.0
Episode 496: step = 15 , reward= -15.0
Episode 497: step = 26 , reward= -125.0
Episode 498: step = 13 , reward= -13.0
Episode 499: step = 14 , reward= -14.0
o o o o o o o o o o o o
o o o o o o o o o o o o
x o o o o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o o o o
o x o o o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o o o o
o o x o o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o o o o
o o o x o o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o o o o
o o o o x o o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o o o o
o o o o o x o o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o o o o
o o o o o o x o o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o o o o
o o o o o o o x o o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o o o o
o o o o o o o o x o o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o o o o
o o o o o o o o o x o o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o o o o
o o o o o o o o o o x o
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o o o o
o o o o o o o o o o o x
o C C C C C C C C C C T
o o o o o o o o o o o o
o o o o o o o o o o o o
o o o o o o o o o o o o
o C C C C C C C C C C x
test reward = -13.0