【2021-12-07】基于CartPole环境训练
import pakages
import os
import gym
from stable_baselines3 import PPO
from stable_baselines3.common.vec_env import DummyVecEnv
from stable_baselines3.common.evaluation import evaluate_policy
Load environments
environment_name = 'CartPole-v0'
env = gym.make(environment_name)
episodes = 5
for episode in range(1,episodes+1):
state = env.reset()
done = False
score = 0
while not done:
env.render()
action = env.action_space.sample()
n_state,reward,done,info = env.step(action)
score += reward
print('episode:{} score:{}'.format(episode,score))
env.close()
episode:1 score:19.0
episode:2 score:28.0
episode:3 score:23.0
episode:4 score:36.0
episode:5 score:26.0
env.reset()
train an agent
# make your directories first
log_path = os.path.join('Training','Logs')
log_path
'Training\\Logs'
env = gym.make(environment_name)
env = DummyVecEnv([lambda:env])
model = PPO('MlpPolicy',env,verbose = 1,tensorboard_log = log_path)
Using cpu device
model.learn(total_timesteps = 20000)
Logging to Training\Logs\PPO_4
-----------------------------
| time/ | |
| fps | 923 |
| iterations | 1 |
| time_elapsed | 2 |
| total_timesteps | 2048 |
-----------------------------
-----------------------------------------
| time/ | |
| fps | 952 |
| iterations | 2 |
| time_elapsed | 4 |
| total_timesteps | 4096 |
| train/ | |
| approx_kl | 0.008253496 |
| clip_fraction | 0.094 |
| clip_range | 0.2 |
| entropy_loss | -0.686 |
| explained_variance | 0.00826 |
| learning_rate | 0.0003 |
| loss | 5.01 |
| n_updates | 10 |
| policy_gradient_loss | -0.0152 |
| value_loss | 51.7 |
-----------------------------------------
-----------------------------------------
| time/ | |
| fps | 953 |
| iterations | 3 |
| time_elapsed | 6 |
| total_timesteps | 6144 |
| train/ | |
| approx_kl | 0.008827355 |
| clip_fraction | 0.0568 |
| clip_range | 0.2 |
| entropy_loss | -0.667 |
| explained_variance | 0.08 |
| learning_rate | 0.0003 |
| loss | 14.8 |
| n_updates | 20 |
| policy_gradient_loss | -0.0173 |
| value_loss | 38.1 |
-----------------------------------------
-----------------------------------------
| time/ | |
| fps | 953 |
| iterations | 4 |
| time_elapsed | 8 |
| total_timesteps | 8192 |
| train/ | |
| approx_kl | 0.013173467 |
| clip_fraction | 0.157 |
| clip_range | 0.2 |
| entropy_loss | -0.62 |
| explained_variance | 0.319 |
| learning_rate | 0.0003 |
| loss | 22.3 |
| n_updates | 30 |
| policy_gradient_loss | -0.0252 |
| value_loss | 43.1 |
-----------------------------------------
------------------------------------------
| time/ | |
| fps | 951 |
| iterations | 5 |
| time_elapsed | 10 |
| total_timesteps | 10240 |
| train/ | |
| approx_kl | 0.0064320453 |
| clip_fraction | 0.0451 |
| clip_range | 0.2 |
| entropy_loss | -0.613 |
| explained_variance | 0.22 |
| learning_rate | 0.0003 |
| loss | 19.4 |
| n_updates | 40 |
| policy_gradient_loss | -0.0138 |
| value_loss | 62.2 |
------------------------------------------
-----------------------------------------
| time/ | |
| fps | 952 |
| iterations | 6 |
| time_elapsed | 12 |
| total_timesteps | 12288 |
| train/ | |
| approx_kl | 0.005620899 |
| clip_fraction | 0.0605 |
| clip_range | 0.2 |
| entropy_loss | -0.596 |
| explained_variance | 0.529 |
| learning_rate | 0.0003 |
| loss | 17.2 |
| n_updates | 50 |
| policy_gradient_loss | -0.016 |
| value_loss | 52.4 |
-----------------------------------------
-----------------------------------------
| time/ | |
| fps | 951 |
| iterations | 7 |
| time_elapsed | 15 |
| total_timesteps | 14336 |
| train/ | |
| approx_kl | 0.010773982 |
| clip_fraction | 0.115 |
| clip_range | 0.2 |
| entropy_loss | -0.6 |
| explained_variance | 0.781 |
| learning_rate | 0.0003 |
| loss | 8.86 |
| n_updates | 60 |
| policy_gradient_loss | -0.0135 |
| value_loss | 41.3 |
-----------------------------------------
-----------------------------------------
| time/ | |
| fps | 955 |
| iterations | 8 |
| time_elapsed | 17 |
| total_timesteps | 16384 |
| train/ | |
| approx_kl | 0.007079646 |
| clip_fraction | 0.0451 |
| clip_range | 0.2 |
| entropy_loss | -0.562 |
| explained_variance | 0.264 |
| learning_rate | 0.0003 |
| loss | 55.1 |
| n_updates | 70 |
| policy_gradient_loss | -0.00813 |
| value_loss | 93.9 |
-----------------------------------------
------------------------------------------
| time/ | |
| fps | 957 |
| iterations | 9 |
| time_elapsed | 19 |
| total_timesteps | 18432 |
| train/ | |
| approx_kl | 0.0034424942 |
| clip_fraction | 0.0265 |
| clip_range | 0.2 |
| entropy_loss | -0.581 |
| explained_variance | 0.343 |
| learning_rate | 0.0003 |
| loss | 29.5 |
| n_updates | 80 |
| policy_gradient_loss | -0.00381 |
| value_loss | 76 |
------------------------------------------
------------------------------------------
| time/ | |
| fps | 955 |
| iterations | 10 |
| time_elapsed | 21 |
| total_timesteps | 20480 |
| train/ | |
| approx_kl | 0.0026272014 |
| clip_fraction | 0.0136 |
| clip_range | 0.2 |
| entropy_loss | -0.576 |
| explained_variance | 0.115 |
| learning_rate | 0.0003 |
| loss | 122 |
| n_updates | 90 |
| policy_gradient_loss | -0.00157 |
| value_loss | 107 |
------------------------------------------
Save and reload model
PPO_Path = os.path.join('Taining','Save Models','PPO_Model_CartPole')
model.save(PPO_Path)
del model
model = PPO.load(PPO_Path,env = env)
model.learn(total_timesteps = 10000)
Logging to Training\Logs\PPO_5
-----------------------------
| time/ | |
| fps | 1823 |
| iterations | 1 |
| time_elapsed | 1 |
| total_timesteps | 2048 |
-----------------------------
------------------------------------------
| time/ | |
| fps | 1237 |
| iterations | 2 |
| time_elapsed | 3 |
| total_timesteps | 4096 |
| train/ | |
| approx_kl | 0.0072634863 |
| clip_fraction | 0.0615 |
| clip_range | 0.2 |
| entropy_loss | -0.553 |
| explained_variance | 0.354 |
| learning_rate | 0.0003 |
| loss | 36.4 |
| n_updates | 110 |
| policy_gradient_loss | -0.00478 |
| value_loss | 112 |
------------------------------------------
------------------------------------------
| time/ | |
| fps | 1124 |
| iterations | 3 |
| time_elapsed | 5 |
| total_timesteps | 6144 |
| train/ | |
| approx_kl | 0.0051936586 |
| clip_fraction | 0.0519 |
| clip_range | 0.2 |
| entropy_loss | -0.563 |
| explained_variance | 0.576 |
| learning_rate | 0.0003 |
| loss | 121 |
| n_updates | 120 |
| policy_gradient_loss | -0.00451 |
| value_loss | 102 |
------------------------------------------
------------------------------------------
| time/ | |
| fps | 1074 |
| iterations | 4 |
| time_elapsed | 7 |
| total_timesteps | 8192 |
| train/ | |
| approx_kl | 0.0016217632 |
| clip_fraction | 0.00737 |
| clip_range | 0.2 |
| entropy_loss | -0.55 |
| explained_variance | 0.323 |
| learning_rate | 0.0003 |
| loss | 108 |
| n_updates | 130 |
| policy_gradient_loss | -0.00079 |
| value_loss | 116 |
------------------------------------------
------------------------------------------
| time/ | |
| fps | 1046 |
| iterations | 5 |
| time_elapsed | 9 |
| total_timesteps | 10240 |
| train/ | |
| approx_kl | 0.0019479269 |
| clip_fraction | 0.0187 |
| clip_range | 0.2 |
| entropy_loss | -0.544 |
| explained_variance | 0.659 |
| learning_rate | 0.0003 |
| loss | 64.3 |
| n_updates | 140 |
| policy_gradient_loss | -0.00166 |
| value_loss | 93 |
------------------------------------------
Evaluating
evaluate_policy(model,env,n_eval_episodes=10,render=True)
C:\ProgramData\Anaconda3\envs\uav\lib\site-packages\stable_baselines3\common\evaluation.py:69: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.
UserWarning,
(200.0, 0.0)
env.close()
Test model
episodes = 5
for episode in range(1,episodes+1):
obs = env.reset()
done = False
score = 0
while not done:
env.render()
action,_ = model.predict(obs) # predict
obs,reward,done,info = env.step(action)
score += reward
print('episode:{} score:{}'.format(episode,score))
# env.close()
episode:1 score:[200.]
episode:2 score:[200.]
episode:3 score:[200.]
episode:4 score:[200.]
episode:5 score:[200.]
env.close()
obs = env.reset()
model.predict(obs) #返回动作和下一个状态
(array([0], dtype=int64), None)
env.action_space.sample()
1
viewing logs in tensorboard
training_log_path = os.path.join(log_path,'PPO_1') #序号代表按照顺序训练的几次模型
!tensorboard --logdir={training_log_path} # localhost:6006
2021-12-07 15:28:43.632786: W tensorflow/stream_executor/platform/default/dso_loader.cc:55] Could not load dynamic library 'cudart64_100.dll'; dlerror: cudart64_100.dll not found
2021-12-07 15:28:43.633094: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
usage: tensorboard [-h] [--helpfull] [--logdir PATH] [--host ADDR]
[--port PORT] [--purge_orphaned_data BOOL] [--db URI]
[--db_import] [--inspect] [--version_tb] [--tag TAG]
[--event_file PATH] [--path_prefix PATH]
[--window_title TEXT] [--max_reload_threads COUNT]
[--reload_interval SECONDS] [--reload_task TYPE]
[--reload_multifile BOOL]
[--reload_multifile_inactive_secs SECONDS]
[--generic_data TYPE]
[--samples_per_plugin SAMPLES_PER_PLUGIN]
[--debugger_data_server_grpc_port PORT]
[--debugger_port PORT] [--master_tpu_unsecure_channel ADDR]
tensorboard: error: unrecognized arguments: # localhost:6006
add a callback to the training stage
回调是在训练过程的特定阶段调用的一组函数，可以使用回调来获取训练期间内部状态和模型统计信息的视图。
EarlyStopping
从字面上理解， EarlyStopping 就是提前终止训练，主要是为了防止过拟合。过拟合是机器学习从业者的噩梦，简单说，就是在训练数据集上精度很高，但在测试数据集上精度很低。解决过拟合有多种手段，有时还需要多种手段并用，其中一种方法是尽早终止训练过程。 EarlyStopping 函数有好几种度量参数，通过修改这些参数，可以控制合适的时机停止训练过程。下面是一些相关度量参数：
monitor：
监控的度量指标，比如:acc, val_acc, loss和val_loss等
min_delta：
监控值的最小变化。
例如，min_delta = 1表示如果监视值的绝对值变化小于1，则将停止训练过程
patience：
没有改善的epoch数，如果过了数个epoch之后结果没有改善，训练将停止
restore_best_weights：
如果要在停止后保存最佳权重，请将此参数设置为True
ModelCheckpoint
此回调用于在训练周期中保存模型检查点。保存检查点的作用在于保存训练中间的模型，下次在训练时，可以加载模型，而无需重新训练，减少训练时间。它有以一些相关参数：
filepath：
要保存模型的文件路径
monitor：
监控的度量指标，比如：
acc, val_acc, loss和val_loss等
save_best_only：
如果您不想最新的最佳模型被覆盖，请将此值设置为True
save_weights_only: 如果设为True，将只保存模型权重
mode：
auto，min或max。
例如，如果监控的度量指标是val_loss，并且想要最小化它，则设置mode =’min’。
period：
检查点之间的间隔（epoch数）。
LearningRateScheduler
在深度学习中，学习率的选择也是一件让人头疼的事情，值选择小了，可能会收敛缓慢，值选大了，可能会导致震荡，无法到达局部最优点。后来专家们设计出一种自适应的学习率，比如在训练开始阶段，选择比较大的学习率值，加速收敛，训练一段时间之后，选择小的学习率值，防止震荡。 LearningRateScheduler 用于定义学习率的变化策略，参数如下：
schedule：
一个函数，以epoch数（整数，从0开始计数）和当前学习速率，作为输入，返回一个新的学习速率作为输出（浮点数）。
verbose：
0：
静默模式，
1：
详细输出信息。
from stable_baselines3.common.callbacks import EvalCallback, StopTrainingOnRewardThreshold
save_path = os.path.join('Training','Saved Models')
stop_callback = StopTrainingOnRewardThreshold(reward_threshold=200,verbose=1) # 超过某个阈值的时候停止训练
eval_callback = EvalCallback(env, # 定期评估模型并保存最好的
callback_on_new_best=stop_callback,
best_model_save_path=save_path,
verbose=1)
model = PPO('MlpPolicy',env,verbose=1,tensorboard_log=log_path)
Using cpu device
model.learn(total_timesteps=20000,callback = eval_callback)
Logging to Training\Logs\PPO_6
-----------------------------
| time/ | |
| fps | 1845 |
| iterations | 1 |
| time_elapsed | 1 |
| total_timesteps | 2048 |
-----------------------------
-----------------------------------------
| time/ | |
| fps | 1271 |
| iterations | 2 |
| time_elapsed | 3 |
| total_timesteps | 4096 |
| train/ | |
| approx_kl | 0.008285078 |
| clip_fraction | 0.114 |
| clip_range | 0.2 |
| entropy_loss | -0.686 |
| explained_variance | -0.00127 |
| learning_rate | 0.0003 |
| loss | 6.67 |
| n_updates | 10 |
| policy_gradient_loss | -0.0168 |
| value_loss | 51.4 |
-----------------------------------------
-----------------------------------------
| time/ | |
| fps | 1157 |
| iterations | 3 |
| time_elapsed | 5 |
| total_timesteps | 6144 |
| train/ | |
| approx_kl | 0.008662224 |
| clip_fraction | 0.061 |
| clip_range | 0.2 |
| entropy_loss | -0.665 |
| explained_variance | 0.0965 |
| learning_rate | 0.0003 |
| loss | 9.36 |
| n_updates | 20 |
| policy_gradient_loss | -0.0157 |
| value_loss | 32.3 |
-----------------------------------------
-----------------------------------------
| time/ | |
| fps | 1081 |
| iterations | 4 |
| time_elapsed | 7 |
| total_timesteps | 8192 |
| train/ | |
| approx_kl | 0.009598523 |
| clip_fraction | 0.108 |
| clip_range | 0.2 |
| entropy_loss | -0.632 |
| explained_variance | 0.259 |
| learning_rate | 0.0003 |
| loss | 21.4 |
| n_updates | 30 |
| policy_gradient_loss | -0.0219 |
| value_loss | 45.9 |
-----------------------------------------
C:\ProgramData\Anaconda3\envs\uav\lib\site-packages\stable_baselines3\common\evaluation.py:69: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.
UserWarning,
Eval num_timesteps=10000, episode_reward=200.00 +/- 0.00
Episode length: 200.00 +/- 0.00
-----------------------------------------
| eval/ | |
| mean_ep_length | 200 |
| mean_reward | 200 |
| time/ | |
| total_timesteps | 10000 |
| train/ | |
| approx_kl | 0.008597443 |
| clip_fraction | 0.0816 |
| clip_range | 0.2 |
| entropy_loss | -0.614 |
| explained_variance | 0.339 |
| learning_rate | 0.0003 |
| loss | 26.3 |
| n_updates | 40 |
| policy_gradient_loss | -0.0168 |
| value_loss | 61.1 |
-----------------------------------------
New best mean reward!
Stopping training because the mean reward 200.00 is above the threshold 200
Changing Policies
net_arch = [dict(pi=[128,128,128,128],vf=[128,128,128,128])]
model = PPO('MlpPolicy',env,verbose=1,tensorboard_log = log_path,policy_kwargs={'net_arch':net_arch})
Using cpu device
model.learn(total_timesteps=20000,callback = eval_callback)
Logging to Training\Logs\PPO_7
-----------------------------
| time/ | |
| fps | 1329 |
| iterations | 1 |
| time_elapsed | 1 |
| total_timesteps | 2048 |
-----------------------------
-----------------------------------------
| time/ | |
| fps | 834 |
| iterations | 2 |
| time_elapsed | 4 |
| total_timesteps | 4096 |
| train/ | |
| approx_kl | 0.013661863 |
| clip_fraction | 0.205 |
| clip_range | 0.2 |
| entropy_loss | -0.681 |
| explained_variance | -0.0138 |
| learning_rate | 0.0003 |
| loss | 2.67 |
| n_updates | 10 |
| policy_gradient_loss | -0.0229 |
| value_loss | 17.3 |
-----------------------------------------
-----------------------------------------
| time/ | |
| fps | 731 |
| iterations | 3 |
| time_elapsed | 8 |
| total_timesteps | 6144 |
| train/ | |
| approx_kl | 0.014550514 |
| clip_fraction | 0.194 |
| clip_range | 0.2 |
| entropy_loss | -0.642 |
| explained_variance | 0.448 |
| learning_rate | 0.0003 |
| loss | 9.55 |
| n_updates | 20 |
| policy_gradient_loss | -0.0322 |
| value_loss | 23.7 |
-----------------------------------------
-----------------------------------------
| time/ | |
| fps | 693 |
| iterations | 4 |
| time_elapsed | 11 |
| total_timesteps | 8192 |
| train/ | |
| approx_kl | 0.012955727 |
| clip_fraction | 0.168 |
| clip_range | 0.2 |
| entropy_loss | -0.606 |
| explained_variance | 0.455 |
| learning_rate | 0.0003 |
| loss | 11 |
| n_updates | 30 |
| policy_gradient_loss | -0.0264 |
| value_loss | 37.8 |
-----------------------------------------
Eval num_timesteps=10000, episode_reward=200.00 +/- 0.00
Episode length: 200.00 +/- 0.00
-----------------------------------------
| eval/ | |
| mean_ep_length | 200 |
| mean_reward | 200 |
| time/ | |
| total_timesteps | 10000 |
| train/ | |
| approx_kl | 0.008317121 |
| clip_fraction | 0.101 |
| clip_range | 0.2 |
| entropy_loss | -0.565 |
| explained_variance | 0.42 |
| learning_rate | 0.0003 |
| loss | 18.8 |
| n_updates | 40 |
| policy_gradient_loss | -0.0152 |
| value_loss | 50.3 |
-----------------------------------------
------------------------------
| time/ | |
| fps | 647 |
| iterations | 5 |
| time_elapsed | 15 |
| total_timesteps | 10240 |
------------------------------
------------------------------------------
| time/ | |
| fps | 639 |
| iterations | 6 |
| time_elapsed | 19 |
| total_timesteps | 12288 |
| train/ | |
| approx_kl | 0.0050881114 |
| clip_fraction | 0.0684 |
| clip_range | 0.2 |
| entropy_loss | -0.55 |
| explained_variance | 0.465 |
| learning_rate | 0.0003 |
| loss | 33.2 |
| n_updates | 50 |
| policy_gradient_loss | -0.00452 |
| value_loss | 55.3 |
------------------------------------------
-----------------------------------------
| time/ | |
| fps | 634 |
| iterations | 7 |
| time_elapsed | 22 |
| total_timesteps | 14336 |
| train/ | |
| approx_kl | 0.003173225 |
| clip_fraction | 0.0269 |
| clip_range | 0.2 |
| entropy_loss | -0.552 |
| explained_variance | 0.433 |
| learning_rate | 0.0003 |
| loss | 15.8 |
| n_updates | 60 |
| policy_gradient_loss | -0.00311 |
| value_loss | 52.5 |
-----------------------------------------
-----------------------------------------
| time/ | |
| fps | 627 |
| iterations | 8 |
| time_elapsed | 26 |
| total_timesteps | 16384 |
| train/ | |
| approx_kl | 0.006802251 |
| clip_fraction | 0.0502 |
| clip_range | 0.2 |
| entropy_loss | -0.538 |
| explained_variance | 0.479 |
| learning_rate | 0.0003 |
| loss | 23.9 |
| n_updates | 70 |
| policy_gradient_loss | -0.0038 |
| value_loss | 67.2 |
-----------------------------------------
-----------------------------------------
| time/ | |
| fps | 620 |
| iterations | 9 |
| time_elapsed | 29 |
| total_timesteps | 18432 |
| train/ | |
| approx_kl | 0.005398168 |
| clip_fraction | 0.0543 |
| clip_range | 0.2 |
| entropy_loss | -0.524 |
| explained_variance | 0.764 |
| learning_rate | 0.0003 |
| loss | 13.9 |
| n_updates | 80 |
| policy_gradient_loss | -0.00316 |
| value_loss | 39.7 |
-----------------------------------------
Eval num_timesteps=20000, episode_reward=197.60 +/- 4.80
Episode length: 197.60 +/- 4.80
------------------------------------------
| eval/ | |
| mean_ep_length | 198 |
| mean_reward | 198 |
| time/ | |
| total_timesteps | 20000 |
| train/ | |
| approx_kl | 0.0038945822 |
| clip_fraction | 0.068 |
| clip_range | 0.2 |
| entropy_loss | -0.524 |
| explained_variance | 0.904 |
| learning_rate | 0.0003 |
| loss | 6.48 |
| n_updates | 90 |
| policy_gradient_loss | -0.00412 |
| value_loss | 20.7 |
------------------------------------------
------------------------------
| time/ | |
| fps | 598 |
| iterations | 10 |
| time_elapsed | 34 |
| total_timesteps | 20480 |
------------------------------
Using an Alternate Algorithm
from stable_baselines3 import DQN
model = DQN('MlpPolicy',env,verbose=1,tensorboard_log = log_path)
Using cpu device
model.learn(total_timesteps=20000)
Logging to Training\Logs\DQN_1
----------------------------------
| rollout/ | |
| exploration_rate | 0.954 |
| time/ | |
| episodes | 4 |
| fps | 2338 |
| time_elapsed | 0 |
| total_timesteps | 97 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.926 |
| time/ | |
| episodes | 8 |
| fps | 2747 |
| time_elapsed | 0 |
| total_timesteps | 155 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.896 |
| time/ | |
| episodes | 12 |
| fps | 3068 |
| time_elapsed | 0 |
| total_timesteps | 219 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.867 |
| time/ | |
| episodes | 16 |
| fps | 3088 |
| time_elapsed | 0 |
| total_timesteps | 279 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.812 |
| time/ | |
| episodes | 20 |
| fps | 3549 |
| time_elapsed | 0 |
| total_timesteps | 395 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.781 |
| time/ | |
| episodes | 24 |
| fps | 3740 |
| time_elapsed | 0 |
| total_timesteps | 461 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.743 |
| time/ | |
| episodes | 28 |
| fps | 3811 |
| time_elapsed | 0 |
| total_timesteps | 542 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.721 |
| time/ | |
| episodes | 32 |
| fps | 3814 |
| time_elapsed | 0 |
| total_timesteps | 588 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.684 |
| time/ | |
| episodes | 36 |
| fps | 3979 |
| time_elapsed | 0 |
| total_timesteps | 665 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.635 |
| time/ | |
| episodes | 40 |
| fps | 3937 |
| time_elapsed | 0 |
| total_timesteps | 768 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.595 |
| time/ | |
| episodes | 44 |
| fps | 3949 |
| time_elapsed | 0 |
| total_timesteps | 853 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.56 |
| time/ | |
| episodes | 48 |
| fps | 4044 |
| time_elapsed | 0 |
| total_timesteps | 926 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.529 |
| time/ | |
| episodes | 52 |
| fps | 4100 |
| time_elapsed | 0 |
| total_timesteps | 992 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.461 |
| time/ | |
| episodes | 56 |
| fps | 4384 |
| time_elapsed | 0 |
| total_timesteps | 1135 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.423 |
| time/ | |
| episodes | 60 |
| fps | 4486 |
| time_elapsed | 0 |
| total_timesteps | 1215 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.388 |
| time/ | |
| episodes | 64 |
| fps | 4525 |
| time_elapsed | 0 |
| total_timesteps | 1289 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.358 |
| time/ | |
| episodes | 68 |
| fps | 4555 |
| time_elapsed | 0 |
| total_timesteps | 1352 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.3 |
| time/ | |
| episodes | 72 |
| fps | 4709 |
| time_elapsed | 0 |
| total_timesteps | 1473 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.257 |
| time/ | |
| episodes | 76 |
| fps | 4802 |
| time_elapsed | 0 |
| total_timesteps | 1564 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.196 |
| time/ | |
| episodes | 80 |
| fps | 4952 |
| time_elapsed | 0 |
| total_timesteps | 1692 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.157 |
| time/ | |
| episodes | 84 |
| fps | 4935 |
| time_elapsed | 0 |
| total_timesteps | 1775 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.119 |
| time/ | |
| episodes | 88 |
| fps | 4992 |
| time_elapsed | 0 |
| total_timesteps | 1855 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.0814 |
| time/ | |
| episodes | 92 |
| fps | 5016 |
| time_elapsed | 0 |
| total_timesteps | 1934 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 96 |
| fps | 5051 |
| time_elapsed | 0 |
| total_timesteps | 2013 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 100 |
| fps | 5065 |
| time_elapsed | 0 |
| total_timesteps | 2064 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 104 |
| fps | 5108 |
| time_elapsed | 0 |
| total_timesteps | 2148 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 108 |
| fps | 5108 |
| time_elapsed | 0 |
| total_timesteps | 2204 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 112 |
| fps | 5133 |
| time_elapsed | 0 |
| total_timesteps | 2266 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 116 |
| fps | 5128 |
| time_elapsed | 0 |
| total_timesteps | 2325 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 120 |
| fps | 5229 |
| time_elapsed | 0 |
| total_timesteps | 2449 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 124 |
| fps | 5227 |
| time_elapsed | 0 |
| total_timesteps | 2516 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 128 |
| fps | 5208 |
| time_elapsed | 0 |
| total_timesteps | 2595 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 132 |
| fps | 5261 |
| time_elapsed | 0 |
| total_timesteps | 2695 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 136 |
| fps | 5301 |
| time_elapsed | 0 |
| total_timesteps | 2816 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 140 |
| fps | 5295 |
| time_elapsed | 0 |
| total_timesteps | 2892 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 144 |
| fps | 5303 |
| time_elapsed | 0 |
| total_timesteps | 2960 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 148 |
| fps | 5350 |
| time_elapsed | 0 |
| total_timesteps | 3050 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 152 |
| fps | 5315 |
| time_elapsed | 0 |
| total_timesteps | 3152 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 156 |
| fps | 5344 |
| time_elapsed | 0 |
| total_timesteps | 3260 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 160 |
| fps | 5335 |
| time_elapsed | 0 |
| total_timesteps | 3334 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 164 |
| fps | 5377 |
| time_elapsed | 0 |
| total_timesteps | 3462 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 168 |
| fps | 5378 |
| time_elapsed | 0 |
| total_timesteps | 3549 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 172 |
| fps | 5377 |
| time_elapsed | 0 |
| total_timesteps | 3634 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 176 |
| fps | 5385 |
| time_elapsed | 0 |
| total_timesteps | 3693 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 180 |
| fps | 5365 |
| time_elapsed | 0 |
| total_timesteps | 3765 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 184 |
| fps | 5369 |
| time_elapsed | 0 |
| total_timesteps | 3843 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 188 |
| fps | 5387 |
| time_elapsed | 0 |
| total_timesteps | 3915 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 192 |
| fps | 5384 |
| time_elapsed | 0 |
| total_timesteps | 4009 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 196 |
| fps | 5414 |
| time_elapsed | 0 |
| total_timesteps | 4129 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 200 |
| fps | 5435 |
| time_elapsed | 0 |
| total_timesteps | 4215 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 204 |
| fps | 5417 |
| time_elapsed | 0 |
| total_timesteps | 4266 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 208 |
| fps | 5430 |
| time_elapsed | 0 |
| total_timesteps | 4347 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 212 |
| fps | 5442 |
| time_elapsed | 0 |
| total_timesteps | 4438 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 216 |
| fps | 5445 |
| time_elapsed | 0 |
| total_timesteps | 4511 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 220 |
| fps | 5426 |
| time_elapsed | 0 |
| total_timesteps | 4587 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 224 |
| fps | 5430 |
| time_elapsed | 0 |
| total_timesteps | 4666 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 228 |
| fps | 5417 |
| time_elapsed | 0 |
| total_timesteps | 4725 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 232 |
| fps | 5426 |
| time_elapsed | 0 |
| total_timesteps | 4820 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 236 |
| fps | 5443 |
| time_elapsed | 0 |
| total_timesteps | 4922 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 240 |
| fps | 5444 |
| time_elapsed | 0 |
| total_timesteps | 5037 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 244 |
| fps | 5471 |
| time_elapsed | 0 |
| total_timesteps | 5138 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 248 |
| fps | 5502 |
| time_elapsed | 0 |
| total_timesteps | 5255 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 252 |
| fps | 5516 |
| time_elapsed | 0 |
| total_timesteps | 5367 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 256 |
| fps | 5496 |
| time_elapsed | 0 |
| total_timesteps | 5419 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 260 |
| fps | 5480 |
| time_elapsed | 1 |
| total_timesteps | 5502 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 264 |
| fps | 5496 |
| time_elapsed | 1 |
| total_timesteps | 5595 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 268 |
| fps | 5506 |
| time_elapsed | 1 |
| total_timesteps | 5682 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 272 |
| fps | 5480 |
| time_elapsed | 1 |
| total_timesteps | 5753 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 276 |
| fps | 5494 |
| time_elapsed | 1 |
| total_timesteps | 5905 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 280 |
| fps | 5490 |
| time_elapsed | 1 |
| total_timesteps | 6038 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 284 |
| fps | 5487 |
| time_elapsed | 1 |
| total_timesteps | 6127 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 288 |
| fps | 5515 |
| time_elapsed | 1 |
| total_timesteps | 6241 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 292 |
| fps | 5503 |
| time_elapsed | 1 |
| total_timesteps | 6304 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 296 |
| fps | 5518 |
| time_elapsed | 1 |
| total_timesteps | 6382 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 300 |
| fps | 5541 |
| time_elapsed | 1 |
| total_timesteps | 6481 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 304 |
| fps | 5573 |
| time_elapsed | 1 |
| total_timesteps | 6601 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 308 |
| fps | 5585 |
| time_elapsed | 1 |
| total_timesteps | 6688 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 312 |
| fps | 5578 |
| time_elapsed | 1 |
| total_timesteps | 6752 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 316 |
| fps | 5581 |
| time_elapsed | 1 |
| total_timesteps | 6850 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 320 |
| fps | 5606 |
| time_elapsed | 1 |
| total_timesteps | 6954 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 324 |
| fps | 5608 |
| time_elapsed | 1 |
| total_timesteps | 7012 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 328 |
| fps | 5617 |
| time_elapsed | 1 |
| total_timesteps | 7107 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 332 |
| fps | 5650 |
| time_elapsed | 1 |
| total_timesteps | 7245 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 336 |
| fps | 5652 |
| time_elapsed | 1 |
| total_timesteps | 7338 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 340 |
| fps | 5651 |
| time_elapsed | 1 |
| total_timesteps | 7415 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 344 |
| fps | 5637 |
| time_elapsed | 1 |
| total_timesteps | 7476 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 348 |
| fps | 5634 |
| time_elapsed | 1 |
| total_timesteps | 7567 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 352 |
| fps | 5642 |
| time_elapsed | 1 |
| total_timesteps | 7673 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 356 |
| fps | 5643 |
| time_elapsed | 1 |
| total_timesteps | 7753 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 360 |
| fps | 5645 |
| time_elapsed | 1 |
| total_timesteps | 7824 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 364 |
| fps | 5636 |
| time_elapsed | 1 |
| total_timesteps | 7890 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 368 |
| fps | 5646 |
| time_elapsed | 1 |
| total_timesteps | 7977 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 372 |
| fps | 5660 |
| time_elapsed | 1 |
| total_timesteps | 8065 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 376 |
| fps | 5652 |
| time_elapsed | 1 |
| total_timesteps | 8127 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 380 |
| fps | 5658 |
| time_elapsed | 1 |
| total_timesteps | 8226 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 384 |
| fps | 5661 |
| time_elapsed | 1 |
| total_timesteps | 8303 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 388 |
| fps | 5662 |
| time_elapsed | 1 |
| total_timesteps | 8389 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 392 |
| fps | 5662 |
| time_elapsed | 1 |
| total_timesteps | 8469 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 396 |
| fps | 5677 |
| time_elapsed | 1 |
| total_timesteps | 8559 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 400 |
| fps | 5685 |
| time_elapsed | 1 |
| total_timesteps | 8633 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 404 |
| fps | 5687 |
| time_elapsed | 1 |
| total_timesteps | 8739 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 408 |
| fps | 5681 |
| time_elapsed | 1 |
| total_timesteps | 8792 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 412 |
| fps | 5669 |
| time_elapsed | 1 |
| total_timesteps | 8869 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 416 |
| fps | 5679 |
| time_elapsed | 1 |
| total_timesteps | 8987 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 420 |
| fps | 5681 |
| time_elapsed | 1 |
| total_timesteps | 9047 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 424 |
| fps | 5675 |
| time_elapsed | 1 |
| total_timesteps | 9128 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 428 |
| fps | 5684 |
| time_elapsed | 1 |
| total_timesteps | 9211 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 432 |
| fps | 5680 |
| time_elapsed | 1 |
| total_timesteps | 9288 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 436 |
| fps | 5695 |
| time_elapsed | 1 |
| total_timesteps | 9410 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 440 |
| fps | 5707 |
| time_elapsed | 1 |
| total_timesteps | 9526 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 444 |
| fps | 5719 |
| time_elapsed | 1 |
| total_timesteps | 9626 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 448 |
| fps | 5744 |
| time_elapsed | 1 |
| total_timesteps | 9765 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 452 |
| fps | 5760 |
| time_elapsed | 1 |
| total_timesteps | 9867 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 456 |
| fps | 5761 |
| time_elapsed | 1 |
| total_timesteps | 9972 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 460 |
| fps | 5763 |
| time_elapsed | 1 |
| total_timesteps | 10085 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 464 |
| fps | 5764 |
| time_elapsed | 1 |
| total_timesteps | 10144 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 468 |
| fps | 5755 |
| time_elapsed | 1 |
| total_timesteps | 10209 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 472 |
| fps | 5754 |
| time_elapsed | 1 |
| total_timesteps | 10328 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 476 |
| fps | 5768 |
| time_elapsed | 1 |
| total_timesteps | 10446 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 480 |
| fps | 5773 |
| time_elapsed | 1 |
| total_timesteps | 10518 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 484 |
| fps | 5775 |
| time_elapsed | 1 |
| total_timesteps | 10601 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 488 |
| fps | 5778 |
| time_elapsed | 1 |
| total_timesteps | 10682 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 492 |
| fps | 5775 |
| time_elapsed | 1 |
| total_timesteps | 10786 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 496 |
| fps | 5778 |
| time_elapsed | 1 |
| total_timesteps | 10866 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 500 |
| fps | 5780 |
| time_elapsed | 1 |
| total_timesteps | 10934 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 504 |
| fps | 5769 |
| time_elapsed | 1 |
| total_timesteps | 11011 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 508 |
| fps | 5754 |
| time_elapsed | 1 |
| total_timesteps | 11080 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 512 |
| fps | 5756 |
| time_elapsed | 1 |
| total_timesteps | 11159 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 516 |
| fps | 5753 |
| time_elapsed | 1 |
| total_timesteps | 11228 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 520 |
| fps | 5753 |
| time_elapsed | 1 |
| total_timesteps | 11319 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 524 |
| fps | 5747 |
| time_elapsed | 1 |
| total_timesteps | 11439 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 528 |
| fps | 5758 |
| time_elapsed | 2 |
| total_timesteps | 11575 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 532 |
| fps | 5750 |
| time_elapsed | 2 |
| total_timesteps | 11639 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 536 |
| fps | 5745 |
| time_elapsed | 2 |
| total_timesteps | 11710 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 540 |
| fps | 5742 |
| time_elapsed | 2 |
| total_timesteps | 11789 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 544 |
| fps | 5745 |
| time_elapsed | 2 |
| total_timesteps | 11899 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 548 |
| fps | 5770 |
| time_elapsed | 2 |
| total_timesteps | 12060 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 552 |
| fps | 5771 |
| time_elapsed | 2 |
| total_timesteps | 12142 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 556 |
| fps | 5778 |
| time_elapsed | 2 |
| total_timesteps | 12256 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 560 |
| fps | 5775 |
| time_elapsed | 2 |
| total_timesteps | 12313 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 564 |
| fps | 5778 |
| time_elapsed | 2 |
| total_timesteps | 12428 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 568 |
| fps | 5794 |
| time_elapsed | 2 |
| total_timesteps | 12556 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 572 |
| fps | 5808 |
| time_elapsed | 2 |
| total_timesteps | 12685 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 576 |
| fps | 5799 |
| time_elapsed | 2 |
| total_timesteps | 12763 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 580 |
| fps | 5789 |
| time_elapsed | 2 |
| total_timesteps | 12838 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 584 |
| fps | 5783 |
| time_elapsed | 2 |
| total_timesteps | 12900 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 588 |
| fps | 5780 |
| time_elapsed | 2 |
| total_timesteps | 13004 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 592 |
| fps | 5777 |
| time_elapsed | 2 |
| total_timesteps | 13090 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 596 |
| fps | 5773 |
| time_elapsed | 2 |
| total_timesteps | 13172 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 600 |
| fps | 5782 |
| time_elapsed | 2 |
| total_timesteps | 13274 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 604 |
| fps | 5788 |
| time_elapsed | 2 |
| total_timesteps | 13351 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 608 |
| fps | 5790 |
| time_elapsed | 2 |
| total_timesteps | 13431 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 612 |
| fps | 5788 |
| time_elapsed | 2 |
| total_timesteps | 13507 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 616 |
| fps | 5788 |
| time_elapsed | 2 |
| total_timesteps | 13621 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 620 |
| fps | 5788 |
| time_elapsed | 2 |
| total_timesteps | 13726 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 624 |
| fps | 5787 |
| time_elapsed | 2 |
| total_timesteps | 13810 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 628 |
| fps | 5800 |
| time_elapsed | 2 |
| total_timesteps | 13922 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 632 |
| fps | 5807 |
| time_elapsed | 2 |
| total_timesteps | 14014 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 636 |
| fps | 5812 |
| time_elapsed | 2 |
| total_timesteps | 14130 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 640 |
| fps | 5811 |
| time_elapsed | 2 |
| total_timesteps | 14226 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 644 |
| fps | 5814 |
| time_elapsed | 2 |
| total_timesteps | 14298 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 648 |
| fps | 5814 |
| time_elapsed | 2 |
| total_timesteps | 14368 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 652 |
| fps | 5819 |
| time_elapsed | 2 |
| total_timesteps | 14449 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 656 |
| fps | 5828 |
| time_elapsed | 2 |
| total_timesteps | 14554 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 660 |
| fps | 5825 |
| time_elapsed | 2 |
| total_timesteps | 14610 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 664 |
| fps | 5829 |
| time_elapsed | 2 |
| total_timesteps | 14712 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 668 |
| fps | 5827 |
| time_elapsed | 2 |
| total_timesteps | 14788 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 672 |
| fps | 5823 |
| time_elapsed | 2 |
| total_timesteps | 14849 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 676 |
| fps | 5826 |
| time_elapsed | 2 |
| total_timesteps | 14919 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 680 |
| fps | 5816 |
| time_elapsed | 2 |
| total_timesteps | 14969 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 684 |
| fps | 5812 |
| time_elapsed | 2 |
| total_timesteps | 15018 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 688 |
| fps | 5812 |
| time_elapsed | 2 |
| total_timesteps | 15105 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 692 |
| fps | 5818 |
| time_elapsed | 2 |
| total_timesteps | 15195 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 696 |
| fps | 5821 |
| time_elapsed | 2 |
| total_timesteps | 15280 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 700 |
| fps | 5817 |
| time_elapsed | 2 |
| total_timesteps | 15355 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 704 |
| fps | 5819 |
| time_elapsed | 2 |
| total_timesteps | 15443 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 708 |
| fps | 5817 |
| time_elapsed | 2 |
| total_timesteps | 15494 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 712 |
| fps | 5816 |
| time_elapsed | 2 |
| total_timesteps | 15550 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 716 |
| fps | 5813 |
| time_elapsed | 2 |
| total_timesteps | 15616 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 720 |
| fps | 5815 |
| time_elapsed | 2 |
| total_timesteps | 15687 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 724 |
| fps | 5818 |
| time_elapsed | 2 |
| total_timesteps | 15793 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 728 |
| fps | 5825 |
| time_elapsed | 2 |
| total_timesteps | 15889 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 732 |
| fps | 5826 |
| time_elapsed | 2 |
| total_timesteps | 15990 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 736 |
| fps | 5821 |
| time_elapsed | 2 |
| total_timesteps | 16116 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 740 |
| fps | 5815 |
| time_elapsed | 2 |
| total_timesteps | 16184 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 744 |
| fps | 5812 |
| time_elapsed | 2 |
| total_timesteps | 16242 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 748 |
| fps | 5814 |
| time_elapsed | 2 |
| total_timesteps | 16315 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 752 |
| fps | 5805 |
| time_elapsed | 2 |
| total_timesteps | 16390 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 756 |
| fps | 5799 |
| time_elapsed | 2 |
| total_timesteps | 16465 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 760 |
| fps | 5802 |
| time_elapsed | 2 |
| total_timesteps | 16559 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 764 |
| fps | 5802 |
| time_elapsed | 2 |
| total_timesteps | 16659 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 768 |
| fps | 5808 |
| time_elapsed | 2 |
| total_timesteps | 16763 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 772 |
| fps | 5816 |
| time_elapsed | 2 |
| total_timesteps | 16901 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 776 |
| fps | 5811 |
| time_elapsed | 2 |
| total_timesteps | 16968 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 780 |
| fps | 5819 |
| time_elapsed | 2 |
| total_timesteps | 17091 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 784 |
| fps | 5825 |
| time_elapsed | 2 |
| total_timesteps | 17200 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 788 |
| fps | 5823 |
| time_elapsed | 2 |
| total_timesteps | 17266 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 792 |
| fps | 5831 |
| time_elapsed | 2 |
| total_timesteps | 17376 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 796 |
| fps | 5825 |
| time_elapsed | 2 |
| total_timesteps | 17468 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 800 |
| fps | 5827 |
| time_elapsed | 3 |
| total_timesteps | 17543 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 804 |
| fps | 5837 |
| time_elapsed | 3 |
| total_timesteps | 17651 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 808 |
| fps | 5842 |
| time_elapsed | 3 |
| total_timesteps | 17770 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 812 |
| fps | 5836 |
| time_elapsed | 3 |
| total_timesteps | 17844 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 816 |
| fps | 5836 |
| time_elapsed | 3 |
| total_timesteps | 17950 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 820 |
| fps | 5841 |
| time_elapsed | 3 |
| total_timesteps | 18035 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 824 |
| fps | 5850 |
| time_elapsed | 3 |
| total_timesteps | 18137 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 828 |
| fps | 5854 |
| time_elapsed | 3 |
| total_timesteps | 18226 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 832 |
| fps | 5856 |
| time_elapsed | 3 |
| total_timesteps | 18304 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 836 |
| fps | 5850 |
| time_elapsed | 3 |
| total_timesteps | 18372 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 840 |
| fps | 5855 |
| time_elapsed | 3 |
| total_timesteps | 18470 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 844 |
| fps | 5858 |
| time_elapsed | 3 |
| total_timesteps | 18538 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 848 |
| fps | 5857 |
| time_elapsed | 3 |
| total_timesteps | 18592 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 852 |
| fps | 5861 |
| time_elapsed | 3 |
| total_timesteps | 18668 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 856 |
| fps | 5862 |
| time_elapsed | 3 |
| total_timesteps | 18730 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 860 |
| fps | 5868 |
| time_elapsed | 3 |
| total_timesteps | 18814 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 864 |
| fps | 5859 |
| time_elapsed | 3 |
| total_timesteps | 18886 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 868 |
| fps | 5857 |
| time_elapsed | 3 |
| total_timesteps | 18954 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 872 |
| fps | 5858 |
| time_elapsed | 3 |
| total_timesteps | 19044 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 876 |
| fps | 5872 |
| time_elapsed | 3 |
| total_timesteps | 19196 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 880 |
| fps | 5872 |
| time_elapsed | 3 |
| total_timesteps | 19279 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 884 |
| fps | 5879 |
| time_elapsed | 3 |
| total_timesteps | 19396 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 888 |
| fps | 5888 |
| time_elapsed | 3 |
| total_timesteps | 19507 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 892 |
| fps | 5889 |
| time_elapsed | 3 |
| total_timesteps | 19575 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 896 |
| fps | 5890 |
| time_elapsed | 3 |
| total_timesteps | 19672 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 900 |
| fps | 5881 |
| time_elapsed | 3 |
| total_timesteps | 19771 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 904 |
| fps | 5879 |
| time_elapsed | 3 |
| total_timesteps | 19875 |
----------------------------------
----------------------------------
| rollout/ | |
| exploration_rate | 0.05 |
| time/ | |
| episodes | 908 |
| fps | 5873 |
| time_elapsed | 3 |
| total_timesteps | 19944 |
----------------------------------