使用DQN问题整理
一、原始github地址
https://github.com/devsisters/DQN-tensorflow
二、安装环境
- python 3.7.13
- gym 0.15.7
- tqdm 4.61.2
- tensorflow 1.15.0
三. 相关问题
问题1
Traceback (most recent call last):
File "/home/user/code/python/DQN-tf1.15/main.py", line 3, in <module>
import tensorflow as tf
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow/__init__.py", line 99, in <module>
from tensorflow_core import *
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/__init__.py", line 28, in <module>
from tensorflow.python import pywrap_tensorflow # pylint: disable=unused-import
File "" , line 1019, in _handle_fromlist
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow/__init__.py", line 50, in __getattr__
module = self._load()
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow/__init__.py", line 44, in _load
module = _importlib.import_module(self.__name__)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/python/__init__.py", line 52, in <module>
from tensorflow.core.framework.graph_pb2 import *
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/core/framework/graph_pb2.py", line 16, in <module>
from tensorflow.core.framework import node_def_pb2 as tensorflow_dot_core_dot_framework_dot_node__def__pb2
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/core/framework/node_def_pb2.py", line 16, in <module>
from tensorflow.core.framework import attr_value_pb2 as tensorflow_dot_core_dot_framework_dot_attr__value__pb2
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/core/framework/attr_value_pb2.py", line 16, in <module>
from tensorflow.core.framework import tensor_pb2 as tensorflow_dot_core_dot_framework_dot_tensor__pb2
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/core/framework/tensor_pb2.py", line 16, in <module>
from tensorflow.core.framework import resource_handle_pb2 as tensorflow_dot_core_dot_framework_dot_resource__handle__pb2
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/core/framework/resource_handle_pb2.py", line 16, in <module>
from tensorflow.core.framework import tensor_shape_pb2 as tensorflow_dot_core_dot_framework_dot_tensor__shape__pb2
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/core/framework/tensor_shape_pb2.py", line 42, in <module>
serialized_options=None, file=DESCRIPTOR),
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/google/protobuf/descriptor.py", line 560, in __new__
_message.Message._CheckCalledFromGeneratedFile()
TypeError: Descriptors cannot not be created directly.
If this call came from a _pb2.py file, your generated code is out of date and must be regenerated with protoc >= 3.19.0.
If you cannot immediately regenerate your protos, some other possible workarounds are:
1. Downgrade the protobuf package to 3.20.x or lower.
2. Set PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python (but this will use pure-Python parsing and will be much slower).
More information: https://developers.google.com/protocol-buffers/docs/news/2022-05-06#python-updates
Process finished with exit code 1
解决方案
pip install 'protobuf~=3.19.0'
问题2
Traceback (most recent call last):
File "/home/user/code/python/DQN-tf1.15/dqn/utils.py", line 11, in <module>
from scipy.misc import imresize
ModuleNotFoundError: No module named 'scipy'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/user/code/python/DQN-tf1.15/main.py", line 5, in <module>
from dqn.agent import Agent
File "/home/user/code/python/DQN-tf1.15/dqn/agent.py", line 11, in <module>
from .replay_memory import ReplayMemory
File "/home/user/code/python/DQN-tf1.15/dqn/replay_memory.py", line 8, in <module>
from .utils import save_npy, load_npy
File "/home/user/code/python/DQN-tf1.15/dqn/utils.py", line 13, in <module>
import cv2
ModuleNotFoundError: No module named 'cv2'
Process finished with exit code 1
解决方案
pip install opencv-python
问题3
Traceback (most recent call last):
File "/home/user/code/python/DQN-tf1.15/main.py", line 70, in <module>
tf.app.run()
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/python/platform/app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 308, in run
_run_main(main, args)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 254, in _run_main
sys.exit(main(argv))
File "/home/user/code/python/DQN-tf1.15/main.py", line 49, in main
config = get_config(FLAGS) or FLAGS
File "/home/user/code/python/DQN-tf1.15/config.py", line 58, in get_config
for k, v in FLAGS.__dict__['__flags'].items():
KeyError: '__flags'
Process finished with exit code 1
解决方案
将
for k, v in FLAGS.__dict__['__flags'].items():
替换为
for k in FLAGS:
v = FLAGS[k].value
问题4
Traceback (most recent call last):
File "/home/user/code/python/DQN-tf1.15/main.py", line 70, in <module>
tf.app.run()
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/python/platform/app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 308, in run
_run_main(main, args)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 254, in _run_main
sys.exit(main(argv))
File "/home/user/code/python/DQN-tf1.15/main.py", line 54, in main
env = GymEnvironment(config)
File "/home/user/code/python/DQN-tf1.15/dqn/environment.py", line 67, in __init__
super(GymEnvironment, self).__init__(config)
File "/home/user/code/python/DQN-tf1.15/dqn/environment.py", line 8, in __init__
self.env = gym.make(config.env_name)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/gym/envs/registration.py", line 569, in make
_check_version_exists(ns, name, version)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/gym/envs/registration.py", line 219, in _check_version_exists
_check_name_exists(ns, name)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/gym/envs/registration.py", line 198, in _check_name_exists
f"Environment {name} doesn't exist{namespace_msg}. {suggestion_msg}"
gym.error.NameNotFound: Environment Breakout doesn't exist.
Process finished with exit code 1
解决方案
pip install ale-py
pip install gym[accept-rom-license]
问题5
Traceback (most recent call last):
File "/home/user/code/python/DQN-tf1.15/main.py", line 70, in <module>
tf.app.run()
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/python/platform/app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 308, in run
_run_main(main, args)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 254, in _run_main
sys.exit(main(argv))
File "/home/user/code/python/DQN-tf1.15/main.py", line 57, in main
raise Exception("use_gpu flag is true when no GPUs are available")
Exception: use_gpu flag is true when no GPUs are available
Process finished with exit code 1
解决方案
flags.DEFINE_boolean('use_gpu', True, 'Whether to use gpu or not')
## 修改为
flags.DEFINE_boolean('use_gpu', False, 'Whether to use gpu or not')
问题6
Traceback (most recent call last):
File "/home/user/code/python/DQN-tf1.15/main.py", line 70, in <module>
tf.app.run()
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/python/platform/app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 308, in run
_run_main(main, args)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 254, in _run_main
sys.exit(main(argv))
File "/home/user/code/python/DQN-tf1.15/main.py", line 62, in main
agent = Agent(config, env, sess)
File "/home/user/code/python/DQN-tf1.15/dqn/agent.py", line 30, in __init__
self.build_dqn()
File "/home/user/code/python/DQN-tf1.15/dqn/agent.py", line 201, in build_dqn
self.l3_flat = tf.reshape(self.l3, [-1, reduce(lambda x, y: x * y, shape[1:])])
NameError: name 'reduce' is not defined
Process finished with exit code 1
解决方案
from functools import reduce
问题7
Traceback (most recent call last):
File "/home/user/code/python/DQN-tf1.15/main.py", line 70, in <module>
tf.app.run()
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/python/platform/app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 308, in run
_run_main(main, args)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 254, in _run_main
sys.exit(main(argv))
File "/home/user/code/python/DQN-tf1.15/main.py", line 62, in main
agent = Agent(config, env, sess)
File "/home/user/code/python/DQN-tf1.15/dqn/agent.py", line 31, in __init__
self.build_dqn()
File "/home/user/code/python/DQN-tf1.15/dqn/agent.py", line 228, in build_dqn
for idx in xrange(self.env.action_size):
NameError: name 'xrange' is not defined
Process finished with exit code 1
解决方案
xrange
# 替换为
range
问题8
Traceback (most recent call last):
File "/home/user/code/python/DQN-tf1.15/main.py", line 70, in <module>
tf.app.run()
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/python/platform/app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 308, in run
_run_main(main, args)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 254, in _run_main
sys.exit(main(argv))
File "/home/user/code/python/DQN-tf1.15/main.py", line 62, in main
agent = Agent(config, env, sess)
File "/home/user/code/python/DQN-tf1.15/dqn/agent.py", line 31, in __init__
self.build_dqn()
File "/home/user/code/python/DQN-tf1.15/dqn/agent.py", line 329, in build_dqn
self._saver = tf.train.Saver(self.w.values() + [self.step_op], max_to_keep=30)
TypeError: unsupported operand type(s) for +: 'dict_values' and 'list'
Process finished with exit code 1
解决方案
self._saver = tf.train.Saver(self.w.values() + [self.step_op], max_to_keep=30)
# 替换为
self._saver = tf.train.Saver(list(self.w.values()) + [self.step_op], max_to_keep=30)
问题9
Traceback (most recent call last):
File "/home/user/code/python/DQN-tf1.15/main.py", line 70, in <module>
tf.app.run()
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/python/platform/app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 308, in run
_run_main(main, args)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 254, in _run_main
sys.exit(main(argv))
File "/home/user/code/python/DQN-tf1.15/main.py", line 65, in main
agent.train()
File "/home/user/code/python/DQN-tf1.15/dqn/agent.py", line 42, in train
screen, reward, action, terminal = self.env.new_random_game()
File "/home/user/code/python/DQN-tf1.15/dqn/environment.py", line 28, in new_random_game
self.new_game(True)
File "/home/user/code/python/DQN-tf1.15/dqn/environment.py", line 23, in new_game
self._step(0)
File "/home/user/code/python/DQN-tf1.15/dqn/environment.py", line 35, in _step
self._screen, self.reward, self.terminal, _ = self.env.step(action)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/gym/wrappers/order_enforcing.py", line 36, in step
raise ResetNeeded("Cannot call env.step() before calling env.reset()")
gym.error.ResetNeeded: Cannot call env.step() before calling env.reset()
Process finished with exit code 1
解决方案
# 在environment.py文件中的def init语句内把
self._screen = None
# 更改为
self._screen = self.env.reset()
问题10
Traceback (most recent call last):
File "/home/user/code/python/DQN-tf1.15/main.py", line 70, in <module>
tf.app.run()
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/python/platform/app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 308, in run
_run_main(main, args)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 254, in _run_main
sys.exit(main(argv))
File "/home/user/code/python/DQN-tf1.15/main.py", line 65, in main
agent.train()
File "/home/user/code/python/DQN-tf1.15/dqn/agent.py", line 42, in train
screen, reward, action, terminal = self.env.new_random_game()
File "/home/user/code/python/DQN-tf1.15/dqn/environment.py", line 28, in new_random_game
self.new_game(True)
File "/home/user/code/python/DQN-tf1.15/dqn/environment.py", line 23, in new_game
self._step(0)
File "/home/user/code/python/DQN-tf1.15/dqn/environment.py", line 35, in _step
self._screen, self.reward, self.terminal, _ = self.env.step(action)
ValueError: too many values to unpack (expected 4)
Process finished with exit code 1
解决方案
pip install gym==0.15.7
问题11
Traceback (most recent call last):
File "/home/user/code/python/DQN-tf1.15/main.py", line 70, in <module>
tf.app.run()
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/python/platform/app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 308, in run
_run_main(main, args)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 254, in _run_main
sys.exit(main(argv))
File "/home/user/code/python/DQN-tf1.15/main.py", line 54, in main
env = GymEnvironment(config)
File "/home/user/code/python/DQN-tf1.15/dqn/environment.py", line 67, in __init__
super(GymEnvironment, self).__init__(config)
File "/home/user/code/python/DQN-tf1.15/dqn/environment.py", line 8, in __init__
self.env = gym.make(config.env_name)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/gym/envs/registration.py", line 156, in make
return registry.make(id, **kwargs)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/gym/envs/registration.py", line 101, in make
env = spec.make(**kwargs)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/gym/envs/registration.py", line 72, in make
cls = load(self.entry_point)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/gym/envs/registration.py", line 17, in load
mod = importlib.import_module(mod_name)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "" , line 1006, in _gcd_import
File "" , line 983, in _find_and_load
File "" , line 967, in _find_and_load_unlocked
File "" , line 677, in _load_unlocked
File "" , line 728, in exec_module
File "" , line 219, in _call_with_frames_removed
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/gym/envs/atari/__init__.py", line 1, in <module>
from gym.envs.atari.atari_env import AtariEnv
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/gym/envs/atari/atari_env.py", line 13, in <module>
"'pip install gym[atari]'.)".format(e))
gym.error.DependencyNotInstalled: No module named 'atari_py'. (HINT: you can install Atari dependencies by running 'pip install gym[atari]'.)
Process finished with exit code 1
解决方案
pip install gym[atari]
问题12
Traceback (most recent call last):
File "/home/user/code/python/DQN-tf1.15/main.py", line 70, in <module>
tf.app.run()
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/tensorflow_core/python/platform/app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 308, in run
_run_main(main, args)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/absl/app.py", line 254, in _run_main
sys.exit(main(argv))
File "/home/user/code/python/DQN-tf1.15/main.py", line 54, in main
env = GymEnvironment(config)
File "/home/user/code/python/DQN-tf1.15/dqn/environment.py", line 67, in __init__
super(GymEnvironment, self).__init__(config)
File "/home/user/code/python/DQN-tf1.15/dqn/environment.py", line 8, in __init__
self.env = gym.make(config.env_name)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/gym/envs/registration.py", line 156, in make
return registry.make(id, **kwargs)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/gym/envs/registration.py", line 101, in make
env = spec.make(**kwargs)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/gym/envs/registration.py", line 73, in make
env = cls(**_kwargs)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/gym/envs/atari/atari_env.py", line 49, in __init__
self.game_path = atari_py.get_game_path(game)
File "/home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/atari_py/games.py", line 20, in get_game_path
raise Exception('ROM is missing for %s, see https://github.com/openai/atari-py#roms for instructions' % (game_name,))
Exception: ROM is missing for breakout, see https://github.com/openai/atari-py#roms for instructions
Process finished with exit code 1
解决方案
# 下载该文件
wget http://www.atarimania.com/roms/Roms.rar
# 解压该文件,解压后目录ROMS的路径为/home/user/code/python/DQN-tensorflow/Roms/ROMS
├── Roms
│ ├── HC ROMS
│ └── ROMS
# 进入安装包目录
cd /home/user/.conda/envs/tf_1.15/lib/python3.7/site-packages/
# 运行以下命令
python -m atari_py.import_roms /home/user/code/python/DQN-tensorflow/Roms/ROMS
四、修改后代码
https://github.com/Rory602/DQN-tf1.15