强化学习井字棋游戏
强化学习井字棋游戏实现
这是一个简单的强化学习例子Tic-Tac-Toe。在一个3×3的九宫格里，两个人人论留下，直到有个人的棋子满足三个一横一竖或者一斜，赢得比赛游戏结束，或者九宫格填满也没有人赢，则和棋。
程序实现用两个电脑选手训练模型，然后可以让任何机器对战。
下面进行对代码介绍：
import numpy as np
import pickle
BOARD_ROWS = 3
BOARD_COLS = 3
BOARD_SIZE = BOARD_ROWS * BOARD_COLS
state状态类：每个状态用自定义hash值描述，主要方法为get_all_states和next_state
class State:
def __init__(self):
# the board is represented by an n * n array,
# 1 represents a chessman of the player who moves first,
# -1 represents a chessman of another player
# 0 represents an empty position
self.data = np.zeros((BOARD_ROWS, BOARD_COLS)) # 初始化为0
self.winner = None # 赢家初始化为None
self.hash_val = None # 值函数也初始化为None
self.end = None # 结束初始化为None
# compute the hash value for one state, it's unique
# 计算一步的hash价值
def hash(self):
if self.hash_val is None: # 初始化的时候是None
self.hash_val = 0 # 将其值设置为0
for i in self.data.reshape(BOARD_ROWS * BOARD_COLS): # 改变数组的形状，便利数组里面的值
if i == -1: # 即原来取值-1,0,1，现在将-1设置为2，为了hash方便
i = 2
self.hash_val = self.hash_val * 3 + i
return int(self.hash_val)
# check whether a player has won the game, or it's a tie
# 检查玩家是否赢得了游戏，或者是否是平局
def is_end(self):
if self.end is not None: # 初始化是None，如果是None，那么就是游戏已经结束了
return self.end # 初始化值是None
results = [] # 结果，这是一个列表
# check row
for i in range(0, BOARD_ROWS):
results.append(np.sum(self.data[i, :])) # 对数组的行求和，将结果放入results数组之中
# check columns
for i in range(0, BOARD_COLS):
results.append(np.sum(self.data[:, i])) # 对数组的列求和，将结果放入results数组之中
# check diagonals，检查对角线
results.append(0) # 数组里面加一个0
for i in range(0, BOARD_ROWS): # 主对角线
results[-1] += self.data[i, i] # 用主对角线元素的求和值去覆盖之前添加的零
results.append(0)
for i in range(0, BOARD_ROWS): # 副对角线
results[-1] += self.data[i, BOARD_ROWS - 1 - i] # 用副对角线元素的求和值去覆盖之前添加的零
for result in results: # 用result变量去遍历results列表
if result == 3:
self.winner = 1
self.end = True
return self.end
if result == -3:
self.winner = -1
self.end = True
return self.end
# 如果是3，则说明是玩家赢了，如果是-3，则说明是机器赢了
# whether it's a tie，在之前的判断之后，玩家也没赢。机器也没赢，下面判断是否是平局
sum = np.sum(np.abs(self.data)) # 将列表中所有的数据加绝对值后求和
if sum == BOARD_ROWS * BOARD_COLS: # 如果求和的结果等于列表元素的个数之和
self.winner = 0 # 则玩家没有赢
self.end = True # 游戏结束了
return self.end
# game is still going on，如果以上情况都没有发生，则说明游戏没有结束，则游戏继续
self.end = False
return self.end
# @symbol: 1 or -1
# put chessman symbol in position (i, j)
# 根据符号标记来确定下一个状态
def next_state(self, i, j, symbol):
# 新创建一个对象，该对象是对State类的实例化，将原来的状态和更新后的状态存入其中
new_state = State()
new_state.data = np.copy(self.data)
new_state.data[i, j] = symbol
return new_state
# print the board，打印一下棋盘数据
def print(self):
for i in range(0, BOARD_ROWS):
print('-------------')
out = '| '
for j in range(0, BOARD_COLS):
if self.data[i, j] == 1:
token = '*'
if self.data[i, j] == 0:
token = '0'
if self.data[i, j] == -1:
token = 'x'
out += token + ' | '
print(out)
print('-------------')
# 这个函数创建一个字典all_state，以hash值为key，value为(state, is_End)
def get_all_states_impl(current_state, current_symbol, all_states):
# all_states：字典，以hash值为key，value为(state,is_End)
for i in range(0, BOARD_ROWS):
for j in range(0, BOARD_COLS): # 遍历列表
if current_state.data[i][j] == 0:
newState = current_state.next_state(i, j, current_symbol) # 根据当前状态来判断新的状态
newHash = newState.hash() # 计算新的hash值
if newHash not in all_states.keys(): # 如果新的hash值不在所有状态的键值里面
isEnd = newState.is_end()
all_states[newHash] = (newState, isEnd) # 值是一个键值对
# 如果没有结束对局，下一个选手继续下
if not isEnd:
get_all_states_impl(newState, -current_symbol, all_states) # 这是一个递归函数，下一次调用换了一个玩家玩。
# 运行一次得到所有状态
def get_all_states():
current_symbol = 1 # symbol标记是为了检验是哪一个玩家在玩，为1时时玩家1，为-1是玩家2
current_state = State()
all_states = dict() # 字典类型
all_states[current_state.hash()] = (current_state, current_state.is_end()) # 字典的value数据类型为(state, is_End)
get_all_states_impl(current_state, current_symbol, all_states)
return all_states
# all possible board configurations
all_states = get_all_states()
裁判：监督选手轮流下棋。主要方法为alternate（轮流选手），play（监督游戏执行，play里重要的为选手的act方法）
class Judger:
# @player1: the player who will move first, its chessman will be 1
# @player2: another player with a chessman -1
# @feedback: if True, both players will receive rewards when game is end
def __init__(self, player1, player2):
self.p1 = player1 # 玩家一
self.p2 = player2 # 玩家二
self.current_player = None # 当前玩家
self.p1_symbol = 1 # 玩家一标志
self.p2_symbol = -1 # 玩家二标志
self.p1.set_symbol(self.p1_symbol) # 设置标志
self.p2.set_symbol(self.p2_symbol)
self.current_state = State() # 创建一个state对象
def reset(self):
self.p1.reset()
self.p2.reset()
# 这个函数实现了两个玩家轮流下棋的功能
def alternate(self):
while True:
yield self.p1
yield self.p2
# @print: if True, print each board during the game
# 这个方法监督游戏执行
def play(self, print=False):
alternator = self.alternate() # 调用迭代器函数
self.reset() # 重置
current_state = State() # 创建一个对象，是当前状态
self.p1.set_state(current_state) # 把current_state的内容存入对象的属性里面
self.p2.set_state(current_state)
while True:
player = next(alternator)
if print:
current_state.print()
[i, j, symbol] = player.act()
next_state_hash = current_state.next_state(i, j, symbol).hash()
current_state, is_end = all_states[next_state_hash]
self.p1.set_state(current_state)
self.p2.set_state(current_state)
if is_end:
if print:
current_state.print()
return current_state.winner
AI选手：estimations表示不同状态下的分值，用以进行下一状态的选择，greedy区分随机行为，即随机行为不参与更新状态的分值主要方法为set_symbol（设置对于每个选手各状态分值的初始值），backup（更新状态分值，如果下一状态分值更高，那么当前状态的分值也要提高，即将长远的结果反作用到现在），act（获取下一步坐标）
class Player:
# @step_size: the step size to update estimations
# @epsilon: the probability to explore
def __init__(self, step_size=0.1, epsilon=0.1):
self.estimations = dict() # estimations是字典类型
self.step_size = step_size # 步长
self.epsilon = epsilon # 贪婪策略的小概率
self.states = []
self.greedy = []
def reset(self):
self.states = []
self.greedy = []
def set_state(self, state):
self.states.append(state)
self.greedy.append(True)
def set_symbol(self, symbol): # 设置对每个选手各状态分值的初始值
self.symbol = symbol
# 对状态分值初始化，最终赢了得1分，输了不得分，平局0.5分，
# 未到终局设置为0.5分
for hash_val in all_states.keys():
(state, is_end) = all_states[hash_val]
if is_end:
if state.winner == self.symbol:
self.estimations[hash_val] = 1.0
elif state.winner == 0:
# we need to distinguish between a tie and a lose
self.estimations[hash_val] = 0.5
else:
self.estimations[hash_val] = 0 # 输了不得分
else:
self.estimations[hash_val] = 0.5
# update value estimation
# 将在贪心动作之后得到的状态对应的价值”回溯更新“到动作之前的状态上。（对早先的状态的价值进行调整，使其更接近于后面的状态所对应的价值）
def backup(self): # 更新状态分值，如果下一状态分值更高，那么当前状态分值也要提高，即将长远的结果反作用到现在。
# for debug
# print('player trajectory')
# for state in self.states:
# state.print()
self.states = [state.hash() for state in self.states]
# 顺序更新
# 反转的迭代器
for i in reversed(range(len(self.states) - 1)):
state = self.states[i]
td_error = self.greedy[i] * (self.estimations[self.states[i + 1]] - self.estimations[state]) # 计算TD误差
self.estimations[state] += self.step_size * td_error
# choose an action based on the state
def act(self): # 获取下一步坐标
# 取出当前（最后一个）状态
state = self.states[-1]
# 下一步可能的状态的hash
next_states = []
# 下一步可能的坐标
next_positions = []
for i in range(BOARD_ROWS):
for j in range(BOARD_COLS):
if state.data[i, j] == 0:
next_positions.append([i, j])
next_states.append(state.next_state(i, j, self.symbol).hash())
# 小概率随机探索
if np.random.rand() < self.epsilon:
action = next_positions[np.random.randint(len(next_positions))]
action.append(self.symbol)
# 表示随机动作不参与价值更新
self.greedy[-1] = False
return action
# 大概率按照奖励最高行动
values = []
for hash, pos in zip(next_states, next_positions):
values.append((self.estimations[hash], pos))
np.random.shuffle(values) # 将这个数组中的元素随机进行排序
values.sort(key=lambda x: x[0], reverse=True)
action = values[0][1]
action.append(self.symbol)
return action
def save_policy(self):
with open('policy_%s.bin' % ('first' if self.symbol == 1 else 'second'), 'wb') as f:
pickle.dump(self.estimations, f)
def load_policy(self):
with open('policy_%s.bin' % ('first' if self.symbol == 1 else 'second'), 'rb') as f:
self.estimations = pickle.load(f)
人类玩家
# human interface
# input a number to put a chessman
# | q | w | e |
# | a | s | d |
# | z | x | c |
class HumanPlayer:
def __init__(self, **kwargs):
self.symbol = None
self.keys = ['q', 'w', 'e', 'a', 's', 'd', 'z', 'x', 'c']
self.state = None
return
def reset(self):
return
def set_state(self, state):
self.state = state
def set_symbol(self, symbol):
self.symbol = symbol
return
def backup(self, _):
return
def act(self):
self.state.print()
key = input("Input your position:")
data = self.keys.index(key)
i = data // int(BOARD_COLS)
j = data % BOARD_COLS
return (i, j, self.symbol)
训练：
def train(epochs): # 训练重复epochs次
player1 = Player(epsilon=0.01) # 实例化玩家1
player2 = Player(epsilon=0.01) # 实例化玩家二
judger = Judger(player1, player2) # 判断玩家一和玩家二的胜负
player1_win = 0.0 # 玩家一胜利场数
player2_win = 0.0 # 玩家二胜利场数
for i in range(1, epochs + 1): # epochs为迭代次数
winner = judger.play(print=False)
if winner == 1:
player1_win += 1
if winner == -1:
player2_win += 1
# 输出两个选手的获胜概率，到最后基本是平局
print('Epoch %d, player 1 win %.02f, player 2 win %.02f' % (i, player1_win / i, player2_win / i))
player1.backup()
player2.backup()
judger.reset() # 一盘对决结束就重置游戏
player1.save_policy() # 保存状态价值，其实训练获取的就是各状态分别对每个选手的价值
player2.save_policy() # 保存策略
AI自测：
def compete(turns):
# 不允许随机行为
player1 = Player(epsilon=0)
player2 = Player(epsilon=0)
judger = Judger(player1, player2)
player1.load_policy()
player2.load_policy()
player1_win = 0.0
player2_win = 0.0
for i in range(0, turns):
winner = judger.play()
if winner == 1:
player1_win += 1
if winner == -1:
player2_win += 1
judger.reset()
print('%d turns, player 1 win %.02f, player 2 win %.02f' % (turns, player1_win / turns, player2_win / turns))
人机对战：
# The game is a zero sum game. If both players are playing with an optimal strategy, every game will end in a tie.
# So we test whether the AI can guarantee at least a tie if it goes second.
def play():
while True:
player1 = HumanPlayer()
player2 = Player(epsilon=0)
judger = Judger(player1, player2)
player2.load_policy()
winner = judger.play()
if winner == player2.symbol:
print("You lose!")
elif winner == player1.symbol:
print("You win!")
else:
print("It is a tie!")
调用程序:
if __name__ == '__main__':
train(int(1e5))
compete(int(1e3))
play()