ChatGPTに強化学習について聞いてみた2(Q学習で迷路ゲームのエージェント作成)

import numpy as np
import random

class MazeEnv:
    def __init__(self, maze):
        self.maze = maze
        self.actions = ['up', 'down', 'left', 'right']
        self.state = None
        self.start = None
        self.end = None

    def reset(self):
        self.state = self.start
        return self.state

    def step(self, action):
        next_state = self._next_state(action)
        reward = self._reward(next_state)
        done = next_state == self.end
        self.state = next_state
        return next_state, reward, done

    def _next_state(self, action):
        # (row, col)の形式で現在の状態を取得
        row, col = self.state

        # 各アクションに対する移動先を定義
        if action == 'up':
            row -= 1
        elif action == 'down':
            row += 1
        elif action == 'left':
            col -= 1
        elif action == 'right':
            col += 1

        # 移動先が迷路外または壁の場合、状態を変更しない
        if (row < 0 or row >= len(self.maze) or
                col < 0 or col >= len(self.maze[0]) or
                self.maze[row][col] == 1):
            return self.state

        return (row, col)

    def _reward(self, next_state):
        if next_state == self.end:
            return 100
        else:
            return -1

class QLearningAgent:
    def __init__(self, env, alpha=0.1, gamma=0.99, epsilon=0.1):
        self.env = env
        self.alpha = alpha
        self.gamma = gamma
        self.epsilon = epsilon
        self.q_table = np.zeros((len(env.maze), len(env.maze[0]), len(env.actions)))

    def choose_action(self, state):
        if np.random.uniform(0, 1) < self.epsilon:
            return random.choice(self.env.actions)
        else:
            action_values = self.q_table[state[0], state[1]]
            return self.env.actions[np.argmax(action_values)]

    def learn(self, state, action, reward, next_state):
        action_index = self.env.actions.index(action)
        current_q = self.q_table[state[0], state[1], action_index]
        next_max_q = np.max(self.q_table[next_state[0], next_state[1]])
        self.q_table[state[0], state[1], action_index] = current_q + self.alpha * (reward + self.gamma * next_max_q - current_q)

# 迷路の定義 (0: 通路, 1: 壁, S: スタート, G: ゴール)
maze = [
    [0, 0, 0, 0, 0],
    [0, 1, 1, 1, 0],
    [0, 0, 0, 0, 0],
    [1, 1, 1, 0, 1],
    [0, 0, 0, 0, 'G']
]

# スタートとゴールの座標を取得
start = (0, 0)
end = (len(maze)-1, len(maze[0])-1)

# 迷路環境を作成
env = MazeEnv(maze)
env.start = start
env.end = end

# Q学習エージェントを作成
agent = QLearningAgent(env)

# 学習のエピソード数
num_episodes = 1000

for episode in range(num_episodes):
    state = env.reset()
    done = False

    while not done:
        action = agent.choose_action(state)
        next_state, reward, done = env.step(action)
        agent.learn(state, action, reward, next_state)
        state = next_state

    if (episode + 1) % 100 == 0:
        print(f"Episode {episode + 1} completed.")

state = env.reset()
done = False
steps = 0

while not done:
    action = agent.choose_action(state)
    next_state, reward, done = env.step(action)
    print(f"Step {steps}: {state} -> {action} -> {next_state}")
    state = next_state
    steps += 1

print(f"Goal reached in {steps} steps.")

❯ python maze-q.py 

Episode 100 completed.
Episode 200 completed.
Episode 300 completed.
Episode 400 completed.
Episode 500 completed.
Episode 600 completed.
Episode 700 completed.
Episode 800 completed.
Episode 900 completed.
Episode 1000 completed.
Step 0: (0, 0) -> down -> (1, 0)
Step 1: (1, 0) -> down -> (2, 0)
Step 2: (2, 0) -> right -> (2, 1)
Step 3: (2, 1) -> right -> (2, 2)
Step 4: (2, 2) -> right -> (2, 3)
Step 5: (2, 3) -> down -> (3, 3)
Step 6: (3, 3) -> down -> (4, 3)
Step 7: (4, 3) -> right -> (4, 4)
Goal reached in 8 steps.