# Taxi-v3¶

In [1]:

    %matplotlib inline

    import sys
    import logging
    import itertools

    import numpy as np
    np.random.seed(0)
    import pandas as pd
    import gym
    import matplotlib.pyplot as plt

    logging.basicConfig(level=logging.INFO,
            format='%(asctime)s [%(levelname)s] %(message)s',
            stream=sys.stdout, datefmt='%H:%M:%S')

### Use Environment¶

In [2]:

    env = gym.make('Taxi-v3')
    for key in vars(env):
        logging.info('%s: %s', key, vars(env)[key])
    for key in vars(env.spec):
        logging.info('%s: %s', key, vars(env.spec)[key])

    23:10:32 [INFO] env: <TaxiEnv<Taxi-v3>>
    23:10:32 [INFO] action_space: Discrete(6)
    23:10:32 [INFO] observation_space: Discrete(500)
    23:10:32 [INFO] reward_range: (-inf, inf)
    23:10:32 [INFO] metadata: {'render.modes': ['human', 'ansi']}
    23:10:32 [INFO] _max_episode_steps: 200
    23:10:32 [INFO] _elapsed_steps: None
    23:10:32 [INFO] id: Taxi-v3
    23:10:32 [INFO] entry_point: gym.envs.toy_text:TaxiEnv
    23:10:32 [INFO] reward_threshold: 8
    23:10:32 [INFO] nondeterministic: False
    23:10:32 [INFO] max_episode_steps: 200
    23:10:32 [INFO] _kwargs: {}
    23:10:32 [INFO] _env_name: Taxi

In [3]:

    state, _ = env.reset()
    taxirow, taxicol, passloc, destidx = env.unwrapped.decode(state)
    logging.info('location of taxi = %s', (taxirow, taxicol))
    logging.info('location of passager = %s', env.unwrapped.locs[passloc])
    logging.info('location of destination = %s', env.unwrapped.locs[destidx])
    # env.render()  # not compatible with Jupyter Notebook

    23:10:32 [INFO] location of taxi = (0, 1)
    23:10:32 [INFO] location of passager = (0, 4)
    23:10:32 [INFO] location of destination = (4, 0)
    +---------+
    |R: | : :G|
    | : | : : |
    | : : : : |
    | | : | : |
    |Y| : |B: |
    +---------+

In [4]:

    env.step(0)

Out[4]:

    (126, -1, False, {'prob': 1.0})

In [5]:

    # env.render()  # not compatible with Jupyter Notebook

    +---------+
    |R: | : :G|
    | : | : : |
    | : : : : |
    | | : | : |
    |Y| : |B: |
    +---------+
      (South)

### SARSA¶

In [6]:

    class SARSAAgent:
        def __init__(self, env):
            self.gamma = 0.9
            self.learning_rate = 0.2
            self.epsilon = 0.01
            self.action_n = env.action_space.n
            self.q = np.zeros((env.observation_space.n, env.action_space.n))

        def reset(self, mode=None):
            self.mode = mode
            if self.mode == 'train':
                self.trajectory = []

        def step(self, observation, reward, terminated):
            if self.mode == 'train' and np.random.uniform() < self.epsilon:
                action = np.random.randint(self.action_n)
            else:
                action = self.q[observation].argmax()
            if self.mode == 'train':
                self.trajectory += [observation, reward, terminated, action]
                if len(self.trajectory) >= 8:
                    self.learn()
            return action

        def close(self):
            pass

        def learn(self):
            state, _, _, action, next_state, reward, terminated, next_action = \
                            self.trajectory[-8:]

            target = reward + self.gamma * \
                    self.q[next_state, next_action] * (1. - terminated)
            td_error = target - self.q[state, action]
            self.q[state, action] += self.learning_rate * td_error

    agent = SARSAAgent(env)

In [7]:

    def play_episode(env, agent, seed=None, mode=None, render=False):
        observation, _ = env.reset(seed=seed)
        reward, terminated, truncated = 0., False, False
        agent.reset(mode=mode)
        episode_reward, elapsed_steps = 0., 0
        while True:
            action = agent.step(observation, reward, terminated)
            if render:
                env.render()
            if terminated or truncated:
                break
            observation, reward, terminated, truncated, _ = env.step(action)
            episode_reward += reward
            elapsed_steps += 1
        agent.close()
        return episode_reward, elapsed_steps

    logging.info('==== train ====')
    episode_rewards = []
    for episode in itertools.count():
        episode_reward, elapsed_steps = play_episode(env, agent, seed=episode,
                mode='train')
        episode_rewards.append(episode_reward)
        logging.info('train episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
        if np.mean(episode_rewards[-200:]) > env.spec.reward_threshold:
            break
    plt.plot(episode_rewards)

    logging.info('==== test ====')
    episode_rewards = []
    for episode in range(100):
        episode_reward, elapsed_steps = play_episode(env, agent)
        episode_rewards.append(episode_reward)
        logging.info('test episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
    logging.info('average episode reward = %.2f ± %.2f',
            np.mean(episode_rewards), np.std(episode_rewards))

    23:10:32 [INFO] ==== train ====
    23:10:35 [INFO] ==== test ====
    23:10:35 [INFO] average episode reward = 8.03 ± 2.50

Show optimal action values

In [8]:

    pd.DataFrame(agent.q)

Out[8]:

| 0 | 1 | 2 | 3 | 4 | 5
--- | --- | --- | --- | --- | --- | ---
0 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000
1 | -3.487602 | -3.095619 | -3.095238 | -3.851571 | 1.364833 | -7.108184
2 | -2.128847 | 1.632589 | -2.144477 | 0.719892 | 7.568908 | -2.268642
3 | -3.597061 | -3.596466 | -2.821674 | -2.618224 | 2.069454 | -4.455161
4 | -5.409483 | -5.923042 | -5.927367 | -6.047687 | -7.073330 | -7.108867
... | ... | ... | ... | ... | ... | ...
495 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000
496 | -3.102085 | -2.970985 | -2.883027 | -3.022181 | -3.600000 | -3.636000
497 | -1.681193 | -0.878103 | -1.704407 | -1.658715 | -3.600000 | -3.636000
498 | -3.119792 | -3.005599 | -3.150313 | -3.150968 | -5.014447 | -3.636000
499 | -0.707040 | -0.396000 | -0.707040 | 5.889600 | -3.600000 | -3.636000

500 rows × 6 columns

Show optimal policy

In [9]:

    policy = np.eye(agent.action_n)[agent.q.argmax(axis=-1)]
    pd.DataFrame(policy)

Out[9]:

| 0 | 1 | 2 | 3 | 4 | 5
--- | --- | --- | --- | --- | --- | ---
0 | 1.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0
1 | 0.0 | 0.0 | 0.0 | 0.0 | 1.0 | 0.0
2 | 0.0 | 0.0 | 0.0 | 0.0 | 1.0 | 0.0
3 | 0.0 | 0.0 | 0.0 | 0.0 | 1.0 | 0.0
4 | 1.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0
... | ... | ... | ... | ... | ... | ...
495 | 1.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0
496 | 0.0 | 0.0 | 1.0 | 0.0 | 0.0 | 0.0
497 | 0.0 | 1.0 | 0.0 | 0.0 | 0.0 | 0.0
498 | 0.0 | 1.0 | 0.0 | 0.0 | 0.0 | 0.0
499 | 0.0 | 0.0 | 0.0 | 1.0 | 0.0 | 0.0

500 rows × 6 columns

In [10]:

    env.close()