# Use Closed-Form Policy to Play MoutainCar-v0¶

In [1]:

    import sys
    import logging
    import itertools

    import numpy as np
    np.random.seed(0)
    import gym

    logging.basicConfig(level=logging.INFO,
            format='%(asctime)s [%(levelname)s] %(message)s',
            stream=sys.stdout, datefmt='%H:%M:%S')

In [2]:

    env = gym.make('MountainCar-v0')
    for key in vars(env.spec):
        logging.info('%s: %s', key, vars(env.spec)[key])
    for key in vars(env.unwrapped):
        logging.info('%s: %s', key, vars(env.unwrapped)[key])

    00:00:00 [INFO] id: MountainCar-v0
    00:00:00 [INFO] entry_point: gym.envs.classic_control:MountainCarEnv
    00:00:00 [INFO] reward_threshold: -110.0
    00:00:00 [INFO] nondeterministic: False
    00:00:00 [INFO] max_episode_steps: 200
    00:00:00 [INFO] order_enforce: True
    00:00:00 [INFO] _kwargs: {}
    00:00:00 [INFO] _env_name: MountainCar
    00:00:00 [INFO] min_position: -1.2
    00:00:00 [INFO] max_position: 0.6
    00:00:00 [INFO] max_speed: 0.07
    00:00:00 [INFO] goal_position: 0.5
    00:00:00 [INFO] goal_velocity: 0
    00:00:00 [INFO] force: 0.001
    00:00:00 [INFO] gravity: 0.0025
    00:00:00 [INFO] low: [-1.2  -0.07]
    00:00:00 [INFO] high: [0.6  0.07]
    00:00:00 [INFO] viewer: None
    00:00:00 [INFO] action_space: Discrete(3)
    00:00:00 [INFO] observation_space: Box([-1.2  -0.07], [0.6  0.07], (2,), float32)
    00:00:00 [INFO] np_random: RandomState(MT19937)
    00:00:00 [INFO] spec: EnvSpec(MountainCar-v0)

In [3]:

    class CloseFormAgent:
        def __init__(self, _):
            pass

        def reset(self, mode=None):
            pass

        def step(self, observation, reward, terminated):
            position, velocity = observation
            lb = min(-0.09 * (position + 0.25) ** 2 + 0.03,
                    0.3 * (position + 0.9) ** 4 - 0.008)
            ub = -0.07 * (position + 0.38) ** 2 + 0.07
            if lb < velocity < ub:
                action = 2 # push right
            else:
                action = 0 # push left
            return action

        def close(self):
            pass

    agent = CloseFormAgent(env)

In [4]:

    def play_episode(env, agent, seed=None, mode=None, render=False):
        observation, _ = env.reset(seed=seed)
        reward, terminated, truncated = 0., False, False
        agent.reset(mode=mode)
        episode_reward, elapsed_steps = 0., 0
        while True:
            action = agent.step(observation, reward, terminated)
            if render:
                env.render()
            if terminated or truncated:
                break
            observation, reward, terminated, truncated, _ = env.step(action)
            episode_reward += reward
            elapsed_steps += 1
        agent.close()
        return episode_reward, elapsed_steps

    logging.info('==== test ====')
    episode_rewards = []
    for episode in range(100):
        episode_reward, elapsed_steps = play_episode(env, agent)
        episode_rewards.append(episode_reward)
        logging.info('test episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
    logging.info('average episode reward = %.2f ± %.2f',
            np.mean(episode_rewards), np.std(episode_rewards))

    00:00:00 [INFO] ==== test ====
    00:00:00 [INFO] test episode 0: reward = -105.00, steps = 105
    00:00:00 [INFO] test episode 1: reward = -84.00, steps = 84
    00:00:00 [INFO] test episode 2: reward = -133.00, steps = 133
    00:00:00 [INFO] test episode 3: reward = -105.00, steps = 105
    00:00:00 [INFO] test episode 4: reward = -105.00, steps = 105
    00:00:00 [INFO] test episode 5: reward = -100.00, steps = 100
    00:00:00 [INFO] test episode 6: reward = -104.00, steps = 104
    00:00:00 [INFO] test episode 7: reward = -100.00, steps = 100
    00:00:00 [INFO] test episode 8: reward = -105.00, steps = 105
    00:00:00 [INFO] test episode 9: reward = -145.00, steps = 145
    00:00:00 [INFO] test episode 10: reward = -105.00, steps = 105
    00:00:00 [INFO] test episode 11: reward = -83.00, steps = 83
    00:00:00 [INFO] test episode 12: reward = -103.00, steps = 103
    00:00:00 [INFO] test episode 13: reward = -156.00, steps = 156
    00:00:00 [INFO] test episode 14: reward = -84.00, steps = 84
    00:00:00 [INFO] test episode 15: reward = -96.00, steps = 96
    00:00:00 [INFO] test episode 16: reward = -104.00, steps = 104
    00:00:00 [INFO] test episode 17: reward = -85.00, steps = 85
    00:00:00 [INFO] test episode 18: reward = -105.00, steps = 105
    00:00:00 [INFO] test episode 19: reward = -85.00, steps = 85
    00:00:00 [INFO] test episode 20: reward = -89.00, steps = 89
    00:00:00 [INFO] test episode 21: reward = -106.00, steps = 106
    00:00:00 [INFO] test episode 22: reward = -104.00, steps = 104
    00:00:00 [INFO] test episode 23: reward = -97.00, steps = 97
    00:00:00 [INFO] test episode 24: reward = -86.00, steps = 86
    00:00:00 [INFO] test episode 25: reward = -89.00, steps = 89
    00:00:00 [INFO] test episode 26: reward = -92.00, steps = 92
    00:00:00 [INFO] test episode 27: reward = -105.00, steps = 105
    00:00:00 [INFO] test episode 28: reward = -103.00, steps = 103
    00:00:01 [INFO] test episode 29: reward = -105.00, steps = 105
    00:00:01 [INFO] test episode 30: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 31: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 32: reward = -103.00, steps = 103
    00:00:01 [INFO] test episode 33: reward = -85.00, steps = 85
    00:00:01 [INFO] test episode 34: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 35: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 36: reward = -85.00, steps = 85
    00:00:01 [INFO] test episode 37: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 38: reward = -103.00, steps = 103
    00:00:01 [INFO] test episode 39: reward = -105.00, steps = 105
    00:00:01 [INFO] test episode 40: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 41: reward = -84.00, steps = 84
    00:00:01 [INFO] test episode 42: reward = -106.00, steps = 106
    00:00:01 [INFO] test episode 43: reward = -105.00, steps = 105
    00:00:01 [INFO] test episode 44: reward = -84.00, steps = 84
    00:00:01 [INFO] test episode 45: reward = -105.00, steps = 105
    00:00:01 [INFO] test episode 46: reward = -85.00, steps = 85
    00:00:01 [INFO] test episode 47: reward = -86.00, steps = 86
    00:00:01 [INFO] test episode 48: reward = -146.00, steps = 146
    00:00:01 [INFO] test episode 49: reward = -85.00, steps = 85
    00:00:01 [INFO] test episode 50: reward = -105.00, steps = 105
    00:00:01 [INFO] test episode 51: reward = -105.00, steps = 105
    00:00:01 [INFO] test episode 52: reward = -86.00, steps = 86
    00:00:01 [INFO] test episode 53: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 54: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 55: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 56: reward = -98.00, steps = 98
    00:00:01 [INFO] test episode 57: reward = -84.00, steps = 84
    00:00:01 [INFO] test episode 58: reward = -83.00, steps = 83
    00:00:01 [INFO] test episode 59: reward = -105.00, steps = 105
    00:00:01 [INFO] test episode 60: reward = -86.00, steps = 86
    00:00:01 [INFO] test episode 61: reward = -105.00, steps = 105
    00:00:01 [INFO] test episode 62: reward = -105.00, steps = 105
    00:00:01 [INFO] test episode 63: reward = -176.00, steps = 176
    00:00:01 [INFO] test episode 64: reward = -156.00, steps = 156
    00:00:01 [INFO] test episode 65: reward = -105.00, steps = 105
    00:00:01 [INFO] test episode 66: reward = -89.00, steps = 89
    00:00:01 [INFO] test episode 67: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 68: reward = -85.00, steps = 85
    00:00:01 [INFO] test episode 69: reward = -92.00, steps = 92
    00:00:01 [INFO] test episode 70: reward = -105.00, steps = 105
    00:00:01 [INFO] test episode 71: reward = -92.00, steps = 92
    00:00:01 [INFO] test episode 72: reward = -106.00, steps = 106
    00:00:01 [INFO] test episode 73: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 74: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 75: reward = -103.00, steps = 103
    00:00:01 [INFO] test episode 76: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 77: reward = -83.00, steps = 83
    00:00:01 [INFO] test episode 78: reward = -90.00, steps = 90
    00:00:01 [INFO] test episode 79: reward = -100.00, steps = 100
    00:00:01 [INFO] test episode 80: reward = -105.00, steps = 105
    00:00:01 [INFO] test episode 81: reward = -90.00, steps = 90
    00:00:01 [INFO] test episode 82: reward = -106.00, steps = 106
    00:00:01 [INFO] test episode 83: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 84: reward = -150.00, steps = 150
    00:00:01 [INFO] test episode 85: reward = -168.00, steps = 168
    00:00:01 [INFO] test episode 86: reward = -103.00, steps = 103
    00:00:01 [INFO] test episode 87: reward = -93.00, steps = 93
    00:00:01 [INFO] test episode 88: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 89: reward = -103.00, steps = 103
    00:00:01 [INFO] test episode 90: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 91: reward = -105.00, steps = 105
    00:00:01 [INFO] test episode 92: reward = -105.00, steps = 105
    00:00:01 [INFO] test episode 93: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 94: reward = -100.00, steps = 100
    00:00:01 [INFO] test episode 95: reward = -86.00, steps = 86
    00:00:01 [INFO] test episode 96: reward = -104.00, steps = 104
    00:00:01 [INFO] test episode 97: reward = -88.00, steps = 88
    00:00:01 [INFO] test episode 98: reward = -103.00, steps = 103
    00:00:01 [INFO] test episode 99: reward = -105.00, steps = 105
    00:00:01 [INFO] average episode reward = -102.61 ± 17.46

In [5]:

    env.close()