# Use Closed-Form Policy to Play LunarLanderContinuous-v2¶

In [1]:

    import sys
    import logging
    import itertools

    import numpy as np
    np.random.seed(0)
    import gym

    logging.basicConfig(level=logging.INFO,
            format='%(asctime)s [%(levelname)s] %(message)s',
            stream=sys.stdout, datefmt='%H:%M:%S')

In [2]:

    env = gym.make('LunarLanderContinuous-v2')
    for key in vars(env):
        logging.info('%s: %s', key, vars(env)[key])
    for key in vars(env.spec):
        logging.info('%s: %s', key, vars(env.spec)[key])

    00:00:00 [INFO] env: <LunarLanderContinuous<LunarLanderContinuous-v2>>
    00:00:00 [INFO] action_space: Box(-1.0, 1.0, (2,), float32)
    00:00:00 [INFO] observation_space: Box(-inf, inf, (8,), float32)
    00:00:00 [INFO] reward_range: (-inf, inf)
    00:00:00 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 50}
    00:00:00 [INFO] _max_episode_steps: 1000
    00:00:00 [INFO] _elapsed_steps: None
    00:00:00 [INFO] id: LunarLanderContinuous-v2
    00:00:00 [INFO] entry_point: gym.envs.box2d:LunarLanderContinuous
    00:00:00 [INFO] reward_threshold: 200
    00:00:00 [INFO] nondeterministic: False
    00:00:00 [INFO] max_episode_steps: 1000
    00:00:00 [INFO] _kwargs: {}
    00:00:00 [INFO] _env_name: LunarLanderContinuous

In [3]:

    class ClosedFormAgent:
        def __init__(self, _):
            pass

        def reset(self, mode=None):
            pass

        def step(self, observation, reward, terminated):
            x, y, v_x, v_y, angle, v_angle, contact_left, contact_right = observation

            if contact_left or contact_right:  # legs have contact
                f_y = -10. * v_y - 1.
                f_angle = 0.
            else:
                f_y = 5.5 * np.abs(x) - 10. * y - 10. * v_y - 1.
                f_angle = -np.clip(5. * x + 10. * v_x, -4, 4) + 10. * angle + 20. \
                        * v_angle

            action = np.array([f_y, f_angle])
            return action

        def close(self):
            pass

    agent = ClosedFormAgent(env)

In [4]:

    def play_episode(env, agent, seed=None, mode=None, render=False):
        observation, _ = env.reset(seed=seed)
        reward, terminated, truncated = 0., False, False
        agent.reset(mode=mode)
        episode_reward, elapsed_steps = 0., 0
        while True:
            action = agent.step(observation, reward, terminated)
            if render:
                env.render()
            if terminated or truncated:
                break
            observation, reward, terminated, truncated, _ = env.step(action)
            episode_reward += reward
            elapsed_steps += 1
        agent.close()
        return episode_reward, elapsed_steps

    logging.info('==== test ====')
    episode_rewards = []
    for episode in range(100):
        episode_reward, elapsed_steps = play_episode(env, agent)
        episode_rewards.append(episode_reward)
        logging.info('test episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
    logging.info('average episode reward = %.2f ± %.2f',
            np.mean(episode_rewards), np.std(episode_rewards))

    00:00:00 [INFO] ==== test ====
    00:00:00 [INFO] test episode 0: reward = 250.94, steps = 172
    00:00:00 [INFO] test episode 1: reward = 263.77, steps = 186
    00:00:00 [INFO] test episode 2: reward = 305.75, steps = 202
    00:00:00 [INFO] test episode 3: reward = 300.88, steps = 241
    00:00:00 [INFO] test episode 4: reward = 303.26, steps = 227
    00:00:00 [INFO] test episode 5: reward = 287.23, steps = 205
    00:00:00 [INFO] test episode 6: reward = 303.74, steps = 227
    00:00:01 [INFO] test episode 7: reward = 299.80, steps = 231
    00:00:01 [INFO] test episode 8: reward = 257.93, steps = 181
    00:00:01 [INFO] test episode 9: reward = 292.88, steps = 207
    00:00:01 [INFO] test episode 10: reward = 292.44, steps = 177
    00:00:01 [INFO] test episode 11: reward = 307.51, steps = 237
    00:00:01 [INFO] test episode 12: reward = 283.18, steps = 211
    00:00:01 [INFO] test episode 13: reward = 300.59, steps = 189
    00:00:01 [INFO] test episode 14: reward = 275.73, steps = 187
    00:00:01 [INFO] test episode 15: reward = 295.38, steps = 208
    00:00:01 [INFO] test episode 16: reward = 270.13, steps = 194
    00:00:01 [INFO] test episode 17: reward = 282.46, steps = 196
    00:00:01 [INFO] test episode 18: reward = 246.60, steps = 178
    00:00:01 [INFO] test episode 19: reward = 265.08, steps = 168
    00:00:01 [INFO] test episode 20: reward = 298.85, steps = 245
    00:00:02 [INFO] test episode 21: reward = 271.68, steps = 185
    00:00:02 [INFO] test episode 22: reward = 283.59, steps = 194
    00:00:02 [INFO] test episode 23: reward = 283.59, steps = 196
    00:00:02 [INFO] test episode 24: reward = 265.66, steps = 184
    00:00:02 [INFO] test episode 25: reward = 275.56, steps = 193
    00:00:02 [INFO] test episode 26: reward = 301.05, steps = 241
    00:00:02 [INFO] test episode 27: reward = 281.74, steps = 203
    00:00:02 [INFO] test episode 28: reward = 296.80, steps = 191
    00:00:02 [INFO] test episode 29: reward = 313.79, steps = 221
    00:00:02 [INFO] test episode 30: reward = 297.17, steps = 272
    00:00:03 [INFO] test episode 31: reward = 316.78, steps = 210
    00:00:03 [INFO] test episode 32: reward = 309.11, steps = 243
    00:00:03 [INFO] test episode 33: reward = 267.28, steps = 170
    00:00:03 [INFO] test episode 34: reward = 305.94, steps = 230
    00:00:03 [INFO] test episode 35: reward = 266.49, steps = 177
    00:00:03 [INFO] test episode 36: reward = 293.06, steps = 158
    00:00:03 [INFO] test episode 37: reward = 276.52, steps = 191
    00:00:03 [INFO] test episode 38: reward = 280.69, steps = 172
    00:00:03 [INFO] test episode 39: reward = 293.93, steps = 213
    00:00:03 [INFO] test episode 40: reward = 274.05, steps = 187
    00:00:03 [INFO] test episode 41: reward = 251.04, steps = 171
    00:00:03 [INFO] test episode 42: reward = 304.78, steps = 256
    00:00:03 [INFO] test episode 43: reward = 249.49, steps = 177
    00:00:04 [INFO] test episode 44: reward = 294.19, steps = 207
    00:00:04 [INFO] test episode 45: reward = 279.96, steps = 181
    00:00:04 [INFO] test episode 46: reward = 278.93, steps = 207
    00:00:04 [INFO] test episode 47: reward = 282.50, steps = 162
    00:00:04 [INFO] test episode 48: reward = 296.50, steps = 227
    00:00:04 [INFO] test episode 49: reward = 273.29, steps = 199
    00:00:04 [INFO] test episode 50: reward = 267.30, steps = 180
    00:00:04 [INFO] test episode 51: reward = 271.27, steps = 158
    00:00:04 [INFO] test episode 52: reward = 275.61, steps = 188
    00:00:04 [INFO] test episode 53: reward = 292.77, steps = 213
    00:00:04 [INFO] test episode 54: reward = 255.89, steps = 169
    00:00:04 [INFO] test episode 55: reward = 271.30, steps = 201
    00:00:04 [INFO] test episode 56: reward = 291.56, steps = 165
    00:00:04 [INFO] test episode 57: reward = 257.25, steps = 192
    00:00:05 [INFO] test episode 58: reward = 280.17, steps = 194
    00:00:05 [INFO] test episode 59: reward = 276.95, steps = 200
    00:00:05 [INFO] test episode 60: reward = 304.41, steps = 256
    00:00:05 [INFO] test episode 61: reward = 276.07, steps = 165
    00:00:05 [INFO] test episode 62: reward = 266.62, steps = 187
    00:00:05 [INFO] test episode 63: reward = 299.65, steps = 178
    00:00:05 [INFO] test episode 64: reward = 271.71, steps = 165
    00:00:05 [INFO] test episode 65: reward = 266.51, steps = 177
    00:00:05 [INFO] test episode 66: reward = 274.34, steps = 192
    00:00:05 [INFO] test episode 67: reward = 255.10, steps = 187
    00:00:05 [INFO] test episode 68: reward = 258.03, steps = 186
    00:00:05 [INFO] test episode 69: reward = 271.00, steps = 168
    00:00:05 [INFO] test episode 70: reward = 288.76, steps = 150
    00:00:06 [INFO] test episode 71: reward = 285.59, steps = 183
    00:00:06 [INFO] test episode 72: reward = 243.31, steps = 174
    00:00:06 [INFO] test episode 73: reward = 292.97, steps = 214
    00:00:06 [INFO] test episode 74: reward = 283.21, steps = 195
    00:00:06 [INFO] test episode 75: reward = 274.53, steps = 155
    00:00:06 [INFO] test episode 76: reward = 280.81, steps = 176
    00:00:06 [INFO] test episode 77: reward = 289.10, steps = 157
    00:00:06 [INFO] test episode 78: reward = 301.13, steps = 226
    00:00:06 [INFO] test episode 79: reward = 287.61, steps = 201
    00:00:06 [INFO] test episode 80: reward = 303.46, steps = 222
    00:00:06 [INFO] test episode 81: reward = 306.60, steps = 237
    00:00:06 [INFO] test episode 82: reward = 263.94, steps = 180
    00:00:07 [INFO] test episode 83: reward = 298.16, steps = 241
    00:00:07 [INFO] test episode 84: reward = 242.42, steps = 170
    00:00:07 [INFO] test episode 85: reward = 270.51, steps = 168
    00:00:07 [INFO] test episode 86: reward = 252.28, steps = 173
    00:00:07 [INFO] test episode 87: reward = 292.73, steps = 217
    00:00:07 [INFO] test episode 88: reward = 277.99, steps = 166
    00:00:07 [INFO] test episode 89: reward = 303.27, steps = 260
    00:00:07 [INFO] test episode 90: reward = 306.44, steps = 242
    00:00:07 [INFO] test episode 91: reward = 295.07, steps = 218
    00:00:07 [INFO] test episode 92: reward = 280.06, steps = 177
    00:00:07 [INFO] test episode 93: reward = 290.12, steps = 152
    00:00:08 [INFO] test episode 94: reward = 256.74, steps = 191
    00:00:08 [INFO] test episode 95: reward = 286.54, steps = 178
    00:00:08 [INFO] test episode 96: reward = 274.86, steps = 205
    00:00:08 [INFO] test episode 97: reward = 310.17, steps = 247
    00:00:08 [INFO] test episode 98: reward = 305.73, steps = 230
    00:00:08 [INFO] test episode 99: reward = 279.47, steps = 193
    00:00:08 [INFO] average episode reward = 282.56 ± 17.56

In [5]:

    env.close()