Use Closed-Form Policy to Play BreakoutNoFrameskip-v4¶

In [1]:
import sys
import logging
import itertools

import numpy as np
np.random.seed(0)
import gym

logging.basicConfig(level=logging.INFO,
        format='%(asctime)s [%(levelname)s] %(message)s',
        stream=sys.stdout, datefmt='%H:%M:%S')
In [2]:
env = gym.make('BreakoutNoFrameskip-v4')
for key in vars(env):
    logging.info('%s: %s', key, vars(env)[key])
for key in vars(env.spec):
    logging.info('%s: %s', key, vars(env.spec)[key])

00:03:37 [INFO] env: <AtariEnv<BreakoutNoFrameskip-v4>>
00:03:37 [INFO] action_space: Discrete(4)
00:03:37 [INFO] observation_space: Box(0, 255, (210, 160, 3), uint8)
00:03:37 [INFO] reward_range: (-inf, inf)
00:03:37 [INFO] metadata: {'render.modes': ['human', 'rgb_array']}
00:03:37 [INFO] _max_episode_steps: 400000
00:03:37 [INFO] _elapsed_steps: None
00:03:37 [INFO] id: BreakoutNoFrameskip-v4
00:03:37 [INFO] entry_point: gym.envs.atari:AtariEnv
00:03:37 [INFO] reward_threshold: None
00:03:37 [INFO] nondeterministic: False
00:03:37 [INFO] max_episode_steps: 400000
00:03:37 [INFO] _kwargs: {'game': 'breakout', 'obs_type': 'image', 'frameskip': 1}
00:03:37 [INFO] _env_name: BreakoutNoFrameskip
In [3]:
def calc_mean(locs, value=float('nan')):
    indices = locs.nonzero()[0]
    if len(indices) == 0:
        return value
    return np.nanmean(indices)


class ClosedFormAgent:
    def __init__(self, _):
        pass

    def reset(self, mode=None):
        self.pad_x = 72.
        self.ball_x = 72.
        self.ball_y = 95.

    def step(self, observation, reward, terminated):
        pixels = np.flipud(observation[95:190, 8:152, 0]) == 200
        pad_x = calc_mean(pixels[0])
        ball_x = calc_mean(pixels[1:].any(axis=0))
        ball_y = calc_mean(pixels[1:].any(axis=1)) + 1.

        pad_xv = pad_x - self.pad_x
        ball_xv = ball_x - self.ball_x
        ball_yv = ball_y - self.ball_y
        target_x = abs(ball_x - ball_xv / ball_yv * ball_y)
        pred_x = pad_x + pad_xv / 2. + np.random.randn() / 3.
        if pred_x < target_x - 1 and pred_x + 5. < pixels.shape[1]:
            action = 2 # right
        elif pred_x > target_x + 1 and pred_x - 5. >= 0:
            action = 3 # left
        else:
            action = 1 # no move
        self.pad_x = pad_x
        self.ball_x = ball_x
        self.ball_y = ball_y
        return action

    def close(self):
        pass


agent = ClosedFormAgent(env)
In [4]:
def play_episode(env, agent, seed=None, mode=None, render=False):
    observation, _ = env.reset(seed=seed)
    reward, terminated, truncated = 0., False, False
    agent.reset(mode=mode)
    episode_reward, elapsed_steps = 0., 0
    while True:
        action = agent.step(observation, reward, terminated)
        if render:
            env.render()
        if terminated or truncated:
            break
        observation, reward, terminated, truncated, _ = env.step(action)
        episode_reward += reward
        elapsed_steps += 1
    agent.close()
    return episode_reward, elapsed_steps


logging.info('==== test ====')
episode_rewards = []
for episode in range(100):
    episode_reward, elapsed_steps = play_episode(env, agent)
    episode_rewards.append(episode_reward)
    logging.info('test episode %d: reward = %.2f, steps = %d',
            episode, episode_reward, elapsed_steps)
logging.info('average episode reward = %.2f ± %.2f',
        np.mean(episode_rewards), np.std(episode_rewards))

00:03:37 [INFO] ==== test ====
00:15:32 [INFO] test episode 0: reward = 397.00, steps = 400000
00:16:12 [INFO] test episode 1: reward = 860.00, steps = 22312
00:27:02 [INFO] test episode 2: reward = 864.00, steps = 400000
00:27:51 [INFO] test episode 3: reward = 864.00, steps = 27045
00:28:42 [INFO] test episode 4: reward = 864.00, steps = 28393
00:30:24 [INFO] test episode 5: reward = 801.00, steps = 57369
00:41:15 [INFO] test episode 6: reward = 428.00, steps = 400000
00:42:26 [INFO] test episode 7: reward = 864.00, steps = 39369
00:53:12 [INFO] test episode 8: reward = 864.00, steps = 400000
00:54:33 [INFO] test episode 9: reward = 848.00, steps = 46194
00:55:14 [INFO] test episode 10: reward = 824.00, steps = 23012
00:55:44 [INFO] test episode 11: reward = 669.00, steps = 16171
00:56:15 [INFO] test episode 12: reward = 745.00, steps = 17462
00:57:22 [INFO] test episode 13: reward = 629.00, steps = 37797
00:58:10 [INFO] test episode 14: reward = 864.00, steps = 27505
00:59:23 [INFO] test episode 15: reward = 864.00, steps = 42170
01:10:13 [INFO] test episode 16: reward = 856.00, steps = 400000
01:11:47 [INFO] test episode 17: reward = 453.00, steps = 53991
01:13:15 [INFO] test episode 18: reward = 843.00, steps = 50266
01:13:37 [INFO] test episode 19: reward = 432.00, steps = 12492
01:14:09 [INFO] test episode 20: reward = 810.00, steps = 18044
01:24:57 [INFO] test episode 21: reward = 863.00, steps = 400000
01:25:09 [INFO] test episode 22: reward = 215.00, steps = 6719
01:25:26 [INFO] test episode 23: reward = 386.00, steps = 9549
01:27:16 [INFO] test episode 24: reward = 864.00, steps = 63149
01:28:45 [INFO] test episode 25: reward = 864.00, steps = 51313
01:38:44 [INFO] test episode 26: reward = 864.00, steps = 400000
01:39:12 [INFO] test episode 27: reward = 431.00, steps = 16731
01:39:39 [INFO] test episode 28: reward = 548.00, steps = 16400
01:40:09 [INFO] test episode 29: reward = 843.00, steps = 18426
01:50:03 [INFO] test episode 30: reward = 864.00, steps = 400000
01:59:52 [INFO] test episode 31: reward = 864.00, steps = 400000
02:00:53 [INFO] test episode 32: reward = 848.00, steps = 37361
02:02:06 [INFO] test episode 33: reward = 859.00, steps = 45441
02:03:28 [INFO] test episode 34: reward = 864.00, steps = 50338
02:03:59 [INFO] test episode 35: reward = 476.00, steps = 19107
02:04:36 [INFO] test episode 36: reward = 425.00, steps = 22636
02:05:49 [INFO] test episode 37: reward = 847.00, steps = 44566
02:15:37 [INFO] test episode 38: reward = 864.00, steps = 400000
02:25:35 [INFO] test episode 39: reward = 856.00, steps = 400000
02:35:31 [INFO] test episode 40: reward = 856.00, steps = 400000
02:35:58 [INFO] test episode 41: reward = 406.00, steps = 16452
02:36:49 [INFO] test episode 42: reward = 728.00, steps = 31606
02:38:02 [INFO] test episode 43: reward = 864.00, steps = 45497
02:39:42 [INFO] test episode 44: reward = 424.00, steps = 61532
02:49:38 [INFO] test episode 45: reward = 850.00, steps = 400000
02:59:29 [INFO] test episode 46: reward = 864.00, steps = 400000
03:00:29 [INFO] test episode 47: reward = 438.00, steps = 37438
03:01:17 [INFO] test episode 48: reward = 853.00, steps = 29429
03:11:15 [INFO] test episode 49: reward = 424.00, steps = 400000
03:21:22 [INFO] test episode 50: reward = 425.00, steps = 400000
03:22:48 [INFO] test episode 51: reward = 864.00, steps = 53779
03:23:11 [INFO] test episode 52: reward = 453.00, steps = 14042
03:24:04 [INFO] test episode 53: reward = 450.00, steps = 33371
03:25:30 [INFO] test episode 54: reward = 856.00, steps = 53673
03:26:28 [INFO] test episode 55: reward = 864.00, steps = 35714
03:27:17 [INFO] test episode 56: reward = 425.00, steps = 29852
03:27:54 [INFO] test episode 57: reward = 864.00, steps = 22913
03:28:49 [INFO] test episode 58: reward = 835.00, steps = 34279
03:38:48 [INFO] test episode 59: reward = 831.00, steps = 400000
03:48:36 [INFO] test episode 60: reward = 864.00, steps = 400000
03:58:58 [INFO] test episode 61: reward = 615.00, steps = 400000
03:59:46 [INFO] test episode 62: reward = 864.00, steps = 29846
04:00:34 [INFO] test episode 63: reward = 619.00, steps = 29569
04:01:43 [INFO] test episode 64: reward = 864.00, steps = 43296
04:02:29 [INFO] test episode 65: reward = 771.00, steps = 28955
04:03:41 [INFO] test episode 66: reward = 532.00, steps = 45177
04:04:21 [INFO] test episode 67: reward = 864.00, steps = 24223
04:14:12 [INFO] test episode 68: reward = 428.00, steps = 400000
04:14:50 [INFO] test episode 69: reward = 856.00, steps = 24051
04:15:35 [INFO] test episode 70: reward = 442.00, steps = 28101
04:16:10 [INFO] test episode 71: reward = 782.00, steps = 21433
04:16:46 [INFO] test episode 72: reward = 807.00, steps = 22304
04:26:37 [INFO] test episode 73: reward = 431.00, steps = 400000
04:27:37 [INFO] test episode 74: reward = 857.00, steps = 37242
04:37:22 [INFO] test episode 75: reward = 864.00, steps = 400000
04:47:07 [INFO] test episode 76: reward = 864.00, steps = 400000
04:48:11 [INFO] test episode 77: reward = 551.00, steps = 40579
04:48:37 [INFO] test episode 78: reward = 504.00, steps = 15823
04:49:10 [INFO] test episode 79: reward = 840.00, steps = 20024
04:49:46 [INFO] test episode 80: reward = 738.00, steps = 22279
04:51:01 [INFO] test episode 81: reward = 864.00, steps = 46719
05:00:49 [INFO] test episode 82: reward = 428.00, steps = 400000
05:10:40 [INFO] test episode 83: reward = 860.00, steps = 400000
05:12:03 [INFO] test episode 84: reward = 864.00, steps = 52588
05:12:38 [INFO] test episode 85: reward = 857.00, steps = 21521
05:13:16 [INFO] test episode 86: reward = 436.00, steps = 23530
05:14:00 [INFO] test episode 87: reward = 860.00, steps = 26924
05:23:50 [INFO] test episode 88: reward = 428.00, steps = 400000
05:24:14 [INFO] test episode 89: reward = 415.00, steps = 14829
05:34:04 [INFO] test episode 90: reward = 864.00, steps = 400000
05:34:32 [INFO] test episode 91: reward = 455.00, steps = 17489
05:36:00 [INFO] test episode 92: reward = 863.00, steps = 53499
05:37:25 [INFO] test episode 93: reward = 864.00, steps = 53216
05:47:26 [INFO] test episode 94: reward = 836.00, steps = 400000
05:57:22 [INFO] test episode 95: reward = 852.00, steps = 400000
06:07:16 [INFO] test episode 96: reward = 428.00, steps = 400000
06:17:03 [INFO] test episode 97: reward = 864.00, steps = 400000
06:17:54 [INFO] test episode 98: reward = 853.00, steps = 31289
06:18:26 [INFO] test episode 99: reward = 809.00, steps = 19906
06:18:26 [INFO] average episode reward = 715.19 ± 191.04
In [5]:
env.close()