# Use OffPAC to Play Acrobot-v1¶

PyTorch version

In [1]:

    %matplotlib inline

    import sys
    import logging
    import itertools

    import numpy as np
    np.random.seed(0)
    import pandas as pd
    import gym
    import matplotlib.pyplot as plt
    import torch
    import torch.nn as nn
    import torch.optim as optim
    import torch.distributions as distributions
    torch.manual_seed(0)

    logging.basicConfig(level=logging.INFO,
            format='%(asctime)s [%(levelname)s] %(message)s',
            stream=sys.stdout, datefmt='%H:%M:%S')

In [2]:

    env = gym.make('Acrobot-v1')
    for key in vars(env):
        logging.info('%s: %s', key, vars(env)[key])
    for key in vars(env.spec):
        logging.info('%s: %s', key, vars(env.spec)[key])

    22:37:18 [INFO] env: <AcrobotEnv<Acrobot-v1>>
    22:37:18 [INFO] action_space: Discrete(3)
    22:37:18 [INFO] observation_space: Box(-28.274333953857422, 28.274333953857422, (6,), float32)
    22:37:18 [INFO] reward_range: (-inf, inf)
    22:37:18 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 15}
    22:37:18 [INFO] _max_episode_steps: 500
    22:37:18 [INFO] _elapsed_steps: None
    22:37:18 [INFO] id: Acrobot-v1
    22:37:18 [INFO] entry_point: gym.envs.classic_control:AcrobotEnv
    22:37:18 [INFO] reward_threshold: -100.0
    22:37:18 [INFO] nondeterministic: False
    22:37:18 [INFO] max_episode_steps: 500
    22:37:18 [INFO] _kwargs: {}
    22:37:18 [INFO] _env_name: Acrobot

In [3]:

    class OffPACAgent:
        def __init__(self, env):
            self.action_n = env.action_space.n
            self.gamma = 0.99

            self.actor_net = self.build_net(
                    input_size=env.observation_space.shape[0],
                    hidden_sizes=[100,],
                    output_size=env.action_space.n, output_activator=nn.Softmax(1))
            self.actor_optimizer = optim.Adam(self.actor_net.parameters(), 0.0002)
            self.critic_net = self.build_net(
                    input_size=env.observation_space.shape[0],
                    hidden_sizes=[100,], output_size=self.action_n)
            self.critic_optimizer = optim.Adam(self.critic_net.parameters(), 0.0004)
            self.critic_loss = nn.MSELoss()

        def build_net(self, input_size, hidden_sizes, output_size,
                output_activator=None):
            layers = []
            for input_size, output_size in zip(
                    [input_size,] + hidden_sizes, hidden_sizes + [output_size,]):
                layers.append(nn.Linear(input_size, output_size))
                layers.append(nn.ReLU())
            layers = layers[:-1]
            if output_activator:
                layers.append(output_activator)
            net = nn.Sequential(*layers)
            return net

        def reset(self, mode=None):
            self.mode = mode
            if self.mode == 'train':
                self.trajectory = []
                self.discount = 1.

        def step(self, observation, reward, terminated):
            if self.mode == 'train':
                action = np.random.choice(self.action_n)
                self.trajectory += [observation, reward, terminated, action]
                if len(self.trajectory) >= 8:
                    self.learn()
                self.discount *= self.gamma
            else:
                state_tensor = torch.as_tensor(observation,
                        dtype=torch.float).unsqueeze(0)
                prob_tensor = self.actor_net(state_tensor)
                action_tensor = distributions.Categorical(prob_tensor).sample()
                action = action_tensor.numpy()[0]
            return action

        def close(self):
            pass

        def learn(self):
            state, _, _, action, next_state, reward, terminated, next_action = \
                    self.trajectory[-8:]
            state_tensor = torch.as_tensor(state, dtype=torch.float).unsqueeze(0)
            next_state_tensor = torch.as_tensor(state, dtype=torch.float).unsqueeze(0)

            # update actor
            q_tensor = self.critic_net(state_tensor)[0, action]
            pi_tensor = self.actor_net(state_tensor)[0, action]
            behavior_prob = 1. / self.action_n
            actor_loss_tensor = -self.discount * q_tensor / behavior_prob * pi_tensor
            self.actor_optimizer.zero_grad()
            actor_loss_tensor.backward()
            self.actor_optimizer.step()

            # update critic
            next_q_tensor = self.critic_net(next_state_tensor)[:, next_action]
            target_tensor = reward + (1. - terminated) * self.gamma * next_q_tensor
            pred_tensor = self.critic_net(state_tensor)[:, action]
            critic_loss_tensor = self.critic_loss(pred_tensor, target_tensor)
            pi_tensor = self.actor_net(state_tensor)[0, action]
            ratio_tensor = pi_tensor / behavior_prob # importance sampling ratio
            critic_loss_tensor *= ratio_tensor
            self.critic_optimizer.zero_grad()
            critic_loss_tensor.backward()
            self.critic_optimizer.step()

    agent = OffPACAgent(env)

In [4]:

    def play_episode(env, agent, seed=None, mode=None, render=False):
        observation, _ = env.reset(seed=seed)
        reward, terminated, truncated = 0., False, False
        agent.reset(mode=mode)
        episode_reward, elapsed_steps = 0., 0
        while True:
            action = agent.step(observation, reward, terminated)
            if render:
                env.render()
            if terminated or truncated:
                break
            observation, reward, terminated, truncated, _ = env.step(action)
            episode_reward += reward
            elapsed_steps += 1
        agent.close()
        return episode_reward, elapsed_steps

    logging.info('==== train ====')
    episode_rewards = []
    for episode in itertools.count():
        play_episode(env, agent, seed=episode,
                mode='train')
        episode_reward, elapsed_steps = play_episode(env, agent)
        episode_rewards.append(episode_reward)
        logging.info('train episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
        if np.mean(episode_rewards[-10:]) > -120:
            break
    plt.plot(episode_rewards)

    logging.info('==== test ====')
    episode_rewards = []
    for episode in range(100):
        episode_reward, elapsed_steps = play_episode(env, agent)
        episode_rewards.append(episode_reward)
        logging.info('test episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
    logging.info('average episode reward = %.2f ± %.2f',
            np.mean(episode_rewards), np.std(episode_rewards))

    22:37:18 [INFO] ==== train ====
    22:37:20 [INFO] train episode 0: reward = -500.00, steps = 500
    22:37:22 [INFO] train episode 1: reward = -431.00, steps = 432
    22:37:24 [INFO] train episode 2: reward = -500.00, steps = 500
    22:37:26 [INFO] train episode 3: reward = -500.00, steps = 500
    22:37:28 [INFO] train episode 4: reward = -466.00, steps = 467
    22:37:30 [INFO] train episode 5: reward = -500.00, steps = 500
    22:37:31 [INFO] train episode 6: reward = -348.00, steps = 349
    22:37:33 [INFO] train episode 7: reward = -325.00, steps = 326
    22:37:35 [INFO] train episode 8: reward = -500.00, steps = 500
    22:37:36 [INFO] train episode 9: reward = -500.00, steps = 500
    22:37:38 [INFO] train episode 10: reward = -457.00, steps = 458
    22:37:40 [INFO] train episode 11: reward = -356.00, steps = 357
    22:37:42 [INFO] train episode 12: reward = -500.00, steps = 500
    22:37:43 [INFO] train episode 13: reward = -500.00, steps = 500
    22:37:45 [INFO] train episode 14: reward = -480.00, steps = 481
    22:37:47 [INFO] train episode 15: reward = -500.00, steps = 500
    22:37:49 [INFO] train episode 16: reward = -500.00, steps = 500
    22:37:50 [INFO] train episode 17: reward = -500.00, steps = 500
    22:37:52 [INFO] train episode 18: reward = -500.00, steps = 500
    22:37:53 [INFO] train episode 19: reward = -500.00, steps = 500
    22:37:55 [INFO] train episode 20: reward = -500.00, steps = 500
    22:37:57 [INFO] train episode 21: reward = -500.00, steps = 500
    22:37:58 [INFO] train episode 22: reward = -500.00, steps = 500
    22:38:00 [INFO] train episode 23: reward = -500.00, steps = 500
    22:38:02 [INFO] train episode 24: reward = -500.00, steps = 500
    22:38:04 [INFO] train episode 25: reward = -500.00, steps = 500
    22:38:06 [INFO] train episode 26: reward = -500.00, steps = 500
    22:38:08 [INFO] train episode 27: reward = -490.00, steps = 491
    22:38:10 [INFO] train episode 28: reward = -500.00, steps = 500
    22:38:12 [INFO] train episode 29: reward = -500.00, steps = 500
    22:38:14 [INFO] train episode 30: reward = -452.00, steps = 453
    22:38:16 [INFO] train episode 31: reward = -500.00, steps = 500
    22:38:18 [INFO] train episode 32: reward = -355.00, steps = 356
    22:38:20 [INFO] train episode 33: reward = -500.00, steps = 500
    22:38:22 [INFO] train episode 34: reward = -500.00, steps = 500
    22:38:24 [INFO] train episode 35: reward = -331.00, steps = 332
    22:38:26 [INFO] train episode 36: reward = -500.00, steps = 500
    22:38:28 [INFO] train episode 37: reward = -473.00, steps = 474
    22:38:29 [INFO] train episode 38: reward = -253.00, steps = 254
    22:38:31 [INFO] train episode 39: reward = -353.00, steps = 354
    22:38:33 [INFO] train episode 40: reward = -500.00, steps = 500
    22:38:35 [INFO] train episode 41: reward = -394.00, steps = 395
    22:38:37 [INFO] train episode 42: reward = -196.00, steps = 197
    22:38:39 [INFO] train episode 43: reward = -374.00, steps = 375
    22:38:40 [INFO] train episode 44: reward = -357.00, steps = 358
    22:38:42 [INFO] train episode 45: reward = -231.00, steps = 232
    22:38:44 [INFO] train episode 46: reward = -451.00, steps = 452
    22:38:46 [INFO] train episode 47: reward = -500.00, steps = 500
    22:38:48 [INFO] train episode 48: reward = -291.00, steps = 292
    22:38:50 [INFO] train episode 49: reward = -500.00, steps = 500
    22:38:52 [INFO] train episode 50: reward = -500.00, steps = 500
    22:38:54 [INFO] train episode 51: reward = -500.00, steps = 500
    22:38:56 [INFO] train episode 52: reward = -500.00, steps = 500
    22:38:58 [INFO] train episode 53: reward = -500.00, steps = 500
    22:39:00 [INFO] train episode 54: reward = -500.00, steps = 500
    22:39:03 [INFO] train episode 55: reward = -500.00, steps = 500
    22:39:05 [INFO] train episode 56: reward = -500.00, steps = 500
    22:39:07 [INFO] train episode 57: reward = -433.00, steps = 434
    22:39:09 [INFO] train episode 58: reward = -500.00, steps = 500
    22:39:11 [INFO] train episode 59: reward = -364.00, steps = 365
    22:39:13 [INFO] train episode 60: reward = -482.00, steps = 483
    22:39:14 [INFO] train episode 61: reward = -500.00, steps = 500
    22:39:16 [INFO] train episode 62: reward = -500.00, steps = 500
    22:39:18 [INFO] train episode 63: reward = -500.00, steps = 500
    22:39:20 [INFO] train episode 64: reward = -489.00, steps = 490
    22:39:22 [INFO] train episode 65: reward = -500.00, steps = 500
    22:39:24 [INFO] train episode 66: reward = -417.00, steps = 418
    22:39:26 [INFO] train episode 67: reward = -500.00, steps = 500
    22:39:28 [INFO] train episode 68: reward = -308.00, steps = 309
    22:39:30 [INFO] train episode 69: reward = -492.00, steps = 493
    22:39:32 [INFO] train episode 70: reward = -234.00, steps = 235
    22:39:34 [INFO] train episode 71: reward = -249.00, steps = 250
    22:39:36 [INFO] train episode 72: reward = -500.00, steps = 500
    22:39:38 [INFO] train episode 73: reward = -264.00, steps = 265
    22:39:40 [INFO] train episode 74: reward = -273.00, steps = 274
    22:39:42 [INFO] train episode 75: reward = -154.00, steps = 155
    22:39:44 [INFO] train episode 76: reward = -136.00, steps = 137
    22:39:46 [INFO] train episode 77: reward = -246.00, steps = 247
    22:39:48 [INFO] train episode 78: reward = -201.00, steps = 202
    22:39:50 [INFO] train episode 79: reward = -348.00, steps = 349
    22:39:52 [INFO] train episode 80: reward = -159.00, steps = 160
    22:39:54 [INFO] train episode 81: reward = -361.00, steps = 362
    22:39:55 [INFO] train episode 82: reward = -131.00, steps = 132
    22:39:58 [INFO] train episode 83: reward = -177.00, steps = 178
    22:39:59 [INFO] train episode 84: reward = -203.00, steps = 204
    22:40:01 [INFO] train episode 85: reward = -146.00, steps = 147
    22:40:03 [INFO] train episode 86: reward = -176.00, steps = 177
    22:40:04 [INFO] train episode 87: reward = -232.00, steps = 233
    22:40:06 [INFO] train episode 88: reward = -237.00, steps = 238
    22:40:08 [INFO] train episode 89: reward = -297.00, steps = 298
    22:40:10 [INFO] train episode 90: reward = -269.00, steps = 270
    22:40:12 [INFO] train episode 91: reward = -198.00, steps = 199
    22:40:13 [INFO] train episode 92: reward = -192.00, steps = 193
    22:40:15 [INFO] train episode 93: reward = -384.00, steps = 385
    22:40:17 [INFO] train episode 94: reward = -307.00, steps = 308
    22:40:19 [INFO] train episode 95: reward = -223.00, steps = 224
    22:40:20 [INFO] train episode 96: reward = -225.00, steps = 226
    22:40:22 [INFO] train episode 97: reward = -282.00, steps = 283
    22:40:24 [INFO] train episode 98: reward = -151.00, steps = 152
    22:40:26 [INFO] train episode 99: reward = -233.00, steps = 234
    22:40:28 [INFO] train episode 100: reward = -240.00, steps = 241
    22:40:29 [INFO] train episode 101: reward = -197.00, steps = 198
    22:40:31 [INFO] train episode 102: reward = -191.00, steps = 192
    22:40:33 [INFO] train episode 103: reward = -172.00, steps = 173
    22:40:34 [INFO] train episode 104: reward = -176.00, steps = 177
    22:40:36 [INFO] train episode 105: reward = -137.00, steps = 138
    22:40:38 [INFO] train episode 106: reward = -191.00, steps = 192
    22:40:39 [INFO] train episode 107: reward = -163.00, steps = 164
    22:40:41 [INFO] train episode 108: reward = -153.00, steps = 154
    22:40:43 [INFO] train episode 109: reward = -124.00, steps = 125
    22:40:44 [INFO] train episode 110: reward = -127.00, steps = 128
    22:40:46 [INFO] train episode 111: reward = -130.00, steps = 131
    22:40:48 [INFO] train episode 112: reward = -91.00, steps = 92
    22:40:49 [INFO] train episode 113: reward = -136.00, steps = 137
    22:40:51 [INFO] train episode 114: reward = -133.00, steps = 134
    22:40:53 [INFO] train episode 115: reward = -121.00, steps = 122
    22:40:54 [INFO] train episode 116: reward = -107.00, steps = 108
    22:40:56 [INFO] train episode 117: reward = -139.00, steps = 140
    22:40:57 [INFO] train episode 118: reward = -118.00, steps = 119
    22:40:59 [INFO] train episode 119: reward = -87.00, steps = 88
    22:40:59 [INFO] ==== test ====
    22:40:59 [INFO] test episode 0: reward = -131.00, steps = 132
    22:40:59 [INFO] test episode 1: reward = -118.00, steps = 119
    22:40:59 [INFO] test episode 2: reward = -118.00, steps = 119
    22:40:59 [INFO] test episode 3: reward = -150.00, steps = 151
    22:41:00 [INFO] test episode 4: reward = -148.00, steps = 149
    22:41:00 [INFO] test episode 5: reward = -128.00, steps = 129
    22:41:00 [INFO] test episode 6: reward = -119.00, steps = 120
    22:41:00 [INFO] test episode 7: reward = -135.00, steps = 136
    22:41:00 [INFO] test episode 8: reward = -103.00, steps = 104
    22:41:00 [INFO] test episode 9: reward = -103.00, steps = 104
    22:41:00 [INFO] test episode 10: reward = -114.00, steps = 115
    22:41:00 [INFO] test episode 11: reward = -122.00, steps = 123
    22:41:00 [INFO] test episode 12: reward = -105.00, steps = 106
    22:41:00 [INFO] test episode 13: reward = -145.00, steps = 146
    22:41:00 [INFO] test episode 14: reward = -135.00, steps = 136
    22:41:00 [INFO] test episode 15: reward = -112.00, steps = 113
    22:41:00 [INFO] test episode 16: reward = -99.00, steps = 100
    22:41:01 [INFO] test episode 17: reward = -144.00, steps = 145
    22:41:01 [INFO] test episode 18: reward = -111.00, steps = 112
    22:41:01 [INFO] test episode 19: reward = -136.00, steps = 137
    22:41:01 [INFO] test episode 20: reward = -121.00, steps = 122
    22:41:01 [INFO] test episode 21: reward = -130.00, steps = 131
    22:41:01 [INFO] test episode 22: reward = -109.00, steps = 110
    22:41:01 [INFO] test episode 23: reward = -154.00, steps = 155
    22:41:01 [INFO] test episode 24: reward = -90.00, steps = 91
    22:41:01 [INFO] test episode 25: reward = -102.00, steps = 103
    22:41:01 [INFO] test episode 26: reward = -123.00, steps = 124
    22:41:01 [INFO] test episode 27: reward = -93.00, steps = 94
    22:41:02 [INFO] test episode 28: reward = -151.00, steps = 152
    22:41:02 [INFO] test episode 29: reward = -106.00, steps = 107
    22:41:02 [INFO] test episode 30: reward = -126.00, steps = 127
    22:41:02 [INFO] test episode 31: reward = -113.00, steps = 114
    22:41:02 [INFO] test episode 32: reward = -128.00, steps = 129
    22:41:02 [INFO] test episode 33: reward = -104.00, steps = 105
    22:41:02 [INFO] test episode 34: reward = -111.00, steps = 112
    22:41:02 [INFO] test episode 35: reward = -92.00, steps = 93
    22:41:02 [INFO] test episode 36: reward = -93.00, steps = 94
    22:41:02 [INFO] test episode 37: reward = -112.00, steps = 113
    22:41:02 [INFO] test episode 38: reward = -135.00, steps = 136
    22:41:02 [INFO] test episode 39: reward = -118.00, steps = 119
    22:41:02 [INFO] test episode 40: reward = -108.00, steps = 109
    22:41:03 [INFO] test episode 41: reward = -147.00, steps = 148
    22:41:03 [INFO] test episode 42: reward = -117.00, steps = 118
    22:41:03 [INFO] test episode 43: reward = -184.00, steps = 185
    22:41:03 [INFO] test episode 44: reward = -113.00, steps = 114
    22:41:03 [INFO] test episode 45: reward = -98.00, steps = 99
    22:41:03 [INFO] test episode 46: reward = -132.00, steps = 133
    22:41:03 [INFO] test episode 47: reward = -94.00, steps = 95
    22:41:03 [INFO] test episode 48: reward = -100.00, steps = 101
    22:41:03 [INFO] test episode 49: reward = -133.00, steps = 134
    22:41:03 [INFO] test episode 50: reward = -125.00, steps = 126
    22:41:03 [INFO] test episode 51: reward = -163.00, steps = 164
    22:41:03 [INFO] test episode 52: reward = -179.00, steps = 180
    22:41:04 [INFO] test episode 53: reward = -152.00, steps = 153
    22:41:04 [INFO] test episode 54: reward = -263.00, steps = 264
    22:41:04 [INFO] test episode 55: reward = -103.00, steps = 104
    22:41:04 [INFO] test episode 56: reward = -110.00, steps = 111
    22:41:04 [INFO] test episode 57: reward = -159.00, steps = 160
    22:41:04 [INFO] test episode 58: reward = -157.00, steps = 158
    22:41:04 [INFO] test episode 59: reward = -138.00, steps = 139
    22:41:04 [INFO] test episode 60: reward = -124.00, steps = 125
    22:41:04 [INFO] test episode 61: reward = -117.00, steps = 118
    22:41:04 [INFO] test episode 62: reward = -184.00, steps = 185
    22:41:05 [INFO] test episode 63: reward = -120.00, steps = 121
    22:41:05 [INFO] test episode 64: reward = -111.00, steps = 112
    22:41:05 [INFO] test episode 65: reward = -116.00, steps = 117
    22:41:05 [INFO] test episode 66: reward = -122.00, steps = 123
    22:41:05 [INFO] test episode 67: reward = -135.00, steps = 136
    22:41:05 [INFO] test episode 68: reward = -138.00, steps = 139
    22:41:05 [INFO] test episode 69: reward = -117.00, steps = 118
    22:41:05 [INFO] test episode 70: reward = -123.00, steps = 124
    22:41:05 [INFO] test episode 71: reward = -96.00, steps = 97
    22:41:05 [INFO] test episode 72: reward = -113.00, steps = 114
    22:41:05 [INFO] test episode 73: reward = -90.00, steps = 91
    22:41:05 [INFO] test episode 74: reward = -116.00, steps = 117
    22:41:05 [INFO] test episode 75: reward = -132.00, steps = 133
    22:41:06 [INFO] test episode 76: reward = -117.00, steps = 118
    22:41:06 [INFO] test episode 77: reward = -104.00, steps = 105
    22:41:06 [INFO] test episode 78: reward = -142.00, steps = 143
    22:41:06 [INFO] test episode 79: reward = -124.00, steps = 125
    22:41:06 [INFO] test episode 80: reward = -92.00, steps = 93
    22:41:06 [INFO] test episode 81: reward = -132.00, steps = 133
    22:41:06 [INFO] test episode 82: reward = -121.00, steps = 122
    22:41:06 [INFO] test episode 83: reward = -93.00, steps = 94
    22:41:06 [INFO] test episode 84: reward = -103.00, steps = 104
    22:41:06 [INFO] test episode 85: reward = -198.00, steps = 199
    22:41:06 [INFO] test episode 86: reward = -190.00, steps = 191
    22:41:07 [INFO] test episode 87: reward = -142.00, steps = 143
    22:41:07 [INFO] test episode 88: reward = -134.00, steps = 135
    22:41:07 [INFO] test episode 89: reward = -126.00, steps = 127
    22:41:07 [INFO] test episode 90: reward = -144.00, steps = 145
    22:41:07 [INFO] test episode 91: reward = -113.00, steps = 114
    22:41:07 [INFO] test episode 92: reward = -103.00, steps = 104
    22:41:07 [INFO] test episode 93: reward = -117.00, steps = 118
    22:41:07 [INFO] test episode 94: reward = -142.00, steps = 143
    22:41:07 [INFO] test episode 95: reward = -132.00, steps = 133
    22:41:07 [INFO] test episode 96: reward = -115.00, steps = 116
    22:41:07 [INFO] test episode 97: reward = -102.00, steps = 103
    22:41:07 [INFO] test episode 98: reward = -86.00, steps = 87
    22:41:07 [INFO] test episode 99: reward = -137.00, steps = 138
    22:41:07 [INFO] average episode reward = -125.25 ± 26.61

In [5]:

    env.close()