# Use PPO to Play Acrobot-v1¶

TensorFlow version

In [1]:

    %matplotlib inline

    import sys
    import logging
    import itertools

    import numpy as np
    np.random.seed(0)
    import pandas as pd
    import scipy.signal as signal
    import gym
    import matplotlib.pyplot as plt
    import tensorflow.compat.v2 as tf
    tf.random.set_seed(0)
    from tensorflow import keras
    from tensorflow import nn
    from tensorflow import optimizers
    from tensorflow import losses
    from tensorflow.keras import layers

    logging.basicConfig(level=logging.INFO,
            format='%(asctime)s [%(levelname)s] %(message)s',
            stream=sys.stdout, datefmt='%H:%M:%S')

In [2]:

    env = gym.make('Acrobot-v1')
    for key in vars(env):
        logging.info('%s: %s', key, vars(env)[key])
    for key in vars(env.spec):
        logging.info('%s: %s', key, vars(env.spec)[key])

    03:33:25 [INFO] env: <AcrobotEnv<Acrobot-v1>>
    03:33:25 [INFO] action_space: Discrete(3)
    03:33:25 [INFO] observation_space: Box(-28.274333953857422, 28.274333953857422, (6,), float32)
    03:33:25 [INFO] reward_range: (-inf, inf)
    03:33:25 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 15}
    03:33:25 [INFO] _max_episode_steps: 500
    03:33:25 [INFO] _elapsed_steps: None
    03:33:25 [INFO] id: Acrobot-v1
    03:33:25 [INFO] entry_point: gym.envs.classic_control:AcrobotEnv
    03:33:25 [INFO] reward_threshold: -100.0
    03:33:25 [INFO] nondeterministic: False
    03:33:25 [INFO] max_episode_steps: 500
    03:33:25 [INFO] _kwargs: {}
    03:33:25 [INFO] _env_name: Acrobot

In [3]:

    class PPOReplayer:
        def __init__(self):
            self.fields = ['state', 'action', 'prob', 'advantage', 'return']
            self.memory = pd.DataFrame(columns=self.fields)

        def store(self, df):
            self.memory = pd.concat([self.memory, df[self.fields]], ignore_index=True)

        def sample(self, size):
            indices = np.random.choice(self.memory.shape[0], size=size)
            return (np.stack(self.memory.loc[indices, field]) for field in
                    self.fields)

In [4]:

    class PPOAgent:
        def __init__(self, env):
            self.action_n = env.action_space.n
            self.gamma = 0.99

            self.replayer = PPOReplayer()

            self.actor_net = self.build_net(hidden_sizes=[100,],
                    output_size=self.action_n, output_activation=nn.softmax,
                    learning_rate=0.001)
            self.critic_net = self.build_net(hidden_sizes=[100,],
                    learning_rate=0.002)

        def build_net(self, input_size=None, hidden_sizes=None, output_size=1,
                    activation=nn.relu, output_activation=None,
                    loss=losses.mse, learning_rate=0.001):
            model = keras.Sequential()
            for hidden_size in hidden_sizes:
                model.add(layers.Dense(units=hidden_size,
                        activation=activation))
            model.add(layers.Dense(units=output_size,
                    activation=output_activation))
            optimizer = optimizers.Adam(learning_rate)
            model.compile(optimizer=optimizer, loss=loss)
            return model

        def reset(self, mode=None):
            self.mode = mode
            if self.mode == 'train':
                self.trajectory = []

        def step(self, observation, reward, terminated):
            probs = self.actor_net.predict(observation[np.newaxis], verbose=0)[0]
            action = np.random.choice(self.action_n, p=probs)
            if self.mode == 'train':
                self.trajectory += [observation, reward, terminated, action]
            return action

        def close(self):
            if self.mode == 'train':
                self.save_trajectory_to_replayer()
                if len(self.replayer.memory) >= 1000:
                    for batch in range(5):  # learn multiple times
                        self.learn()
                    self.replayer = PPOReplayer()
                            # reset replayer after the agent changes itself

        def save_trajectory_to_replayer(self):
            df = pd.DataFrame(
                    np.array(self.trajectory, dtype=object).reshape(-1, 4),
                    columns=['state', 'reward', 'terminated', 'action'], dtype=object)
            states = np.stack(df['state'])
            df['v'] = self.critic_net.predict(states, verbose=0)
            pis = self.actor_net.predict(states, verbose=0)
            df['prob'] = [pi[action] for pi, action in zip(pis, df['action'])]
            df['next_v'] = df['v'].shift(-1).fillna(0.)
            df['u'] = df['reward'] + self.gamma * df['next_v']
            df['delta'] = df['u'] - df['v']
            df['advantage'] = signal.lfilter([1.,], [1., -self.gamma],
                    df['delta'][::-1])[::-1]
            df['return'] = signal.lfilter([1.,], [1., -self.gamma],
                    df['reward'][::-1])[::-1]
            self.replayer.store(df)

        def learn(self):
            states, actions, old_pis, advantages, returns = \
                    self.replayer.sample(size=64)
            state_tensor = tf.convert_to_tensor(states, dtype=tf.float32)
            action_tensor = tf.convert_to_tensor(actions, dtype=tf.int32)
            old_pi_tensor = tf.convert_to_tensor(old_pis, dtype=tf.float32)
            advantage_tensor = tf.convert_to_tensor(advantages, dtype=tf.float32)

            # update actor
            with tf.GradientTape() as tape:
                all_pi_tensor = self.actor_net(state_tensor)
                pi_tensor = tf.gather(all_pi_tensor, action_tensor, batch_dims=1)
                surrogate_advantage_tensor = (pi_tensor / old_pi_tensor) * \
                        advantage_tensor
                clip_times_advantage_tensor = 0.1 * surrogate_advantage_tensor
                max_surrogate_advantage_tensor = advantage_tensor + \
                        tf.where(advantage_tensor > 0.,
                        clip_times_advantage_tensor, -clip_times_advantage_tensor)
                clipped_surrogate_advantage_tensor = tf.minimum(
                        surrogate_advantage_tensor, max_surrogate_advantage_tensor)
                loss_tensor = -tf.reduce_mean(clipped_surrogate_advantage_tensor)
            actor_grads = tape.gradient(loss_tensor, self.actor_net.variables)
            self.actor_net.optimizer.apply_gradients(
                    zip(actor_grads, self.actor_net.variables))

            # update critic
            self.critic_net.fit(states, returns, verbose=0)

    agent = PPOAgent(env)

In [5]:

    def play_episode(env, agent, seed=None, mode=None, render=False):
        observation, _ = env.reset(seed=seed)
        reward, terminated, truncated = 0., False, False
        agent.reset(mode=mode)
        episode_reward, elapsed_steps = 0., 0
        while True:
            action = agent.step(observation, reward, terminated)
            if render:
                env.render()
            if terminated or truncated:
                break
            observation, reward, terminated, truncated, _ = env.step(action)
            episode_reward += reward
            elapsed_steps += 1
        agent.close()
        return episode_reward, elapsed_steps

    logging.info('==== train ====')
    episode_rewards = []
    for episode in itertools.count():
        episode_reward, elapsed_steps = play_episode(env, agent, seed=episode,
                mode='train')
        episode_rewards.append(episode_reward)
        logging.info('train episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
        if np.mean(episode_rewards[-10:]) > -120:
            break
    plt.plot(episode_rewards)

    logging.info('==== test ====')
    episode_rewards = []
    for episode in range(100):
        episode_reward, elapsed_steps = play_episode(env, agent)
        episode_rewards.append(episode_reward)
        logging.info('test episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
    logging.info('average episode reward = %.2f ± %.2f',
            np.mean(episode_rewards), np.std(episode_rewards))

    03:33:26 [INFO] ==== train ====
    03:34:14 [INFO] train episode 0: reward = -500.00, steps = 500
    03:34:14 [INFO] NumExpr defaulting to 8 threads.
    03:35:03 [INFO] train episode 1: reward = -500.00, steps = 500
    03:35:52 [INFO] train episode 2: reward = -500.00, steps = 500
    03:36:40 [INFO] train episode 3: reward = -500.00, steps = 500
    03:37:27 [INFO] train episode 4: reward = -485.00, steps = 486
    03:38:14 [INFO] train episode 5: reward = -500.00, steps = 500
    03:38:45 [INFO] train episode 6: reward = -339.00, steps = 340
    03:39:27 [INFO] train episode 7: reward = -448.00, steps = 449
    03:39:54 [INFO] train episode 8: reward = -292.00, steps = 293
    03:40:29 [INFO] train episode 9: reward = -383.00, steps = 384
    03:41:16 [INFO] train episode 10: reward = -500.00, steps = 500
    03:41:42 [INFO] train episode 11: reward = -283.00, steps = 284
    03:42:05 [INFO] train episode 12: reward = -254.00, steps = 255
    03:42:30 [INFO] train episode 13: reward = -267.00, steps = 268
    03:43:16 [INFO] train episode 14: reward = -500.00, steps = 500
    03:44:01 [INFO] train episode 15: reward = -500.00, steps = 500
    03:44:47 [INFO] train episode 16: reward = -500.00, steps = 500
    03:45:31 [INFO] train episode 17: reward = -500.00, steps = 500
    03:46:17 [INFO] train episode 18: reward = -500.00, steps = 500
    03:47:03 [INFO] train episode 19: reward = -500.00, steps = 500
    03:47:50 [INFO] train episode 20: reward = -500.00, steps = 500
    03:48:36 [INFO] train episode 21: reward = -500.00, steps = 500
    03:49:23 [INFO] train episode 22: reward = -500.00, steps = 500
    03:50:09 [INFO] train episode 23: reward = -500.00, steps = 500
    03:50:45 [INFO] train episode 24: reward = -390.00, steps = 391
    03:51:31 [INFO] train episode 25: reward = -500.00, steps = 500
    03:52:15 [INFO] train episode 26: reward = -487.00, steps = 488
    03:53:02 [INFO] train episode 27: reward = -500.00, steps = 500
    03:53:48 [INFO] train episode 28: reward = -500.00, steps = 500
    03:54:33 [INFO] train episode 29: reward = -500.00, steps = 500
    03:55:18 [INFO] train episode 30: reward = -500.00, steps = 500
    03:56:04 [INFO] train episode 31: reward = -500.00, steps = 500
    03:56:40 [INFO] train episode 32: reward = -398.00, steps = 399
    03:57:16 [INFO] train episode 33: reward = -392.00, steps = 393
    03:58:01 [INFO] train episode 34: reward = -500.00, steps = 500
    03:58:46 [INFO] train episode 35: reward = -500.00, steps = 500
    03:59:32 [INFO] train episode 36: reward = -492.00, steps = 493
    04:00:17 [INFO] train episode 37: reward = -500.00, steps = 500
    04:01:03 [INFO] train episode 38: reward = -500.00, steps = 500
    04:01:49 [INFO] train episode 39: reward = -500.00, steps = 500
    04:02:33 [INFO] train episode 40: reward = -500.00, steps = 500
    04:03:01 [INFO] train episode 41: reward = -302.00, steps = 303
    04:03:26 [INFO] train episode 42: reward = -266.00, steps = 267
    04:03:59 [INFO] train episode 43: reward = -357.00, steps = 358
    04:04:37 [INFO] train episode 44: reward = -416.00, steps = 417
    04:05:17 [INFO] train episode 45: reward = -423.00, steps = 424
    04:06:03 [INFO] train episode 46: reward = -500.00, steps = 500
    04:06:48 [INFO] train episode 47: reward = -500.00, steps = 500
    04:07:35 [INFO] train episode 48: reward = -500.00, steps = 500
    04:08:20 [INFO] train episode 49: reward = -500.00, steps = 500
    04:09:07 [INFO] train episode 50: reward = -500.00, steps = 500
    04:09:53 [INFO] train episode 51: reward = -500.00, steps = 500
    04:10:40 [INFO] train episode 52: reward = -500.00, steps = 500
    04:11:26 [INFO] train episode 53: reward = -500.00, steps = 500
    04:12:13 [INFO] train episode 54: reward = -500.00, steps = 500
    04:13:00 [INFO] train episode 55: reward = -500.00, steps = 500
    04:13:47 [INFO] train episode 56: reward = -500.00, steps = 500
    04:14:33 [INFO] train episode 57: reward = -500.00, steps = 500
    04:15:20 [INFO] train episode 58: reward = -500.00, steps = 500
    04:16:05 [INFO] train episode 59: reward = -500.00, steps = 500
    04:16:51 [INFO] train episode 60: reward = -500.00, steps = 500
    04:17:37 [INFO] train episode 61: reward = -500.00, steps = 500
    04:18:24 [INFO] train episode 62: reward = -500.00, steps = 500
    04:19:10 [INFO] train episode 63: reward = -500.00, steps = 500
    04:19:55 [INFO] train episode 64: reward = -500.00, steps = 500
    04:20:41 [INFO] train episode 65: reward = -500.00, steps = 500
    04:21:27 [INFO] train episode 66: reward = -500.00, steps = 500
    04:22:13 [INFO] train episode 67: reward = -500.00, steps = 500
    04:23:00 [INFO] train episode 68: reward = -500.00, steps = 500
    04:23:45 [INFO] train episode 69: reward = -500.00, steps = 500
    04:24:31 [INFO] train episode 70: reward = -500.00, steps = 500
    04:25:15 [INFO] train episode 71: reward = -479.00, steps = 480
    04:26:00 [INFO] train episode 72: reward = -500.00, steps = 500
    04:26:46 [INFO] train episode 73: reward = -500.00, steps = 500
    04:27:32 [INFO] train episode 74: reward = -500.00, steps = 500
    04:28:18 [INFO] train episode 75: reward = -500.00, steps = 500
    04:29:04 [INFO] train episode 76: reward = -500.00, steps = 500
    04:29:51 [INFO] train episode 77: reward = -500.00, steps = 500
    04:30:37 [INFO] train episode 78: reward = -500.00, steps = 500
    04:31:22 [INFO] train episode 79: reward = -500.00, steps = 500
    04:32:08 [INFO] train episode 80: reward = -500.00, steps = 500
    04:32:55 [INFO] train episode 81: reward = -500.00, steps = 500
    04:33:41 [INFO] train episode 82: reward = -500.00, steps = 500
    04:34:27 [INFO] train episode 83: reward = -500.00, steps = 500
    04:35:14 [INFO] train episode 84: reward = -500.00, steps = 500
    04:36:00 [INFO] train episode 85: reward = -500.00, steps = 500
    04:36:47 [INFO] train episode 86: reward = -500.00, steps = 500
    04:37:33 [INFO] train episode 87: reward = -500.00, steps = 500
    04:38:24 [INFO] train episode 88: reward = -500.00, steps = 500
    04:39:09 [INFO] train episode 89: reward = -474.00, steps = 475
    04:39:55 [INFO] train episode 90: reward = -500.00, steps = 500
    04:40:39 [INFO] train episode 91: reward = -480.00, steps = 481
    04:41:27 [INFO] train episode 92: reward = -500.00, steps = 500
    04:42:13 [INFO] train episode 93: reward = -500.00, steps = 500
    04:43:00 [INFO] train episode 94: reward = -500.00, steps = 500
    04:43:46 [INFO] train episode 95: reward = -500.00, steps = 500
    04:44:33 [INFO] train episode 96: reward = -500.00, steps = 500
    04:45:19 [INFO] train episode 97: reward = -500.00, steps = 500
    04:45:55 [INFO] train episode 98: reward = -382.00, steps = 383
    04:46:41 [INFO] train episode 99: reward = -500.00, steps = 500
    04:47:27 [INFO] train episode 100: reward = -500.00, steps = 500
    04:48:14 [INFO] train episode 101: reward = -500.00, steps = 500
    04:48:42 [INFO] train episode 102: reward = -298.00, steps = 299
    04:49:25 [INFO] train episode 103: reward = -467.00, steps = 468
    04:50:13 [INFO] train episode 104: reward = -500.00, steps = 500
    04:51:03 [INFO] train episode 105: reward = -500.00, steps = 500
    04:51:50 [INFO] train episode 106: reward = -500.00, steps = 500
    04:52:36 [INFO] train episode 107: reward = -500.00, steps = 500
    04:53:23 [INFO] train episode 108: reward = -500.00, steps = 500
    04:54:09 [INFO] train episode 109: reward = -500.00, steps = 500
    04:54:47 [INFO] train episode 110: reward = -393.00, steps = 394
    04:55:33 [INFO] train episode 111: reward = -500.00, steps = 500
    04:56:17 [INFO] train episode 112: reward = -500.00, steps = 500
    04:57:03 [INFO] train episode 113: reward = -500.00, steps = 500
    04:57:49 [INFO] train episode 114: reward = -500.00, steps = 500
    04:58:36 [INFO] train episode 115: reward = -500.00, steps = 500
    04:59:22 [INFO] train episode 116: reward = -500.00, steps = 500
    05:00:07 [INFO] train episode 117: reward = -500.00, steps = 500
    05:00:53 [INFO] train episode 118: reward = -500.00, steps = 500
    05:01:38 [INFO] train episode 119: reward = -500.00, steps = 500
    05:02:17 [INFO] train episode 120: reward = -417.00, steps = 418
    05:03:02 [INFO] train episode 121: reward = -500.00, steps = 500
    05:03:47 [INFO] train episode 122: reward = -500.00, steps = 500
    05:04:31 [INFO] train episode 123: reward = -493.00, steps = 494
    05:05:14 [INFO] train episode 124: reward = -480.00, steps = 481
    05:05:38 [INFO] train episode 125: reward = -254.00, steps = 255
    05:06:11 [INFO] train episode 126: reward = -369.00, steps = 370
    05:06:44 [INFO] train episode 127: reward = -371.00, steps = 372
    05:07:30 [INFO] train episode 128: reward = -496.00, steps = 497
    05:07:58 [INFO] train episode 129: reward = -316.00, steps = 317
    05:08:28 [INFO] train episode 130: reward = -324.00, steps = 325
    05:09:03 [INFO] train episode 131: reward = -388.00, steps = 389
    05:09:41 [INFO] train episode 132: reward = -418.00, steps = 419
    05:10:14 [INFO] train episode 133: reward = -359.00, steps = 360
    05:10:59 [INFO] train episode 134: reward = -500.00, steps = 500
    05:11:47 [INFO] train episode 135: reward = -500.00, steps = 500
    05:12:21 [INFO] train episode 136: reward = -365.00, steps = 366
    05:12:43 [INFO] train episode 137: reward = -238.00, steps = 239
    05:13:13 [INFO] train episode 138: reward = -332.00, steps = 333
    05:13:58 [INFO] train episode 139: reward = -500.00, steps = 500
    05:14:35 [INFO] train episode 140: reward = -401.00, steps = 402
    05:15:02 [INFO] train episode 141: reward = -305.00, steps = 306
    05:15:22 [INFO] train episode 142: reward = -225.00, steps = 226
    05:15:41 [INFO] train episode 143: reward = -227.00, steps = 228
    05:16:08 [INFO] train episode 144: reward = -300.00, steps = 301
    05:16:35 [INFO] train episode 145: reward = -308.00, steps = 309
    05:17:07 [INFO] train episode 146: reward = -365.00, steps = 366
    05:17:31 [INFO] train episode 147: reward = -267.00, steps = 268
    05:18:00 [INFO] train episode 148: reward = -319.00, steps = 320
    05:18:27 [INFO] train episode 149: reward = -312.00, steps = 313
    05:18:40 [INFO] train episode 150: reward = -150.00, steps = 151
    05:19:22 [INFO] train episode 151: reward = -472.00, steps = 473
    05:19:43 [INFO] train episode 152: reward = -227.00, steps = 228
    05:20:09 [INFO] train episode 153: reward = -298.00, steps = 299
    05:20:27 [INFO] train episode 154: reward = -211.00, steps = 212
    05:20:49 [INFO] train episode 155: reward = -244.00, steps = 245
    05:21:09 [INFO] train episode 156: reward = -238.00, steps = 239
    05:21:32 [INFO] train episode 157: reward = -255.00, steps = 256
    05:21:51 [INFO] train episode 158: reward = -220.00, steps = 221
    05:22:21 [INFO] train episode 159: reward = -343.00, steps = 344
    05:22:35 [INFO] train episode 160: reward = -161.00, steps = 162
    05:22:54 [INFO] train episode 161: reward = -210.00, steps = 211
    05:23:20 [INFO] train episode 162: reward = -296.00, steps = 297
    05:23:38 [INFO] train episode 163: reward = -196.00, steps = 197
    05:23:58 [INFO] train episode 164: reward = -235.00, steps = 236
    05:24:24 [INFO] train episode 165: reward = -292.00, steps = 293
    05:24:41 [INFO] train episode 166: reward = -191.00, steps = 192
    05:25:18 [INFO] train episode 167: reward = -415.00, steps = 416
    05:25:36 [INFO] train episode 168: reward = -209.00, steps = 210
    05:26:05 [INFO] train episode 169: reward = -330.00, steps = 331
    05:26:21 [INFO] train episode 170: reward = -173.00, steps = 174
    05:26:37 [INFO] train episode 171: reward = -189.00, steps = 190
    05:26:50 [INFO] train episode 172: reward = -142.00, steps = 143
    05:27:10 [INFO] train episode 173: reward = -234.00, steps = 235
    05:27:28 [INFO] train episode 174: reward = -197.00, steps = 198
    05:27:43 [INFO] train episode 175: reward = -180.00, steps = 181
    05:28:09 [INFO] train episode 176: reward = -284.00, steps = 285
    05:28:36 [INFO] train episode 177: reward = -314.00, steps = 315
    05:28:53 [INFO] train episode 178: reward = -190.00, steps = 191
    05:29:11 [INFO] train episode 179: reward = -209.00, steps = 210
    05:29:29 [INFO] train episode 180: reward = -202.00, steps = 203
    05:29:41 [INFO] train episode 181: reward = -136.00, steps = 137
    05:29:54 [INFO] train episode 182: reward = -142.00, steps = 143
    05:30:16 [INFO] train episode 183: reward = -261.00, steps = 262
    05:30:34 [INFO] train episode 184: reward = -195.00, steps = 196
    05:30:49 [INFO] train episode 185: reward = -170.00, steps = 171
    05:31:04 [INFO] train episode 186: reward = -161.00, steps = 162
    05:31:21 [INFO] train episode 187: reward = -200.00, steps = 201
    05:31:33 [INFO] train episode 188: reward = -130.00, steps = 131
    05:31:45 [INFO] train episode 189: reward = -133.00, steps = 134
    05:32:09 [INFO] train episode 190: reward = -275.00, steps = 276
    05:32:25 [INFO] train episode 191: reward = -183.00, steps = 184
    05:32:44 [INFO] train episode 192: reward = -204.00, steps = 205
    05:33:02 [INFO] train episode 193: reward = -204.00, steps = 205
    05:33:24 [INFO] train episode 194: reward = -255.00, steps = 256
    05:33:38 [INFO] train episode 195: reward = -154.00, steps = 155
    05:33:55 [INFO] train episode 196: reward = -189.00, steps = 190
    05:34:09 [INFO] train episode 197: reward = -156.00, steps = 157
    05:34:21 [INFO] train episode 198: reward = -135.00, steps = 136
    05:34:32 [INFO] train episode 199: reward = -118.00, steps = 119
    05:34:50 [INFO] train episode 200: reward = -196.00, steps = 197
    05:35:06 [INFO] train episode 201: reward = -175.00, steps = 176
    05:35:23 [INFO] train episode 202: reward = -202.00, steps = 203
    05:35:36 [INFO] train episode 203: reward = -144.00, steps = 145
    05:35:52 [INFO] train episode 204: reward = -172.00, steps = 173
    05:36:05 [INFO] train episode 205: reward = -135.00, steps = 136
    05:36:17 [INFO] train episode 206: reward = -145.00, steps = 146
    05:36:30 [INFO] train episode 207: reward = -147.00, steps = 148
    05:36:42 [INFO] train episode 208: reward = -122.00, steps = 123
    05:36:58 [INFO] train episode 209: reward = -176.00, steps = 177
    05:37:10 [INFO] train episode 210: reward = -136.00, steps = 137
    05:37:23 [INFO] train episode 211: reward = -134.00, steps = 135
    05:37:36 [INFO] train episode 212: reward = -140.00, steps = 141
    05:37:50 [INFO] train episode 213: reward = -144.00, steps = 145
    05:38:11 [INFO] train episode 214: reward = -238.00, steps = 239
    05:38:31 [INFO] train episode 215: reward = -218.00, steps = 219
    05:38:50 [INFO] train episode 216: reward = -211.00, steps = 212
    05:39:05 [INFO] train episode 217: reward = -155.00, steps = 156
    05:39:14 [INFO] train episode 218: reward = -112.00, steps = 113
    05:39:27 [INFO] train episode 219: reward = -144.00, steps = 145
    05:39:59 [INFO] train episode 220: reward = -364.00, steps = 365
    05:40:13 [INFO] train episode 221: reward = -165.00, steps = 166
    05:40:27 [INFO] train episode 222: reward = -153.00, steps = 154
    05:40:38 [INFO] train episode 223: reward = -124.00, steps = 125
    05:40:53 [INFO] train episode 224: reward = -169.00, steps = 170
    05:41:03 [INFO] train episode 225: reward = -110.00, steps = 111
    05:41:19 [INFO] train episode 226: reward = -179.00, steps = 180
    05:41:34 [INFO] train episode 227: reward = -164.00, steps = 165
    05:41:43 [INFO] train episode 228: reward = -103.00, steps = 104
    05:41:53 [INFO] train episode 229: reward = -109.00, steps = 110
    05:42:07 [INFO] train episode 230: reward = -160.00, steps = 161
    05:42:25 [INFO] train episode 231: reward = -198.00, steps = 199
    05:42:36 [INFO] train episode 232: reward = -114.00, steps = 115
    05:42:53 [INFO] train episode 233: reward = -189.00, steps = 190
    05:43:05 [INFO] train episode 234: reward = -131.00, steps = 132
    05:43:18 [INFO] train episode 235: reward = -154.00, steps = 155
    05:43:33 [INFO] train episode 236: reward = -161.00, steps = 162
    05:43:46 [INFO] train episode 237: reward = -139.00, steps = 140
    05:43:59 [INFO] train episode 238: reward = -138.00, steps = 139
    05:44:13 [INFO] train episode 239: reward = -164.00, steps = 165
    05:44:26 [INFO] train episode 240: reward = -141.00, steps = 142
    05:44:42 [INFO] train episode 241: reward = -182.00, steps = 183
    05:44:56 [INFO] train episode 242: reward = -161.00, steps = 162
    05:45:07 [INFO] train episode 243: reward = -120.00, steps = 121
    05:45:17 [INFO] train episode 244: reward = -113.00, steps = 114
    05:45:29 [INFO] train episode 245: reward = -129.00, steps = 130
    05:45:41 [INFO] train episode 246: reward = -142.00, steps = 143
    05:45:53 [INFO] train episode 247: reward = -135.00, steps = 136
    05:46:04 [INFO] train episode 248: reward = -120.00, steps = 121
    05:46:16 [INFO] train episode 249: reward = -138.00, steps = 139
    05:46:31 [INFO] train episode 250: reward = -171.00, steps = 172
    05:46:46 [INFO] train episode 251: reward = -162.00, steps = 163
    05:47:03 [INFO] train episode 252: reward = -188.00, steps = 189
    05:47:14 [INFO] train episode 253: reward = -124.00, steps = 125
    05:47:33 [INFO] train episode 254: reward = -210.00, steps = 211
    05:48:17 [INFO] train episode 255: reward = -500.00, steps = 500
    05:48:32 [INFO] train episode 256: reward = -166.00, steps = 167
    05:48:42 [INFO] train episode 257: reward = -111.00, steps = 112
    05:48:55 [INFO] train episode 258: reward = -145.00, steps = 146
    05:49:07 [INFO] train episode 259: reward = -141.00, steps = 142
    05:49:19 [INFO] train episode 260: reward = -133.00, steps = 134
    05:49:36 [INFO] train episode 261: reward = -187.00, steps = 188
    05:49:48 [INFO] train episode 262: reward = -140.00, steps = 141
    05:49:59 [INFO] train episode 263: reward = -118.00, steps = 119
    05:50:12 [INFO] train episode 264: reward = -141.00, steps = 142
    05:50:23 [INFO] train episode 265: reward = -119.00, steps = 120
    05:50:39 [INFO] train episode 266: reward = -184.00, steps = 185
    05:50:50 [INFO] train episode 267: reward = -122.00, steps = 123
    05:51:18 [INFO] train episode 268: reward = -324.00, steps = 325
    05:51:31 [INFO] train episode 269: reward = -146.00, steps = 147
    05:51:43 [INFO] train episode 270: reward = -133.00, steps = 134
    05:51:55 [INFO] train episode 271: reward = -139.00, steps = 140
    05:52:09 [INFO] train episode 272: reward = -155.00, steps = 156
    05:52:21 [INFO] train episode 273: reward = -130.00, steps = 131
    05:52:33 [INFO] train episode 274: reward = -136.00, steps = 137
    05:52:47 [INFO] train episode 275: reward = -158.00, steps = 159
    05:52:57 [INFO] train episode 276: reward = -107.00, steps = 108
    05:53:08 [INFO] train episode 277: reward = -134.00, steps = 135
    05:53:21 [INFO] train episode 278: reward = -139.00, steps = 140
    05:53:37 [INFO] train episode 279: reward = -187.00, steps = 188
    05:53:50 [INFO] train episode 280: reward = -143.00, steps = 144
    05:54:02 [INFO] train episode 281: reward = -141.00, steps = 142
    05:54:13 [INFO] train episode 282: reward = -120.00, steps = 121
    05:54:25 [INFO] train episode 283: reward = -132.00, steps = 133
    05:54:38 [INFO] train episode 284: reward = -135.00, steps = 136
    05:54:52 [INFO] train episode 285: reward = -160.00, steps = 161
    05:55:05 [INFO] train episode 286: reward = -150.00, steps = 151
    05:55:18 [INFO] train episode 287: reward = -148.00, steps = 149
    05:55:32 [INFO] train episode 288: reward = -151.00, steps = 152
    05:55:48 [INFO] train episode 289: reward = -180.00, steps = 181
    05:56:01 [INFO] train episode 290: reward = -139.00, steps = 140
    05:56:15 [INFO] train episode 291: reward = -147.00, steps = 148
    05:56:24 [INFO] train episode 292: reward = -111.00, steps = 112
    05:56:34 [INFO] train episode 293: reward = -109.00, steps = 110
    05:56:48 [INFO] train episode 294: reward = -151.00, steps = 152
    05:57:01 [INFO] train episode 295: reward = -154.00, steps = 155
    05:57:22 [INFO] train episode 296: reward = -228.00, steps = 229
    05:57:37 [INFO] train episode 297: reward = -165.00, steps = 166
    05:57:53 [INFO] train episode 298: reward = -179.00, steps = 180
    05:58:08 [INFO] train episode 299: reward = -174.00, steps = 175
    05:58:17 [INFO] train episode 300: reward = -96.00, steps = 97
    05:58:36 [INFO] train episode 301: reward = -211.00, steps = 212
    05:58:49 [INFO] train episode 302: reward = -154.00, steps = 155
    05:59:02 [INFO] train episode 303: reward = -145.00, steps = 146
    05:59:14 [INFO] train episode 304: reward = -129.00, steps = 130
    05:59:26 [INFO] train episode 305: reward = -129.00, steps = 130
    05:59:41 [INFO] train episode 306: reward = -168.00, steps = 169
    05:59:52 [INFO] train episode 307: reward = -123.00, steps = 124
    06:00:05 [INFO] train episode 308: reward = -135.00, steps = 136
    06:00:43 [INFO] train episode 309: reward = -429.00, steps = 430
    06:00:52 [INFO] train episode 310: reward = -98.00, steps = 99
    06:01:05 [INFO] train episode 311: reward = -136.00, steps = 137
    06:01:17 [INFO] train episode 312: reward = -131.00, steps = 132
    06:01:32 [INFO] train episode 313: reward = -170.00, steps = 171
    06:01:46 [INFO] train episode 314: reward = -151.00, steps = 152
    06:02:00 [INFO] train episode 315: reward = -167.00, steps = 168
    06:02:15 [INFO] train episode 316: reward = -165.00, steps = 166
    06:02:29 [INFO] train episode 317: reward = -150.00, steps = 151
    06:02:40 [INFO] train episode 318: reward = -124.00, steps = 125
    06:02:56 [INFO] train episode 319: reward = -175.00, steps = 176
    06:03:07 [INFO] train episode 320: reward = -128.00, steps = 129
    06:03:21 [INFO] train episode 321: reward = -160.00, steps = 161
    06:03:31 [INFO] train episode 322: reward = -111.00, steps = 112
    06:03:41 [INFO] train episode 323: reward = -109.00, steps = 110
    06:03:56 [INFO] train episode 324: reward = -165.00, steps = 166
    06:04:10 [INFO] train episode 325: reward = -150.00, steps = 151
    06:04:20 [INFO] train episode 326: reward = -105.00, steps = 106
    06:04:32 [INFO] train episode 327: reward = -136.00, steps = 137
    06:04:42 [INFO] train episode 328: reward = -118.00, steps = 119
    06:04:57 [INFO] train episode 329: reward = -172.00, steps = 173
    06:05:09 [INFO] train episode 330: reward = -127.00, steps = 128
    06:05:27 [INFO] train episode 331: reward = -210.00, steps = 211
    06:05:38 [INFO] train episode 332: reward = -116.00, steps = 117
    06:05:48 [INFO] train episode 333: reward = -105.00, steps = 106
    06:06:04 [INFO] train episode 334: reward = -175.00, steps = 176
    06:06:15 [INFO] train episode 335: reward = -124.00, steps = 125
    06:06:30 [INFO] train episode 336: reward = -161.00, steps = 162
    06:06:44 [INFO] train episode 337: reward = -166.00, steps = 167
    06:06:53 [INFO] train episode 338: reward = -90.00, steps = 91
    06:07:02 [INFO] train episode 339: reward = -112.00, steps = 113
    06:07:19 [INFO] train episode 340: reward = -180.00, steps = 181
    06:07:33 [INFO] train episode 341: reward = -146.00, steps = 147
    06:07:42 [INFO] train episode 342: reward = -109.00, steps = 110
    06:07:52 [INFO] train episode 343: reward = -101.00, steps = 102
    06:08:02 [INFO] train episode 344: reward = -118.00, steps = 119
    06:08:12 [INFO] train episode 345: reward = -104.00, steps = 105
    06:08:25 [INFO] train episode 346: reward = -145.00, steps = 146
    06:08:34 [INFO] train episode 347: reward = -101.00, steps = 102
    06:08:46 [INFO] train episode 348: reward = -126.00, steps = 127
    06:08:57 [INFO] train episode 349: reward = -125.00, steps = 126
    06:09:06 [INFO] train episode 350: reward = -86.00, steps = 87
    06:09:06 [INFO] ==== test ====
    06:09:17 [INFO] test episode 0: reward = -132.00, steps = 133
    06:09:28 [INFO] test episode 1: reward = -118.00, steps = 119
    06:09:39 [INFO] test episode 2: reward = -139.00, steps = 140
    06:09:47 [INFO] test episode 3: reward = -87.00, steps = 88
    06:09:57 [INFO] test episode 4: reward = -118.00, steps = 119
    06:10:07 [INFO] test episode 5: reward = -117.00, steps = 118
    06:10:17 [INFO] test episode 6: reward = -111.00, steps = 112
    06:10:27 [INFO] test episode 7: reward = -116.00, steps = 117
    06:10:36 [INFO] test episode 8: reward = -106.00, steps = 107
    06:10:45 [INFO] test episode 9: reward = -106.00, steps = 107
    06:10:55 [INFO] test episode 10: reward = -114.00, steps = 115
    06:11:05 [INFO] test episode 11: reward = -122.00, steps = 123
    06:11:17 [INFO] test episode 12: reward = -137.00, steps = 138
    06:11:26 [INFO] test episode 13: reward = -105.00, steps = 106
    06:11:37 [INFO] test episode 14: reward = -133.00, steps = 134
    06:11:48 [INFO] test episode 15: reward = -125.00, steps = 126
    06:12:02 [INFO] test episode 16: reward = -158.00, steps = 159
    06:12:12 [INFO] test episode 17: reward = -115.00, steps = 116
    06:12:21 [INFO] test episode 18: reward = -118.00, steps = 119
    06:12:32 [INFO] test episode 19: reward = -127.00, steps = 128
    06:12:46 [INFO] test episode 20: reward = -165.00, steps = 166
    06:12:56 [INFO] test episode 21: reward = -118.00, steps = 119
    06:13:09 [INFO] test episode 22: reward = -142.00, steps = 143
    06:13:17 [INFO] test episode 23: reward = -103.00, steps = 104
    06:13:29 [INFO] test episode 24: reward = -134.00, steps = 135
    06:13:50 [INFO] test episode 25: reward = -248.00, steps = 249
    06:13:59 [INFO] test episode 26: reward = -111.00, steps = 112
    06:14:11 [INFO] test episode 27: reward = -133.00, steps = 134
    06:14:25 [INFO] test episode 28: reward = -173.00, steps = 174
    06:14:37 [INFO] test episode 29: reward = -134.00, steps = 135
    06:14:48 [INFO] test episode 30: reward = -132.00, steps = 133
    06:14:58 [INFO] test episode 31: reward = -117.00, steps = 118
    06:15:08 [INFO] test episode 32: reward = -115.00, steps = 116
    06:15:19 [INFO] test episode 33: reward = -131.00, steps = 132
    06:15:28 [INFO] test episode 34: reward = -98.00, steps = 99
    06:15:40 [INFO] test episode 35: reward = -141.00, steps = 142
    06:15:50 [INFO] test episode 36: reward = -119.00, steps = 120
    06:15:59 [INFO] test episode 37: reward = -107.00, steps = 108
    06:16:12 [INFO] test episode 38: reward = -149.00, steps = 150
    06:16:23 [INFO] test episode 39: reward = -120.00, steps = 121
    06:16:45 [INFO] test episode 40: reward = -263.00, steps = 264
    06:16:59 [INFO] test episode 41: reward = -171.00, steps = 172
    06:17:10 [INFO] test episode 42: reward = -122.00, steps = 123
    06:17:19 [INFO] test episode 43: reward = -111.00, steps = 112
    06:17:30 [INFO] test episode 44: reward = -120.00, steps = 121
    06:17:41 [INFO] test episode 45: reward = -126.00, steps = 127
    06:17:58 [INFO] test episode 46: reward = -202.00, steps = 203
    06:18:08 [INFO] test episode 47: reward = -124.00, steps = 125
    06:18:16 [INFO] test episode 48: reward = -87.00, steps = 88
    06:18:24 [INFO] test episode 49: reward = -91.00, steps = 92
    06:18:36 [INFO] test episode 50: reward = -146.00, steps = 147
    06:18:50 [INFO] test episode 51: reward = -169.00, steps = 170
    06:19:01 [INFO] test episode 52: reward = -129.00, steps = 130
    06:19:13 [INFO] test episode 53: reward = -139.00, steps = 140
    06:19:23 [INFO] test episode 54: reward = -115.00, steps = 116
    06:19:37 [INFO] test episode 55: reward = -158.00, steps = 159
    06:19:46 [INFO] test episode 56: reward = -109.00, steps = 110
    06:19:55 [INFO] test episode 57: reward = -107.00, steps = 108
    06:20:07 [INFO] test episode 58: reward = -139.00, steps = 140
    06:20:17 [INFO] test episode 59: reward = -118.00, steps = 119
    06:20:30 [INFO] test episode 60: reward = -150.00, steps = 151
    06:20:46 [INFO] test episode 61: reward = -185.00, steps = 186
    06:21:00 [INFO] test episode 62: reward = -165.00, steps = 166
    06:21:09 [INFO] test episode 63: reward = -101.00, steps = 102
    06:21:20 [INFO] test episode 64: reward = -123.00, steps = 124
    06:21:30 [INFO] test episode 65: reward = -126.00, steps = 127
    06:21:39 [INFO] test episode 66: reward = -102.00, steps = 103
    06:21:48 [INFO] test episode 67: reward = -109.00, steps = 110
    06:21:58 [INFO] test episode 68: reward = -107.00, steps = 108
    06:22:07 [INFO] test episode 69: reward = -109.00, steps = 110
    06:22:15 [INFO] test episode 70: reward = -86.00, steps = 87
    06:22:28 [INFO] test episode 71: reward = -154.00, steps = 155
    06:22:37 [INFO] test episode 72: reward = -106.00, steps = 107
    06:22:47 [INFO] test episode 73: reward = -114.00, steps = 115
    06:22:59 [INFO] test episode 74: reward = -138.00, steps = 139
    06:23:11 [INFO] test episode 75: reward = -145.00, steps = 146
    06:23:23 [INFO] test episode 76: reward = -141.00, steps = 142
    06:23:34 [INFO] test episode 77: reward = -122.00, steps = 123
    06:23:43 [INFO] test episode 78: reward = -113.00, steps = 114
    06:24:00 [INFO] test episode 79: reward = -196.00, steps = 197
    06:24:09 [INFO] test episode 80: reward = -100.00, steps = 101
    06:24:21 [INFO] test episode 81: reward = -145.00, steps = 146
    06:24:36 [INFO] test episode 82: reward = -175.00, steps = 176
    06:24:44 [INFO] test episode 83: reward = -99.00, steps = 100
    06:24:58 [INFO] test episode 84: reward = -160.00, steps = 161
    06:25:09 [INFO] test episode 85: reward = -124.00, steps = 125
    06:25:20 [INFO] test episode 86: reward = -132.00, steps = 133
    06:25:32 [INFO] test episode 87: reward = -147.00, steps = 148
    06:25:44 [INFO] test episode 88: reward = -132.00, steps = 133
    06:25:52 [INFO] test episode 89: reward = -93.00, steps = 94
    06:26:03 [INFO] test episode 90: reward = -137.00, steps = 138
    06:26:17 [INFO] test episode 91: reward = -160.00, steps = 161
    06:26:26 [INFO] test episode 92: reward = -105.00, steps = 106
    06:26:37 [INFO] test episode 93: reward = -137.00, steps = 138
    06:26:48 [INFO] test episode 94: reward = -136.00, steps = 137
    06:26:58 [INFO] test episode 95: reward = -114.00, steps = 115
    06:27:11 [INFO] test episode 96: reward = -148.00, steps = 149
    06:27:26 [INFO] test episode 97: reward = -178.00, steps = 179
    06:27:35 [INFO] test episode 98: reward = -112.00, steps = 113
    06:27:47 [INFO] test episode 99: reward = -135.00, steps = 136
    06:27:47 [INFO] average episode reward = -131.01 ± 29.60

In [6]:

    env.close()