# Use DQN to Play MoutainCar-v0¶

TensorFlow version

In [1]:

    %matplotlib inline

    import sys
    import logging
    import itertools

    import numpy as np
    np.random.seed(0)
    import pandas as pd
    import gym
    import matplotlib.pyplot as plt
    import tensorflow.compat.v2 as tf
    tf.random.set_seed(0)
    from tensorflow import nn
    from tensorflow import losses
    from tensorflow import optimizers
    from tensorflow import keras
    from tensorflow.keras import layers
    from tensorflow.keras import models

    logging.basicConfig(level=logging.INFO,
            format='%(asctime)s [%(levelname)s] %(message)s',
            stream=sys.stdout, datefmt='%H:%M:%S')

In [2]:

    env = gym.make('MountainCar-v0')
    for key in vars(env):
        logging.info('%s: %s', key, vars(env)[key])
    for key in vars(env.spec):
        logging.info('%s: %s', key, vars(env.spec)[key])

    22:21:46 [INFO] env: <MountainCarEnv<MountainCar-v0>>
    22:21:46 [INFO] action_space: Discrete(3)
    22:21:46 [INFO] observation_space: Box(-1.2000000476837158, 0.6000000238418579, (2,), float32)
    22:21:46 [INFO] reward_range: (-inf, inf)
    22:21:46 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 30}
    22:21:46 [INFO] _max_episode_steps: 200
    22:21:46 [INFO] _elapsed_steps: None
    22:21:46 [INFO] id: MountainCar-v0
    22:21:46 [INFO] entry_point: gym.envs.classic_control:MountainCarEnv
    22:21:46 [INFO] reward_threshold: -110.0
    22:21:46 [INFO] nondeterministic: False
    22:21:46 [INFO] max_episode_steps: 200
    22:21:46 [INFO] _kwargs: {}
    22:21:46 [INFO] _env_name: MountainCar

In [3]:

    class DQNReplayer:
        def __init__(self, capacity):
            self.memory = pd.DataFrame(index=range(capacity),
                    columns=['state', 'action', 'reward', 'next_state', 'terminated'])
            self.i = 0
            self.count = 0
            self.capacity = capacity

        def store(self, *args):
            self.memory.loc[self.i] = np.asarray(args, dtype=object)
            self.i = (self.i + 1) % self.capacity
            self.count = min(self.count + 1, self.capacity)

        def sample(self, size):
            indices = np.random.choice(self.count, size=size)
            return (np.stack(self.memory.loc[indices, field]) for field in
                    self.memory.columns)

In [4]:

    class DQNAgent:
        def __init__(self, env):
            self.action_n = env.action_space.n
            self.gamma = 0.99

            self.replayer = DQNReplayer(10000)

            self.evaluate_net = self.build_net(
                    input_size=env.observation_space.shape[0],
                    hidden_sizes=[64, 64], output_size=self.action_n)
            self.target_net = models.clone_model(self.evaluate_net)

        def build_net(self, input_size, hidden_sizes, output_size):
            model = keras.Sequential()
            for layer, hidden_size in enumerate(hidden_sizes):
                kwargs = dict(input_shape=(input_size,)) if not layer else {}
                model.add(layers.Dense(units=hidden_size,
                        activation=nn.relu, **kwargs))
            model.add(layers.Dense(units=output_size))
            optimizer = optimizers.Adam(0.001)
            model.compile(loss=losses.mse, optimizer=optimizer)
            return model

        def reset(self, mode=None):
            self.mode = mode
            if self.mode == 'train':
                self.trajectory = []
                self.target_net.set_weights(self.evaluate_net.get_weights())

        def step(self, observation, reward, terminated):
            if self.mode == 'train' and np.random.rand() < 0.001:
                # epsilon-greedy policy in train mode
                action = np.random.randint(self.action_n)
            else:
                qs = self.evaluate_net.predict(observation[np.newaxis], verbose=0)
                action = np.argmax(qs)
            if self.mode == 'train':
                self.trajectory += [observation, reward, terminated, action]
                if len(self.trajectory) >= 8:
                    state, _, _, act, next_state, reward, terminated, _ = \
                            self.trajectory[-8:]
                    self.replayer.store(state, act, reward, next_state, terminated)
                if self.replayer.count >= self.replayer.capacity * 0.95:
                        # skip first few episodes for speed
                    self.learn()
            return action

        def close(self):
            pass

        def learn(self):
            # replay
            states, actions, rewards, next_states, terminateds = \
                    self.replayer.sample(1024)

            # update value net
            next_qs = self.target_net.predict(next_states, verbose=0)
            next_max_qs = next_qs.max(axis=-1)
            us = rewards + self.gamma * (1. - terminateds) * next_max_qs
            targets = self.evaluate_net.predict(states, verbose=0)
            targets[np.arange(us.shape[0]), actions] = us
            self.evaluate_net.fit(states, targets, verbose=0)

    agent = DQNAgent(env)

In [5]:

    def play_episode(env, agent, seed=None, mode=None, render=False):
        observation, _ = env.reset(seed=seed)
        reward, terminated, truncated = 0., False, False
        agent.reset(mode=mode)
        episode_reward, elapsed_steps = 0., 0
        while True:
            action = agent.step(observation, reward, terminated)
            if render:
                env.render()
            if terminated or truncated:
                break
            observation, reward, terminated, truncated, _ = env.step(action)
            episode_reward += reward
            elapsed_steps += 1
        agent.close()
        return episode_reward, elapsed_steps

    logging.info('==== train ====')
    episode_rewards = []
    for episode in itertools.count():
        episode_reward, elapsed_steps = play_episode(env, agent, seed=episode,
                mode='train')
        episode_rewards.append(episode_reward)
        logging.info('train episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
        if np.mean(episode_rewards[-10:]) > -110:
            break
    plt.plot(episode_rewards)

    logging.info('==== test ====')
    episode_rewards = []
    for episode in range(100):
        episode_reward, elapsed_steps = play_episode(env, agent)
        episode_rewards.append(episode_reward)
        logging.info('test episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
    logging.info('average episode reward = %.2f ± %.2f',
            np.mean(episode_rewards), np.std(episode_rewards))

    22:21:47 [INFO] ==== train ====
    22:21:56 [INFO] train episode 0: reward = -200.00, steps = 200
    22:22:08 [INFO] train episode 1: reward = -200.00, steps = 200
    22:22:19 [INFO] train episode 2: reward = -200.00, steps = 200
    22:22:32 [INFO] train episode 3: reward = -200.00, steps = 200
    22:22:44 [INFO] train episode 4: reward = -200.00, steps = 200
    22:22:59 [INFO] train episode 5: reward = -200.00, steps = 200
    22:23:13 [INFO] train episode 6: reward = -200.00, steps = 200
    22:23:27 [INFO] train episode 7: reward = -200.00, steps = 200
    22:23:46 [INFO] train episode 8: reward = -200.00, steps = 200
    22:24:08 [INFO] train episode 9: reward = -200.00, steps = 200
    22:24:25 [INFO] train episode 10: reward = -200.00, steps = 200
    22:24:43 [INFO] train episode 11: reward = -200.00, steps = 200
    22:25:09 [INFO] train episode 12: reward = -200.00, steps = 200
    22:25:27 [INFO] train episode 13: reward = -200.00, steps = 200
    22:25:40 [INFO] train episode 14: reward = -200.00, steps = 200
    22:25:54 [INFO] train episode 15: reward = -200.00, steps = 200
    22:26:08 [INFO] train episode 16: reward = -200.00, steps = 200
    22:26:21 [INFO] train episode 17: reward = -200.00, steps = 200
    22:26:35 [INFO] train episode 18: reward = -200.00, steps = 200
    22:26:49 [INFO] train episode 19: reward = -200.00, steps = 200
    22:27:03 [INFO] train episode 20: reward = -200.00, steps = 200
    22:27:17 [INFO] train episode 21: reward = -200.00, steps = 200
    22:27:32 [INFO] train episode 22: reward = -200.00, steps = 200
    22:27:46 [INFO] train episode 23: reward = -200.00, steps = 200
    22:28:01 [INFO] train episode 24: reward = -200.00, steps = 200
    22:28:15 [INFO] train episode 25: reward = -200.00, steps = 200
    22:28:30 [INFO] train episode 26: reward = -200.00, steps = 200
    22:28:45 [INFO] train episode 27: reward = -200.00, steps = 200
    22:28:59 [INFO] train episode 28: reward = -200.00, steps = 200
    22:29:13 [INFO] train episode 29: reward = -200.00, steps = 200
    22:29:28 [INFO] train episode 30: reward = -200.00, steps = 200
    22:29:41 [INFO] train episode 31: reward = -200.00, steps = 200
    22:29:54 [INFO] train episode 32: reward = -200.00, steps = 200
    22:30:07 [INFO] train episode 33: reward = -200.00, steps = 200
    22:30:20 [INFO] train episode 34: reward = -200.00, steps = 200
    22:30:39 [INFO] train episode 35: reward = -200.00, steps = 200
    22:30:59 [INFO] train episode 36: reward = -200.00, steps = 200
    22:31:13 [INFO] train episode 37: reward = -200.00, steps = 200
    22:31:27 [INFO] train episode 38: reward = -200.00, steps = 200
    22:31:42 [INFO] train episode 39: reward = -200.00, steps = 200
    22:31:56 [INFO] train episode 40: reward = -200.00, steps = 200
    22:32:13 [INFO] train episode 41: reward = -200.00, steps = 200
    22:32:27 [INFO] train episode 42: reward = -200.00, steps = 200
    22:32:41 [INFO] train episode 43: reward = -200.00, steps = 200
    22:32:55 [INFO] train episode 44: reward = -200.00, steps = 200
    22:33:12 [INFO] train episode 45: reward = -200.00, steps = 200
    22:33:26 [INFO] train episode 46: reward = -200.00, steps = 200
    22:33:55 [INFO] train episode 47: reward = -200.00, steps = 200
    22:35:07 [INFO] train episode 48: reward = -200.00, steps = 200
    22:36:30 [INFO] train episode 49: reward = -200.00, steps = 200
    22:38:01 [INFO] train episode 50: reward = -200.00, steps = 200
    22:39:35 [INFO] train episode 51: reward = -200.00, steps = 200
    22:41:23 [INFO] train episode 52: reward = -200.00, steps = 200
    22:43:15 [INFO] train episode 53: reward = -200.00, steps = 200
    22:45:08 [INFO] train episode 54: reward = -200.00, steps = 200
    22:46:58 [INFO] train episode 55: reward = -200.00, steps = 200
    22:48:36 [INFO] train episode 56: reward = -200.00, steps = 200
    22:50:23 [INFO] train episode 57: reward = -200.00, steps = 200
    22:52:06 [INFO] train episode 58: reward = -200.00, steps = 200
    22:53:51 [INFO] train episode 59: reward = -200.00, steps = 200
    22:55:38 [INFO] train episode 60: reward = -200.00, steps = 200
    22:57:21 [INFO] train episode 61: reward = -200.00, steps = 200
    22:59:05 [INFO] train episode 62: reward = -200.00, steps = 200
    23:00:59 [INFO] train episode 63: reward = -200.00, steps = 200
    23:02:56 [INFO] train episode 64: reward = -200.00, steps = 200
    23:04:51 [INFO] train episode 65: reward = -200.00, steps = 200
    23:06:42 [INFO] train episode 66: reward = -200.00, steps = 200
    23:08:28 [INFO] train episode 67: reward = -200.00, steps = 200
    23:10:20 [INFO] train episode 68: reward = -200.00, steps = 200
    23:12:28 [INFO] train episode 69: reward = -200.00, steps = 200
    23:14:53 [INFO] train episode 70: reward = -200.00, steps = 200
    23:17:14 [INFO] train episode 71: reward = -200.00, steps = 200
    23:19:34 [INFO] train episode 72: reward = -200.00, steps = 200
    23:21:58 [INFO] train episode 73: reward = -200.00, steps = 200
    23:24:23 [INFO] train episode 74: reward = -200.00, steps = 200
    23:26:48 [INFO] train episode 75: reward = -200.00, steps = 200
    23:29:10 [INFO] train episode 76: reward = -200.00, steps = 200
    23:31:10 [INFO] train episode 77: reward = -162.00, steps = 162
    23:33:30 [INFO] train episode 78: reward = -200.00, steps = 200
    23:35:51 [INFO] train episode 79: reward = -200.00, steps = 200
    23:38:18 [INFO] train episode 80: reward = -200.00, steps = 200
    23:40:18 [INFO] train episode 81: reward = -167.00, steps = 167
    23:42:49 [INFO] train episode 82: reward = -200.00, steps = 200
    23:45:21 [INFO] train episode 83: reward = -200.00, steps = 200
    23:48:03 [INFO] train episode 84: reward = -200.00, steps = 200
    23:50:39 [INFO] train episode 85: reward = -200.00, steps = 200
    23:52:20 [INFO] train episode 86: reward = -129.00, steps = 129
    23:54:52 [INFO] train episode 87: reward = -200.00, steps = 200
    23:57:11 [INFO] train episode 88: reward = -200.00, steps = 200
    23:58:37 [INFO] train episode 89: reward = -120.00, steps = 120
    00:00:07 [INFO] train episode 90: reward = -128.00, steps = 128
    00:01:38 [INFO] train episode 91: reward = -129.00, steps = 129
    00:03:01 [INFO] train episode 92: reward = -116.00, steps = 116
    00:04:32 [INFO] train episode 93: reward = -128.00, steps = 128
    00:06:50 [INFO] train episode 94: reward = -200.00, steps = 200
    00:09:11 [INFO] train episode 95: reward = -200.00, steps = 200
    00:11:32 [INFO] train episode 96: reward = -200.00, steps = 200
    00:13:51 [INFO] train episode 97: reward = -200.00, steps = 200
    00:16:10 [INFO] train episode 98: reward = -200.00, steps = 200
    00:17:32 [INFO] train episode 99: reward = -117.00, steps = 117
    00:18:56 [INFO] train episode 100: reward = -119.00, steps = 119
    00:21:15 [INFO] train episode 101: reward = -200.00, steps = 200
    00:23:36 [INFO] train episode 102: reward = -200.00, steps = 200
    00:25:56 [INFO] train episode 103: reward = -200.00, steps = 200
    00:28:19 [INFO] train episode 104: reward = -200.00, steps = 200
    00:30:38 [INFO] train episode 105: reward = -200.00, steps = 200
    00:33:03 [INFO] train episode 106: reward = -200.00, steps = 200
    00:34:32 [INFO] train episode 107: reward = -129.00, steps = 129
    00:36:47 [INFO] train episode 108: reward = -193.00, steps = 193
    00:39:10 [INFO] train episode 109: reward = -200.00, steps = 200
    00:41:29 [INFO] train episode 110: reward = -200.00, steps = 200
    00:43:49 [INFO] train episode 111: reward = -200.00, steps = 200
    00:46:10 [INFO] train episode 112: reward = -200.00, steps = 200
    00:48:28 [INFO] train episode 113: reward = -200.00, steps = 200
    00:50:47 [INFO] train episode 114: reward = -200.00, steps = 200
    00:53:08 [INFO] train episode 115: reward = -200.00, steps = 200
    00:55:27 [INFO] train episode 116: reward = -200.00, steps = 200
    00:57:47 [INFO] train episode 117: reward = -200.00, steps = 200
    00:59:45 [INFO] train episode 118: reward = -170.00, steps = 170
    01:02:04 [INFO] train episode 119: reward = -200.00, steps = 200
    01:04:22 [INFO] train episode 120: reward = -198.00, steps = 198
    01:06:03 [INFO] train episode 121: reward = -147.00, steps = 147
    01:08:22 [INFO] train episode 122: reward = -200.00, steps = 200
    01:10:41 [INFO] train episode 123: reward = -200.00, steps = 200
    01:12:59 [INFO] train episode 124: reward = -200.00, steps = 200
    01:15:19 [INFO] train episode 125: reward = -200.00, steps = 200
    01:17:37 [INFO] train episode 126: reward = -200.00, steps = 200
    01:19:56 [INFO] train episode 127: reward = -200.00, steps = 200
    01:22:17 [INFO] train episode 128: reward = -200.00, steps = 200
    01:24:36 [INFO] train episode 129: reward = -200.00, steps = 200
    01:26:25 [INFO] train episode 130: reward = -156.00, steps = 156
    01:28:12 [INFO] train episode 131: reward = -148.00, steps = 148
    01:29:52 [INFO] train episode 132: reward = -142.00, steps = 142
    01:32:04 [INFO] train episode 133: reward = -189.00, steps = 189
    01:33:49 [INFO] train episode 134: reward = -149.00, steps = 149
    01:35:31 [INFO] train episode 135: reward = -147.00, steps = 147
    01:37:09 [INFO] train episode 136: reward = -140.00, steps = 140
    01:38:51 [INFO] train episode 137: reward = -145.00, steps = 145
    01:40:32 [INFO] train episode 138: reward = -143.00, steps = 143
    01:42:22 [INFO] train episode 139: reward = -157.00, steps = 157
    01:43:32 [INFO] train episode 140: reward = -99.00, steps = 99
    01:44:43 [INFO] train episode 141: reward = -102.00, steps = 102
    01:46:00 [INFO] train episode 142: reward = -111.00, steps = 111
    01:48:17 [INFO] train episode 143: reward = -200.00, steps = 200
    01:50:11 [INFO] train episode 144: reward = -162.00, steps = 162
    01:52:31 [INFO] train episode 145: reward = -200.00, steps = 200
    01:54:23 [INFO] train episode 146: reward = -159.00, steps = 159
    01:55:25 [INFO] train episode 147: reward = -87.00, steps = 87
    01:56:33 [INFO] train episode 148: reward = -96.00, steps = 96
    01:57:34 [INFO] train episode 149: reward = -86.00, steps = 86
    01:59:20 [INFO] train episode 150: reward = -153.00, steps = 153
    02:01:03 [INFO] train episode 151: reward = -147.00, steps = 147
    02:03:22 [INFO] train episode 152: reward = -200.00, steps = 200
    02:05:40 [INFO] train episode 153: reward = -200.00, steps = 200
    02:07:31 [INFO] train episode 154: reward = -158.00, steps = 158
    02:09:25 [INFO] train episode 155: reward = -162.00, steps = 162
    02:10:30 [INFO] train episode 156: reward = -92.00, steps = 92
    02:11:55 [INFO] train episode 157: reward = -122.00, steps = 122
    02:13:07 [INFO] train episode 158: reward = -103.00, steps = 103
    02:15:07 [INFO] train episode 159: reward = -171.00, steps = 171
    02:17:25 [INFO] train episode 160: reward = -200.00, steps = 200
    02:18:24 [INFO] train episode 161: reward = -84.00, steps = 84
    02:20:24 [INFO] train episode 162: reward = -172.00, steps = 172
    02:21:25 [INFO] train episode 163: reward = -87.00, steps = 87
    02:22:53 [INFO] train episode 164: reward = -125.00, steps = 125
    02:24:17 [INFO] train episode 165: reward = -118.00, steps = 118
    02:26:10 [INFO] train episode 166: reward = -162.00, steps = 162
    02:27:15 [INFO] train episode 167: reward = -89.00, steps = 89
    02:28:28 [INFO] train episode 168: reward = -105.00, steps = 105
    02:30:15 [INFO] train episode 169: reward = -154.00, steps = 154
    02:32:05 [INFO] train episode 170: reward = -158.00, steps = 158
    02:33:53 [INFO] train episode 171: reward = -155.00, steps = 155
    02:34:51 [INFO] train episode 172: reward = -83.00, steps = 83
    02:35:58 [INFO] train episode 173: reward = -97.00, steps = 97
    02:37:02 [INFO] train episode 174: reward = -91.00, steps = 91
    02:38:55 [INFO] train episode 175: reward = -160.00, steps = 160
    02:40:44 [INFO] train episode 176: reward = -149.00, steps = 149
    02:41:44 [INFO] train episode 177: reward = -85.00, steps = 85
    02:43:31 [INFO] train episode 178: reward = -156.00, steps = 156
    02:44:54 [INFO] train episode 179: reward = -117.00, steps = 117
    02:46:04 [INFO] train episode 180: reward = -101.00, steps = 101
    02:47:07 [INFO] train episode 181: reward = -91.00, steps = 91
    02:48:28 [INFO] train episode 182: reward = -115.00, steps = 115
    02:49:46 [INFO] train episode 183: reward = -112.00, steps = 112
    02:51:03 [INFO] train episode 184: reward = -111.00, steps = 111
    02:52:23 [INFO] train episode 185: reward = -112.00, steps = 112
    02:53:39 [INFO] train episode 186: reward = -110.00, steps = 110
    02:54:59 [INFO] train episode 187: reward = -112.00, steps = 112
    02:56:16 [INFO] train episode 188: reward = -110.00, steps = 110
    02:56:16 [INFO] ==== test ====
    02:56:30 [INFO] test episode 0: reward = -109.00, steps = 109
    02:56:44 [INFO] test episode 1: reward = -111.00, steps = 111
    02:56:58 [INFO] test episode 2: reward = -112.00, steps = 112
    02:57:12 [INFO] test episode 3: reward = -109.00, steps = 109
    02:57:26 [INFO] test episode 4: reward = -112.00, steps = 112
    02:57:40 [INFO] test episode 5: reward = -112.00, steps = 112
    02:57:53 [INFO] test episode 6: reward = -109.00, steps = 109
    02:58:07 [INFO] test episode 7: reward = -110.00, steps = 110
    02:58:21 [INFO] test episode 8: reward = -114.00, steps = 114
    02:58:34 [INFO] test episode 9: reward = -112.00, steps = 112
    02:58:48 [INFO] test episode 10: reward = -112.00, steps = 112
    02:59:02 [INFO] test episode 11: reward = -114.00, steps = 114
    02:59:16 [INFO] test episode 12: reward = -111.00, steps = 111
    02:59:30 [INFO] test episode 13: reward = -111.00, steps = 111
    02:59:43 [INFO] test episode 14: reward = -109.00, steps = 109
    02:59:56 [INFO] test episode 15: reward = -112.00, steps = 112
    03:00:10 [INFO] test episode 16: reward = -109.00, steps = 109
    03:00:24 [INFO] test episode 17: reward = -112.00, steps = 112
    03:00:38 [INFO] test episode 18: reward = -109.00, steps = 109
    03:00:51 [INFO] test episode 19: reward = -112.00, steps = 112
    03:01:05 [INFO] test episode 20: reward = -109.00, steps = 109
    03:01:18 [INFO] test episode 21: reward = -111.00, steps = 111
    03:01:32 [INFO] test episode 22: reward = -112.00, steps = 112
    03:01:46 [INFO] test episode 23: reward = -112.00, steps = 112
    03:02:00 [INFO] test episode 24: reward = -112.00, steps = 112
    03:02:14 [INFO] test episode 25: reward = -112.00, steps = 112
    03:02:28 [INFO] test episode 26: reward = -109.00, steps = 109
    03:02:41 [INFO] test episode 27: reward = -109.00, steps = 109
    03:02:54 [INFO] test episode 28: reward = -110.00, steps = 110
    03:03:08 [INFO] test episode 29: reward = -112.00, steps = 112
    03:03:22 [INFO] test episode 30: reward = -109.00, steps = 109
    03:03:34 [INFO] test episode 31: reward = -109.00, steps = 109
    03:03:48 [INFO] test episode 32: reward = -109.00, steps = 109
    03:04:02 [INFO] test episode 33: reward = -109.00, steps = 109
    03:04:15 [INFO] test episode 34: reward = -110.00, steps = 110
    03:04:29 [INFO] test episode 35: reward = -112.00, steps = 112
    03:04:43 [INFO] test episode 36: reward = -109.00, steps = 109
    03:04:57 [INFO] test episode 37: reward = -111.00, steps = 111
    03:05:10 [INFO] test episode 38: reward = -109.00, steps = 109
    03:05:24 [INFO] test episode 39: reward = -111.00, steps = 111
    03:05:37 [INFO] test episode 40: reward = -112.00, steps = 112
    03:05:52 [INFO] test episode 41: reward = -112.00, steps = 112
    03:06:05 [INFO] test episode 42: reward = -112.00, steps = 112
    03:06:19 [INFO] test episode 43: reward = -112.00, steps = 112
    03:06:33 [INFO] test episode 44: reward = -112.00, steps = 112
    03:06:47 [INFO] test episode 45: reward = -109.00, steps = 109
    03:07:00 [INFO] test episode 46: reward = -109.00, steps = 109
    03:07:14 [INFO] test episode 47: reward = -111.00, steps = 111
    03:07:27 [INFO] test episode 48: reward = -109.00, steps = 109
    03:07:41 [INFO] test episode 49: reward = -109.00, steps = 109
    03:07:55 [INFO] test episode 50: reward = -109.00, steps = 109
    03:08:09 [INFO] test episode 51: reward = -112.00, steps = 112
    03:08:23 [INFO] test episode 52: reward = -112.00, steps = 112
    03:08:36 [INFO] test episode 53: reward = -111.00, steps = 111
    03:08:49 [INFO] test episode 54: reward = -109.00, steps = 109
    03:09:04 [INFO] test episode 55: reward = -112.00, steps = 112
    03:09:17 [INFO] test episode 56: reward = -109.00, steps = 109
    03:09:31 [INFO] test episode 57: reward = -112.00, steps = 112
    03:09:45 [INFO] test episode 58: reward = -112.00, steps = 112
    03:09:59 [INFO] test episode 59: reward = -109.00, steps = 109
    03:10:13 [INFO] test episode 60: reward = -109.00, steps = 109
    03:10:27 [INFO] test episode 61: reward = -112.00, steps = 112
    03:10:40 [INFO] test episode 62: reward = -109.00, steps = 109
    03:10:54 [INFO] test episode 63: reward = -109.00, steps = 109
    03:11:07 [INFO] test episode 64: reward = -110.00, steps = 110
    03:11:20 [INFO] test episode 65: reward = -109.00, steps = 109
    03:11:34 [INFO] test episode 66: reward = -109.00, steps = 109
    03:11:47 [INFO] test episode 67: reward = -109.00, steps = 109
    03:12:00 [INFO] test episode 68: reward = -109.00, steps = 109
    03:12:14 [INFO] test episode 69: reward = -112.00, steps = 112
    03:12:28 [INFO] test episode 70: reward = -112.00, steps = 112
    03:12:42 [INFO] test episode 71: reward = -109.00, steps = 109
    03:12:56 [INFO] test episode 72: reward = -112.00, steps = 112
    03:13:09 [INFO] test episode 73: reward = -112.00, steps = 112
    03:13:23 [INFO] test episode 74: reward = -110.00, steps = 110
    03:13:36 [INFO] test episode 75: reward = -109.00, steps = 109
    03:13:50 [INFO] test episode 76: reward = -110.00, steps = 110
    03:14:04 [INFO] test episode 77: reward = -110.00, steps = 110
    03:14:17 [INFO] test episode 78: reward = -112.00, steps = 112
    03:14:31 [INFO] test episode 79: reward = -111.00, steps = 111
    03:14:45 [INFO] test episode 80: reward = -110.00, steps = 110
    03:14:58 [INFO] test episode 81: reward = -112.00, steps = 112
    03:15:12 [INFO] test episode 82: reward = -112.00, steps = 112
    03:15:26 [INFO] test episode 83: reward = -112.00, steps = 112
    03:15:40 [INFO] test episode 84: reward = -112.00, steps = 112
    03:15:53 [INFO] test episode 85: reward = -112.00, steps = 112
    03:16:07 [INFO] test episode 86: reward = -110.00, steps = 110
    03:16:21 [INFO] test episode 87: reward = -112.00, steps = 112
    03:16:35 [INFO] test episode 88: reward = -111.00, steps = 111
    03:16:48 [INFO] test episode 89: reward = -112.00, steps = 112
    03:17:01 [INFO] test episode 90: reward = -109.00, steps = 109
    03:17:15 [INFO] test episode 91: reward = -112.00, steps = 112
    03:17:29 [INFO] test episode 92: reward = -109.00, steps = 109
    03:17:43 [INFO] test episode 93: reward = -114.00, steps = 114
    03:17:57 [INFO] test episode 94: reward = -112.00, steps = 112
    03:18:11 [INFO] test episode 95: reward = -112.00, steps = 112
    03:18:24 [INFO] test episode 96: reward = -113.00, steps = 113
    03:18:38 [INFO] test episode 97: reward = -111.00, steps = 111
    03:18:51 [INFO] test episode 98: reward = -109.00, steps = 109
    03:19:05 [INFO] test episode 99: reward = -110.00, steps = 110
    03:19:05 [INFO] average episode reward = -110.71 ± 1.46

In [6]:

    env.close()