# Use VPG with Baseline to Play CartPole-v0¶

TensorFlow version

In [1]:

    %matplotlib inline

    import sys
    import logging
    import itertools

    import numpy as np
    np.random.seed(0)
    import pandas as pd
    import gym
    import matplotlib.pyplot as plt
    import tensorflow.compat.v2 as tf
    tf.random.set_seed(0)
    from tensorflow import keras
    from tensorflow import nn
    from tensorflow import optimizers
    from tensorflow import losses
    from tensorflow.keras import layers

    logging.basicConfig(level=logging.INFO,
            format='%(asctime)s [%(levelname)s] %(message)s',
            stream=sys.stdout, datefmt='%H:%M:%S')

In [2]:

    env = gym.make('CartPole-v0')
    for key in vars(env):
        logging.info('%s: %s', key, vars(env)[key])
    for key in vars(env.spec):
        logging.info('%s: %s', key, vars(env.spec)[key])

    10:46:49 [INFO] env: <CartPoleEnv<CartPole-v0>>
    10:46:49 [INFO] action_space: Discrete(2)
    10:46:49 [INFO] observation_space: Box(-3.4028234663852886e+38, 3.4028234663852886e+38, (4,), float32)
    10:46:49 [INFO] reward_range: (-inf, inf)
    10:46:49 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 50}
    10:46:49 [INFO] _max_episode_steps: 200
    10:46:49 [INFO] _elapsed_steps: None
    10:46:49 [INFO] id: CartPole-v0
    10:46:49 [INFO] entry_point: gym.envs.classic_control:CartPoleEnv
    10:46:49 [INFO] reward_threshold: 195.0
    10:46:49 [INFO] nondeterministic: False
    10:46:49 [INFO] max_episode_steps: 200
    10:46:49 [INFO] _kwargs: {}
    10:46:49 [INFO] _env_name: CartPole

In [3]:

    class VPGwBaselineAgent:
        def __init__(self, env):
            self.action_n = env.action_space.n
            self.gamma = 0.99

            self.trajectory = []

            self.policy_net = self.build_net(hidden_sizes=[],
                    output_size=self.action_n,
                    output_activation=nn.softmax,
                    loss=losses.categorical_crossentropy,
                    learning_rate=0.005)
            self.baseline_net = self.build_net(hidden_sizes=[],
                    learning_rate=0.01)

        def build_net(self, hidden_sizes, output_size=1,
                activation=nn.relu, output_activation=None,
                use_bias=False, loss=losses.mse, learning_rate=0.005):
            model = keras.Sequential()
            for hidden_size in hidden_sizes:
                model.add(layers.Dense(units=hidden_size,
                        activation=activation, use_bias=use_bias))
            model.add(layers.Dense(units=output_size,
                    activation=output_activation, use_bias=use_bias))
            optimizer = optimizers.Adam(learning_rate)
            model.compile(optimizer=optimizer, loss=loss)
            return model

        def reset(self, mode=None):
            self.mode = mode
            if self.mode == 'train':
                self.trajectory = []

        def step(self, observation, reward, terminated):
            probs = self.policy_net.predict(observation[np.newaxis], verbose=0)[0]
            action = np.random.choice(self.action_n, p=probs)
            if self.mode == 'train':
                self.trajectory += [observation, reward, terminated, action]
            return action

        def close(self):
            if self.mode == 'train':
                self.learn()

        def learn(self):
            df = pd.DataFrame(np.array(self.trajectory, dtype=object).reshape(-1, 4),
                    columns=['state', 'reward', 'terminated', 'action'])

            # update baseline
            df['discount'] = self.gamma ** df.index.to_series()
            df['discounted_reward'] = df['discount'] * df['reward'].astype(float)
            df['discounted_return'] = df['discounted_reward'][::-1].cumsum()
            df['return'] = df['discounted_return'] / df['discount']
            states = np.stack(df['state'])
            returns = df[['return',]].values
            self.baseline_net.fit(states, returns, verbose=0)

            # update policy
            df['baseline'] = self.baseline_net.predict(states, verbose=0)
            df['psi'] = df['discounted_return'] - df['baseline'] * df['discount']
            actions = np.eye(self.action_n)[df['action'].astype(int)]
            sample_weight = df[['discounted_return',]].values
            self.policy_net.fit(states, actions, sample_weight=sample_weight,
                    verbose=0)

    agent = VPGwBaselineAgent(env)

In [4]:

    def play_episode(env, agent, seed=None, mode=None, render=False):
        observation, _ = env.reset(seed=seed)
        reward, terminated, truncated = 0., False, False
        agent.reset(mode=mode)
        episode_reward, elapsed_steps = 0., 0
        while True:
            action = agent.step(observation, reward, terminated)
            if render:
                env.render()
            if terminated or truncated:
                break
            observation, reward, terminated, truncated, _ = env.step(action)
            episode_reward += reward
            elapsed_steps += 1
        agent.close()
        return episode_reward, elapsed_steps

    logging.info('==== train ====')
    episode_rewards = []
    for episode in itertools.count():
        episode_reward, elapsed_steps = play_episode(env, agent, seed=episode,
                mode='train')
        episode_rewards.append(episode_reward)
        logging.info('train episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
        if np.mean(episode_rewards[-10:]) > 195:
            break
    plt.plot(episode_rewards)

    logging.info('==== test ====')
    episode_rewards = []
    for episode in range(100):
        episode_reward, elapsed_steps = play_episode(env, agent)
        episode_rewards.append(episode_reward)
        logging.info('test episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
    logging.info('average episode reward = %.2f ± %.2f',
            np.mean(episode_rewards), np.std(episode_rewards))

    10:46:50 [INFO] ==== train ====
    10:46:53 [INFO] train episode 0: reward = 29.00, steps = 29
    10:46:53 [INFO] NumExpr defaulting to 8 threads.
    10:46:59 [INFO] train episode 1: reward = 39.00, steps = 39
    10:47:03 [INFO] train episode 2: reward = 44.00, steps = 44
    10:47:08 [INFO] train episode 3: reward = 38.00, steps = 38
    10:47:11 [INFO] train episode 4: reward = 33.00, steps = 33
    10:47:13 [INFO] train episode 5: reward = 20.00, steps = 20
    10:47:18 [INFO] train episode 6: reward = 49.00, steps = 49
    10:47:24 [INFO] train episode 7: reward = 56.00, steps = 56
    10:47:27 [INFO] train episode 8: reward = 30.00, steps = 30
    10:47:31 [INFO] train episode 9: reward = 44.00, steps = 44
    10:47:34 [INFO] train episode 10: reward = 26.00, steps = 26
    10:47:40 [INFO] train episode 11: reward = 64.00, steps = 64
    10:47:46 [INFO] train episode 12: reward = 56.00, steps = 56
    10:47:48 [INFO] train episode 13: reward = 24.00, steps = 24
    10:47:55 [INFO] train episode 14: reward = 65.00, steps = 65
    10:47:56 [INFO] train episode 15: reward = 14.00, steps = 14
    10:48:01 [INFO] train episode 16: reward = 44.00, steps = 44
    10:48:03 [INFO] train episode 17: reward = 20.00, steps = 20
    10:48:07 [INFO] train episode 18: reward = 39.00, steps = 39
    10:48:11 [INFO] train episode 19: reward = 47.00, steps = 47
    10:48:18 [INFO] train episode 20: reward = 61.00, steps = 61
    10:48:21 [INFO] train episode 21: reward = 38.00, steps = 38
    10:48:23 [INFO] train episode 22: reward = 19.00, steps = 19
    10:48:28 [INFO] train episode 23: reward = 40.00, steps = 40
    10:48:32 [INFO] train episode 24: reward = 39.00, steps = 39
    10:48:38 [INFO] train episode 25: reward = 64.00, steps = 64
    10:48:43 [INFO] train episode 26: reward = 47.00, steps = 47
    10:48:44 [INFO] train episode 27: reward = 15.00, steps = 15
    10:48:53 [INFO] train episode 28: reward = 94.00, steps = 94
    10:48:56 [INFO] train episode 29: reward = 19.00, steps = 19
    10:48:58 [INFO] train episode 30: reward = 22.00, steps = 22
    10:49:03 [INFO] train episode 31: reward = 57.00, steps = 57
    10:49:08 [INFO] train episode 32: reward = 50.00, steps = 50
    10:49:12 [INFO] train episode 33: reward = 31.00, steps = 31
    10:49:15 [INFO] train episode 34: reward = 23.00, steps = 23
    10:49:17 [INFO] train episode 35: reward = 26.00, steps = 26
    10:49:24 [INFO] train episode 36: reward = 73.00, steps = 73
    10:49:27 [INFO] train episode 37: reward = 25.00, steps = 25
    10:49:34 [INFO] train episode 38: reward = 67.00, steps = 67
    10:49:38 [INFO] train episode 39: reward = 46.00, steps = 46
    10:49:43 [INFO] train episode 40: reward = 54.00, steps = 54
    10:49:50 [INFO] train episode 41: reward = 66.00, steps = 66
    10:49:53 [INFO] train episode 42: reward = 30.00, steps = 30
    10:49:58 [INFO] train episode 43: reward = 52.00, steps = 52
    10:50:01 [INFO] train episode 44: reward = 26.00, steps = 26
    10:50:03 [INFO] train episode 45: reward = 22.00, steps = 22
    10:50:06 [INFO] train episode 46: reward = 27.00, steps = 27
    10:50:08 [INFO] train episode 47: reward = 16.00, steps = 16
    10:50:10 [INFO] train episode 48: reward = 18.00, steps = 18
    10:50:12 [INFO] train episode 49: reward = 22.00, steps = 22
    10:50:20 [INFO] train episode 50: reward = 76.00, steps = 76
    10:50:25 [INFO] train episode 51: reward = 58.00, steps = 58
    10:50:30 [INFO] train episode 52: reward = 41.00, steps = 41
    10:50:34 [INFO] train episode 53: reward = 47.00, steps = 47
    10:50:38 [INFO] train episode 54: reward = 41.00, steps = 41
    10:50:45 [INFO] train episode 55: reward = 63.00, steps = 63
    10:50:48 [INFO] train episode 56: reward = 26.00, steps = 26
    10:50:57 [INFO] train episode 57: reward = 93.00, steps = 93
    10:51:02 [INFO] train episode 58: reward = 43.00, steps = 43
    10:51:10 [INFO] train episode 59: reward = 83.00, steps = 83
    10:51:11 [INFO] train episode 60: reward = 14.00, steps = 14
    10:51:15 [INFO] train episode 61: reward = 29.00, steps = 29
    10:51:18 [INFO] train episode 62: reward = 37.00, steps = 37
    10:51:26 [INFO] train episode 63: reward = 76.00, steps = 76
    10:51:28 [INFO] train episode 64: reward = 22.00, steps = 22
    10:51:35 [INFO] train episode 65: reward = 78.00, steps = 78
    10:51:41 [INFO] train episode 66: reward = 54.00, steps = 54
    10:51:45 [INFO] train episode 67: reward = 45.00, steps = 45
    10:51:47 [INFO] train episode 68: reward = 13.00, steps = 13
    10:51:53 [INFO] train episode 69: reward = 68.00, steps = 68
    10:51:59 [INFO] train episode 70: reward = 60.00, steps = 60
    10:52:06 [INFO] train episode 71: reward = 76.00, steps = 76
    10:52:14 [INFO] train episode 72: reward = 86.00, steps = 86
    10:52:20 [INFO] train episode 73: reward = 65.00, steps = 65
    10:52:23 [INFO] train episode 74: reward = 18.00, steps = 18
    10:52:29 [INFO] train episode 75: reward = 57.00, steps = 57
    10:52:32 [INFO] train episode 76: reward = 26.00, steps = 26
    10:52:34 [INFO] train episode 77: reward = 22.00, steps = 22
    10:52:42 [INFO] train episode 78: reward = 71.00, steps = 71
    10:52:50 [INFO] train episode 79: reward = 90.00, steps = 90
    10:52:55 [INFO] train episode 80: reward = 45.00, steps = 45
    10:53:00 [INFO] train episode 81: reward = 50.00, steps = 50
    10:53:03 [INFO] train episode 82: reward = 26.00, steps = 26
    10:53:05 [INFO] train episode 83: reward = 14.00, steps = 14
    10:53:10 [INFO] train episode 84: reward = 46.00, steps = 46
    10:53:15 [INFO] train episode 85: reward = 50.00, steps = 50
    10:53:18 [INFO] train episode 86: reward = 23.00, steps = 23
    10:53:23 [INFO] train episode 87: reward = 46.00, steps = 46
    10:53:27 [INFO] train episode 88: reward = 37.00, steps = 37
    10:53:29 [INFO] train episode 89: reward = 22.00, steps = 22
    10:53:38 [INFO] train episode 90: reward = 80.00, steps = 80
    10:53:43 [INFO] train episode 91: reward = 52.00, steps = 52
    10:53:53 [INFO] train episode 92: reward = 98.00, steps = 98
    10:53:56 [INFO] train episode 93: reward = 35.00, steps = 35
    10:53:59 [INFO] train episode 94: reward = 24.00, steps = 24
    10:54:09 [INFO] train episode 95: reward = 95.00, steps = 95
    10:54:13 [INFO] train episode 96: reward = 38.00, steps = 38
    10:54:18 [INFO] train episode 97: reward = 55.00, steps = 55
    10:54:23 [INFO] train episode 98: reward = 42.00, steps = 42
    10:54:28 [INFO] train episode 99: reward = 50.00, steps = 50
    10:54:37 [INFO] train episode 100: reward = 86.00, steps = 86
    10:54:42 [INFO] train episode 101: reward = 45.00, steps = 45
    10:54:45 [INFO] train episode 102: reward = 31.00, steps = 31
    10:54:49 [INFO] train episode 103: reward = 30.00, steps = 30
    10:54:51 [INFO] train episode 104: reward = 19.00, steps = 19
    10:54:55 [INFO] train episode 105: reward = 34.00, steps = 34
    10:55:02 [INFO] train episode 106: reward = 57.00, steps = 57
    10:55:04 [INFO] train episode 107: reward = 19.00, steps = 19
    10:55:10 [INFO] train episode 108: reward = 52.00, steps = 52
    10:55:13 [INFO] train episode 109: reward = 35.00, steps = 35
    10:55:15 [INFO] train episode 110: reward = 18.00, steps = 18
    10:55:26 [INFO] train episode 111: reward = 118.00, steps = 118
    10:55:30 [INFO] train episode 112: reward = 45.00, steps = 45
    10:55:34 [INFO] train episode 113: reward = 35.00, steps = 35
    10:55:41 [INFO] train episode 114: reward = 69.00, steps = 69
    10:55:44 [INFO] train episode 115: reward = 39.00, steps = 39
    10:55:55 [INFO] train episode 116: reward = 114.00, steps = 114
    10:55:57 [INFO] train episode 117: reward = 18.00, steps = 18
    10:56:03 [INFO] train episode 118: reward = 55.00, steps = 55
    10:56:05 [INFO] train episode 119: reward = 29.00, steps = 29
    10:56:07 [INFO] train episode 120: reward = 15.00, steps = 15
    10:56:12 [INFO] train episode 121: reward = 45.00, steps = 45
    10:56:14 [INFO] train episode 122: reward = 21.00, steps = 21
    10:56:19 [INFO] train episode 123: reward = 57.00, steps = 57
    10:56:24 [INFO] train episode 124: reward = 44.00, steps = 44
    10:56:31 [INFO] train episode 125: reward = 71.00, steps = 71
    10:56:34 [INFO] train episode 126: reward = 38.00, steps = 38
    10:56:41 [INFO] train episode 127: reward = 65.00, steps = 65
    10:56:45 [INFO] train episode 128: reward = 34.00, steps = 34
    10:56:48 [INFO] train episode 129: reward = 29.00, steps = 29
    10:56:51 [INFO] train episode 130: reward = 36.00, steps = 36
    10:56:56 [INFO] train episode 131: reward = 50.00, steps = 50
    10:57:01 [INFO] train episode 132: reward = 53.00, steps = 53
    10:57:07 [INFO] train episode 133: reward = 54.00, steps = 54
    10:57:10 [INFO] train episode 134: reward = 30.00, steps = 30
    10:57:18 [INFO] train episode 135: reward = 80.00, steps = 80
    10:57:29 [INFO] train episode 136: reward = 113.00, steps = 113
    10:57:40 [INFO] train episode 137: reward = 108.00, steps = 108
    10:57:45 [INFO] train episode 138: reward = 60.00, steps = 60
    10:57:51 [INFO] train episode 139: reward = 54.00, steps = 54
    10:57:55 [INFO] train episode 140: reward = 36.00, steps = 36
    10:57:58 [INFO] train episode 141: reward = 35.00, steps = 35
    10:58:03 [INFO] train episode 142: reward = 42.00, steps = 42
    10:58:04 [INFO] train episode 143: reward = 11.00, steps = 11
    10:58:07 [INFO] train episode 144: reward = 30.00, steps = 30
    10:58:17 [INFO] train episode 145: reward = 94.00, steps = 94
    10:58:19 [INFO] train episode 146: reward = 15.00, steps = 15
    10:58:22 [INFO] train episode 147: reward = 40.00, steps = 40
    10:58:26 [INFO] train episode 148: reward = 30.00, steps = 30
    10:58:31 [INFO] train episode 149: reward = 60.00, steps = 60
    10:58:35 [INFO] train episode 150: reward = 30.00, steps = 30
    10:58:38 [INFO] train episode 151: reward = 30.00, steps = 30
    10:58:40 [INFO] train episode 152: reward = 19.00, steps = 19
    10:58:48 [INFO] train episode 153: reward = 88.00, steps = 88
    10:58:55 [INFO] train episode 154: reward = 67.00, steps = 67
    10:59:04 [INFO] train episode 155: reward = 101.00, steps = 101
    10:59:06 [INFO] train episode 156: reward = 21.00, steps = 21
    10:59:18 [INFO] train episode 157: reward = 129.00, steps = 129
    10:59:26 [INFO] train episode 158: reward = 80.00, steps = 80
    10:59:30 [INFO] train episode 159: reward = 43.00, steps = 43
    10:59:37 [INFO] train episode 160: reward = 73.00, steps = 73
    10:59:41 [INFO] train episode 161: reward = 40.00, steps = 40
    10:59:47 [INFO] train episode 162: reward = 58.00, steps = 58
    10:59:50 [INFO] train episode 163: reward = 34.00, steps = 34
    10:59:55 [INFO] train episode 164: reward = 47.00, steps = 47
    11:00:01 [INFO] train episode 165: reward = 63.00, steps = 63
    11:00:13 [INFO] train episode 166: reward = 125.00, steps = 125
    11:00:19 [INFO] train episode 167: reward = 53.00, steps = 53
    11:00:25 [INFO] train episode 168: reward = 66.00, steps = 66
    11:00:32 [INFO] train episode 169: reward = 72.00, steps = 72
    11:00:46 [INFO] train episode 170: reward = 155.00, steps = 155
    11:00:49 [INFO] train episode 171: reward = 26.00, steps = 26
    11:00:53 [INFO] train episode 172: reward = 46.00, steps = 46
    11:00:58 [INFO] train episode 173: reward = 45.00, steps = 45
    11:01:08 [INFO] train episode 174: reward = 112.00, steps = 112
    11:01:20 [INFO] train episode 175: reward = 119.00, steps = 119
    11:01:28 [INFO] train episode 176: reward = 84.00, steps = 84
    11:01:36 [INFO] train episode 177: reward = 82.00, steps = 82
    11:01:41 [INFO] train episode 178: reward = 55.00, steps = 55
    11:01:47 [INFO] train episode 179: reward = 63.00, steps = 63
    11:01:51 [INFO] train episode 180: reward = 34.00, steps = 34
    11:01:55 [INFO] train episode 181: reward = 47.00, steps = 47
    11:01:58 [INFO] train episode 182: reward = 28.00, steps = 28
    11:02:05 [INFO] train episode 183: reward = 70.00, steps = 70
    11:02:09 [INFO] train episode 184: reward = 31.00, steps = 31
    11:02:14 [INFO] train episode 185: reward = 53.00, steps = 53
    11:02:19 [INFO] train episode 186: reward = 49.00, steps = 49
    11:02:33 [INFO] train episode 187: reward = 152.00, steps = 152
    11:02:39 [INFO] train episode 188: reward = 49.00, steps = 49
    11:02:44 [INFO] train episode 189: reward = 53.00, steps = 53
    11:02:54 [INFO] train episode 190: reward = 101.00, steps = 101
    11:03:01 [INFO] train episode 191: reward = 75.00, steps = 75
    11:03:12 [INFO] train episode 192: reward = 111.00, steps = 111
    11:03:17 [INFO] train episode 193: reward = 46.00, steps = 46
    11:03:24 [INFO] train episode 194: reward = 71.00, steps = 71
    11:03:30 [INFO] train episode 195: reward = 58.00, steps = 58
    11:03:32 [INFO] train episode 196: reward = 20.00, steps = 20
    11:03:38 [INFO] train episode 197: reward = 63.00, steps = 63
    11:03:50 [INFO] train episode 198: reward = 117.00, steps = 117
    11:03:56 [INFO] train episode 199: reward = 69.00, steps = 69
    11:04:05 [INFO] train episode 200: reward = 78.00, steps = 78
    11:04:07 [INFO] train episode 201: reward = 28.00, steps = 28
    11:04:17 [INFO] train episode 202: reward = 93.00, steps = 93
    11:04:30 [INFO] train episode 203: reward = 134.00, steps = 134
    11:04:33 [INFO] train episode 204: reward = 29.00, steps = 29
    11:04:47 [INFO] train episode 205: reward = 143.00, steps = 143
    11:04:51 [INFO] train episode 206: reward = 41.00, steps = 41
    11:04:58 [INFO] train episode 207: reward = 71.00, steps = 71
    11:05:02 [INFO] train episode 208: reward = 38.00, steps = 38
    11:05:06 [INFO] train episode 209: reward = 40.00, steps = 40
    11:05:16 [INFO] train episode 210: reward = 104.00, steps = 104
    11:05:20 [INFO] train episode 211: reward = 37.00, steps = 37
    11:05:26 [INFO] train episode 212: reward = 59.00, steps = 59
    11:05:29 [INFO] train episode 213: reward = 25.00, steps = 25
    11:05:32 [INFO] train episode 214: reward = 32.00, steps = 32
    11:05:49 [INFO] train episode 215: reward = 177.00, steps = 177
    11:05:57 [INFO] train episode 216: reward = 77.00, steps = 77
    11:06:01 [INFO] train episode 217: reward = 29.00, steps = 29
    11:06:03 [INFO] train episode 218: reward = 21.00, steps = 21
    11:06:16 [INFO] train episode 219: reward = 140.00, steps = 140
    11:06:20 [INFO] train episode 220: reward = 32.00, steps = 32
    11:06:23 [INFO] train episode 221: reward = 29.00, steps = 29
    11:06:29 [INFO] train episode 222: reward = 58.00, steps = 58
    11:06:36 [INFO] train episode 223: reward = 74.00, steps = 74
    11:06:44 [INFO] train episode 224: reward = 87.00, steps = 87
    11:06:52 [INFO] train episode 225: reward = 86.00, steps = 86
    11:07:09 [INFO] train episode 226: reward = 175.00, steps = 175
    11:07:27 [INFO] train episode 227: reward = 188.00, steps = 188
    11:07:36 [INFO] train episode 228: reward = 91.00, steps = 91
    11:07:43 [INFO] train episode 229: reward = 74.00, steps = 74
    11:07:54 [INFO] train episode 230: reward = 103.00, steps = 103
    11:08:03 [INFO] train episode 231: reward = 106.00, steps = 106
    11:08:22 [INFO] train episode 232: reward = 200.00, steps = 200
    11:08:29 [INFO] train episode 233: reward = 64.00, steps = 64
    11:08:37 [INFO] train episode 234: reward = 82.00, steps = 82
    11:08:43 [INFO] train episode 235: reward = 60.00, steps = 60
    11:08:49 [INFO] train episode 236: reward = 68.00, steps = 68
    11:09:06 [INFO] train episode 237: reward = 179.00, steps = 179
    11:09:13 [INFO] train episode 238: reward = 67.00, steps = 67
    11:09:16 [INFO] train episode 239: reward = 28.00, steps = 28
    11:09:22 [INFO] train episode 240: reward = 62.00, steps = 62
    11:09:25 [INFO] train episode 241: reward = 24.00, steps = 24
    11:09:45 [INFO] train episode 242: reward = 200.00, steps = 200
    11:10:06 [INFO] train episode 243: reward = 200.00, steps = 200
    11:10:18 [INFO] train episode 244: reward = 107.00, steps = 107
    11:10:28 [INFO] train episode 245: reward = 90.00, steps = 90
    11:10:37 [INFO] train episode 246: reward = 89.00, steps = 89
    11:10:46 [INFO] train episode 247: reward = 98.00, steps = 98
    11:11:03 [INFO] train episode 248: reward = 168.00, steps = 168
    11:11:13 [INFO] train episode 249: reward = 108.00, steps = 108
    11:11:27 [INFO] train episode 250: reward = 134.00, steps = 134
    11:11:38 [INFO] train episode 251: reward = 103.00, steps = 103
    11:11:45 [INFO] train episode 252: reward = 59.00, steps = 59
    11:11:51 [INFO] train episode 253: reward = 60.00, steps = 60
    11:12:02 [INFO] train episode 254: reward = 110.00, steps = 110
    11:12:20 [INFO] train episode 255: reward = 162.00, steps = 162
    11:12:37 [INFO] train episode 256: reward = 166.00, steps = 166
    11:12:42 [INFO] train episode 257: reward = 35.00, steps = 35
    11:12:55 [INFO] train episode 258: reward = 118.00, steps = 118
    11:12:59 [INFO] train episode 259: reward = 34.00, steps = 34
    11:13:05 [INFO] train episode 260: reward = 60.00, steps = 60
    11:13:15 [INFO] train episode 261: reward = 110.00, steps = 110
    11:13:24 [INFO] train episode 262: reward = 88.00, steps = 88
    11:13:32 [INFO] train episode 263: reward = 85.00, steps = 85
    11:13:38 [INFO] train episode 264: reward = 68.00, steps = 68
    11:13:47 [INFO] train episode 265: reward = 94.00, steps = 94
    11:13:50 [INFO] train episode 266: reward = 16.00, steps = 16
    11:13:53 [INFO] train episode 267: reward = 32.00, steps = 32
    11:13:57 [INFO] train episode 268: reward = 39.00, steps = 39
    11:14:15 [INFO] train episode 269: reward = 196.00, steps = 196
    11:14:18 [INFO] train episode 270: reward = 22.00, steps = 22
    11:14:23 [INFO] train episode 271: reward = 44.00, steps = 44
    11:14:41 [INFO] train episode 272: reward = 200.00, steps = 200
    11:14:47 [INFO] train episode 273: reward = 59.00, steps = 59
    11:14:57 [INFO] train episode 274: reward = 116.00, steps = 116
    11:15:09 [INFO] train episode 275: reward = 119.00, steps = 119
    11:15:20 [INFO] train episode 276: reward = 126.00, steps = 126
    11:15:36 [INFO] train episode 277: reward = 174.00, steps = 174
    11:15:55 [INFO] train episode 278: reward = 200.00, steps = 200
    11:16:11 [INFO] train episode 279: reward = 175.00, steps = 175
    11:16:19 [INFO] train episode 280: reward = 78.00, steps = 78
    11:16:32 [INFO] train episode 281: reward = 136.00, steps = 136
    11:16:45 [INFO] train episode 282: reward = 139.00, steps = 139
    11:16:47 [INFO] train episode 283: reward = 15.00, steps = 15
    11:16:56 [INFO] train episode 284: reward = 88.00, steps = 88
    11:17:01 [INFO] train episode 285: reward = 52.00, steps = 52
    11:17:04 [INFO] train episode 286: reward = 22.00, steps = 22
    11:17:25 [INFO] train episode 287: reward = 200.00, steps = 200
    11:17:28 [INFO] train episode 288: reward = 27.00, steps = 27
    11:17:44 [INFO] train episode 289: reward = 169.00, steps = 169
    11:18:02 [INFO] train episode 290: reward = 200.00, steps = 200
    11:18:12 [INFO] train episode 291: reward = 101.00, steps = 101
    11:18:30 [INFO] train episode 292: reward = 200.00, steps = 200
    11:18:34 [INFO] train episode 293: reward = 37.00, steps = 37
    11:18:47 [INFO] train episode 294: reward = 146.00, steps = 146
    11:18:52 [INFO] train episode 295: reward = 45.00, steps = 45
    11:19:10 [INFO] train episode 296: reward = 200.00, steps = 200
    11:19:20 [INFO] train episode 297: reward = 111.00, steps = 111
    11:19:32 [INFO] train episode 298: reward = 125.00, steps = 125
    11:19:46 [INFO] train episode 299: reward = 145.00, steps = 145
    11:19:59 [INFO] train episode 300: reward = 139.00, steps = 139
    11:20:17 [INFO] train episode 301: reward = 182.00, steps = 182
    11:20:39 [INFO] train episode 302: reward = 200.00, steps = 200
    11:21:00 [INFO] train episode 303: reward = 199.00, steps = 199
    11:21:10 [INFO] train episode 304: reward = 112.00, steps = 112
    11:21:15 [INFO] train episode 305: reward = 45.00, steps = 45
    11:21:27 [INFO] train episode 306: reward = 128.00, steps = 128
    11:21:30 [INFO] train episode 307: reward = 30.00, steps = 30
    11:21:47 [INFO] train episode 308: reward = 183.00, steps = 183
    11:21:50 [INFO] train episode 309: reward = 27.00, steps = 27
    11:22:03 [INFO] train episode 310: reward = 147.00, steps = 147
    11:22:22 [INFO] train episode 311: reward = 200.00, steps = 200
    11:22:40 [INFO] train episode 312: reward = 200.00, steps = 200
    11:22:58 [INFO] train episode 313: reward = 200.00, steps = 200
    11:23:15 [INFO] train episode 314: reward = 175.00, steps = 175
    11:23:23 [INFO] train episode 315: reward = 90.00, steps = 90
    11:23:31 [INFO] train episode 316: reward = 82.00, steps = 82
    11:23:44 [INFO] train episode 317: reward = 141.00, steps = 141
    11:24:01 [INFO] train episode 318: reward = 187.00, steps = 187
    11:24:06 [INFO] train episode 319: reward = 49.00, steps = 49
    11:24:24 [INFO] train episode 320: reward = 200.00, steps = 200
    11:24:41 [INFO] train episode 321: reward = 178.00, steps = 178
    11:24:52 [INFO] train episode 322: reward = 125.00, steps = 125
    11:25:09 [INFO] train episode 323: reward = 177.00, steps = 177
    11:25:13 [INFO] train episode 324: reward = 37.00, steps = 37
    11:25:16 [INFO] train episode 325: reward = 28.00, steps = 28
    11:25:25 [INFO] train episode 326: reward = 96.00, steps = 96
    11:25:27 [INFO] train episode 327: reward = 16.00, steps = 16
    11:25:32 [INFO] train episode 328: reward = 56.00, steps = 56
    11:25:35 [INFO] train episode 329: reward = 24.00, steps = 24
    11:25:48 [INFO] train episode 330: reward = 143.00, steps = 143
    11:25:59 [INFO] train episode 331: reward = 121.00, steps = 121
    11:26:18 [INFO] train episode 332: reward = 200.00, steps = 200
    11:26:35 [INFO] train episode 333: reward = 185.00, steps = 185
    11:26:48 [INFO] train episode 334: reward = 145.00, steps = 145
    11:26:53 [INFO] train episode 335: reward = 47.00, steps = 47
    11:27:07 [INFO] train episode 336: reward = 148.00, steps = 148
    11:27:25 [INFO] train episode 337: reward = 200.00, steps = 200
    11:27:36 [INFO] train episode 338: reward = 123.00, steps = 123
    11:27:42 [INFO] train episode 339: reward = 55.00, steps = 55
    11:27:53 [INFO] train episode 340: reward = 126.00, steps = 126
    11:28:10 [INFO] train episode 341: reward = 179.00, steps = 179
    11:28:28 [INFO] train episode 342: reward = 197.00, steps = 197
    11:28:31 [INFO] train episode 343: reward = 29.00, steps = 29
    11:28:45 [INFO] train episode 344: reward = 152.00, steps = 152
    11:28:58 [INFO] train episode 345: reward = 144.00, steps = 144
    11:29:03 [INFO] train episode 346: reward = 45.00, steps = 45
    11:29:13 [INFO] train episode 347: reward = 116.00, steps = 116
    11:29:27 [INFO] train episode 348: reward = 141.00, steps = 141
    11:29:40 [INFO] train episode 349: reward = 149.00, steps = 149
    11:29:59 [INFO] train episode 350: reward = 200.00, steps = 200
    11:30:17 [INFO] train episode 351: reward = 200.00, steps = 200
    11:30:28 [INFO] train episode 352: reward = 125.00, steps = 125
    11:30:47 [INFO] train episode 353: reward = 200.00, steps = 200
    11:30:56 [INFO] train episode 354: reward = 97.00, steps = 97
    11:31:07 [INFO] train episode 355: reward = 118.00, steps = 118
    11:31:25 [INFO] train episode 356: reward = 200.00, steps = 200
    11:31:37 [INFO] train episode 357: reward = 125.00, steps = 125
    11:31:44 [INFO] train episode 358: reward = 75.00, steps = 75
    11:31:56 [INFO] train episode 359: reward = 126.00, steps = 126
    11:32:07 [INFO] train episode 360: reward = 127.00, steps = 127
    11:32:11 [INFO] train episode 361: reward = 35.00, steps = 35
    11:32:16 [INFO] train episode 362: reward = 59.00, steps = 59
    11:32:30 [INFO] train episode 363: reward = 145.00, steps = 145
    11:32:34 [INFO] train episode 364: reward = 44.00, steps = 44
    11:32:52 [INFO] train episode 365: reward = 194.00, steps = 194
    11:33:12 [INFO] train episode 366: reward = 200.00, steps = 200
    11:33:28 [INFO] train episode 367: reward = 174.00, steps = 174
    11:33:31 [INFO] train episode 368: reward = 28.00, steps = 28
    11:33:51 [INFO] train episode 369: reward = 199.00, steps = 199
    11:34:10 [INFO] train episode 370: reward = 194.00, steps = 194
    11:34:14 [INFO] train episode 371: reward = 37.00, steps = 37
    11:34:28 [INFO] train episode 372: reward = 151.00, steps = 151
    11:34:46 [INFO] train episode 373: reward = 200.00, steps = 200
    11:35:04 [INFO] train episode 374: reward = 200.00, steps = 200
    11:35:20 [INFO] train episode 375: reward = 165.00, steps = 165
    11:35:38 [INFO] train episode 376: reward = 200.00, steps = 200
    11:35:47 [INFO] train episode 377: reward = 89.00, steps = 89
    11:36:05 [INFO] train episode 378: reward = 200.00, steps = 200
    11:36:23 [INFO] train episode 379: reward = 200.00, steps = 200
    11:36:42 [INFO] train episode 380: reward = 200.00, steps = 200
    11:37:00 [INFO] train episode 381: reward = 200.00, steps = 200
    11:37:18 [INFO] train episode 382: reward = 200.00, steps = 200
    11:37:30 [INFO] train episode 383: reward = 125.00, steps = 125
    11:37:48 [INFO] train episode 384: reward = 200.00, steps = 200
    11:38:07 [INFO] train episode 385: reward = 200.00, steps = 200
    11:38:25 [INFO] train episode 386: reward = 195.00, steps = 195
    11:38:43 [INFO] train episode 387: reward = 200.00, steps = 200
    11:38:57 [INFO] train episode 388: reward = 154.00, steps = 154
    11:39:15 [INFO] train episode 389: reward = 192.00, steps = 192
    11:39:33 [INFO] train episode 390: reward = 200.00, steps = 200
    11:39:51 [INFO] train episode 391: reward = 200.00, steps = 200
    11:40:08 [INFO] train episode 392: reward = 183.00, steps = 183
    11:40:26 [INFO] train episode 393: reward = 197.00, steps = 197
    11:40:45 [INFO] train episode 394: reward = 200.00, steps = 200
    11:41:03 [INFO] train episode 395: reward = 200.00, steps = 200
    11:41:22 [INFO] train episode 396: reward = 200.00, steps = 200
    11:41:40 [INFO] train episode 397: reward = 200.00, steps = 200
    11:41:52 [INFO] train episode 398: reward = 130.00, steps = 130
    11:42:10 [INFO] train episode 399: reward = 200.00, steps = 200
    11:42:29 [INFO] train episode 400: reward = 200.00, steps = 200
    11:42:35 [INFO] train episode 401: reward = 74.00, steps = 74
    11:42:51 [INFO] train episode 402: reward = 163.00, steps = 163
    11:43:03 [INFO] train episode 403: reward = 135.00, steps = 135
    11:43:21 [INFO] train episode 404: reward = 200.00, steps = 200
    11:43:35 [INFO] train episode 405: reward = 154.00, steps = 154
    11:43:47 [INFO] train episode 406: reward = 124.00, steps = 124
    11:43:57 [INFO] train episode 407: reward = 114.00, steps = 114
    11:44:07 [INFO] train episode 408: reward = 111.00, steps = 111
    11:44:18 [INFO] train episode 409: reward = 117.00, steps = 117
    11:44:31 [INFO] train episode 410: reward = 137.00, steps = 137
    11:44:41 [INFO] train episode 411: reward = 105.00, steps = 105
    11:44:54 [INFO] train episode 412: reward = 137.00, steps = 137
    11:45:08 [INFO] train episode 413: reward = 151.00, steps = 151
    11:45:26 [INFO] train episode 414: reward = 200.00, steps = 200
    11:45:44 [INFO] train episode 415: reward = 200.00, steps = 200
    11:45:55 [INFO] train episode 416: reward = 118.00, steps = 118
    11:46:07 [INFO] train episode 417: reward = 126.00, steps = 126
    11:46:20 [INFO] train episode 418: reward = 136.00, steps = 136
    11:46:28 [INFO] train episode 419: reward = 86.00, steps = 86
    11:46:33 [INFO] train episode 420: reward = 56.00, steps = 56
    11:46:47 [INFO] train episode 421: reward = 150.00, steps = 150
    11:46:51 [INFO] train episode 422: reward = 34.00, steps = 34
    11:46:57 [INFO] train episode 423: reward = 66.00, steps = 66
    11:47:16 [INFO] train episode 424: reward = 200.00, steps = 200
    11:47:33 [INFO] train episode 425: reward = 196.00, steps = 196
    11:47:47 [INFO] train episode 426: reward = 145.00, steps = 145
    11:47:59 [INFO] train episode 427: reward = 131.00, steps = 131
    11:48:09 [INFO] train episode 428: reward = 108.00, steps = 108
    11:48:27 [INFO] train episode 429: reward = 188.00, steps = 188
    11:48:45 [INFO] train episode 430: reward = 200.00, steps = 200
    11:49:00 [INFO] train episode 431: reward = 163.00, steps = 163
    11:49:11 [INFO] train episode 432: reward = 117.00, steps = 117
    11:49:25 [INFO] train episode 433: reward = 151.00, steps = 151
    11:49:43 [INFO] train episode 434: reward = 200.00, steps = 200
    11:49:55 [INFO] train episode 435: reward = 123.00, steps = 123
    11:50:13 [INFO] train episode 436: reward = 199.00, steps = 199
    11:50:31 [INFO] train episode 437: reward = 200.00, steps = 200
    11:50:42 [INFO] train episode 438: reward = 114.00, steps = 114
    11:50:50 [INFO] train episode 439: reward = 90.00, steps = 90
    11:51:09 [INFO] train episode 440: reward = 198.00, steps = 198
    11:51:19 [INFO] train episode 441: reward = 114.00, steps = 114
    11:51:38 [INFO] train episode 442: reward = 200.00, steps = 200
    11:51:56 [INFO] train episode 443: reward = 200.00, steps = 200
    11:52:14 [INFO] train episode 444: reward = 200.00, steps = 200
    11:52:33 [INFO] train episode 445: reward = 200.00, steps = 200
    11:52:51 [INFO] train episode 446: reward = 200.00, steps = 200
    11:53:04 [INFO] train episode 447: reward = 137.00, steps = 137
    11:53:22 [INFO] train episode 448: reward = 200.00, steps = 200
    11:53:40 [INFO] train episode 449: reward = 200.00, steps = 200
    11:53:52 [INFO] train episode 450: reward = 130.00, steps = 130
    11:53:58 [INFO] train episode 451: reward = 57.00, steps = 57
    11:54:16 [INFO] train episode 452: reward = 200.00, steps = 200
    11:54:28 [INFO] train episode 453: reward = 135.00, steps = 135
    11:54:47 [INFO] train episode 454: reward = 200.00, steps = 200
    11:54:58 [INFO] train episode 455: reward = 124.00, steps = 124
    11:55:17 [INFO] train episode 456: reward = 200.00, steps = 200
    11:55:27 [INFO] train episode 457: reward = 110.00, steps = 110
    11:55:45 [INFO] train episode 458: reward = 200.00, steps = 200
    11:56:03 [INFO] train episode 459: reward = 197.00, steps = 197
    11:56:14 [INFO] train episode 460: reward = 110.00, steps = 110
    11:56:31 [INFO] train episode 461: reward = 189.00, steps = 189
    11:56:47 [INFO] train episode 462: reward = 170.00, steps = 170
    11:56:58 [INFO] train episode 463: reward = 118.00, steps = 118
    11:57:16 [INFO] train episode 464: reward = 196.00, steps = 196
    11:57:28 [INFO] train episode 465: reward = 127.00, steps = 127
    11:57:30 [INFO] train episode 466: reward = 22.00, steps = 22
    11:57:48 [INFO] train episode 467: reward = 200.00, steps = 200
    11:58:07 [INFO] train episode 468: reward = 200.00, steps = 200
    11:58:25 [INFO] train episode 469: reward = 200.00, steps = 200
    11:58:33 [INFO] train episode 470: reward = 86.00, steps = 86
    11:58:48 [INFO] train episode 471: reward = 167.00, steps = 167
    11:58:54 [INFO] train episode 472: reward = 59.00, steps = 59
    11:59:10 [INFO] train episode 473: reward = 176.00, steps = 176
    11:59:29 [INFO] train episode 474: reward = 200.00, steps = 200
    11:59:47 [INFO] train episode 475: reward = 200.00, steps = 200
    12:00:05 [INFO] train episode 476: reward = 200.00, steps = 200
    12:00:24 [INFO] train episode 477: reward = 200.00, steps = 200
    12:00:38 [INFO] train episode 478: reward = 153.00, steps = 153
    12:00:56 [INFO] train episode 479: reward = 200.00, steps = 200
    12:01:15 [INFO] train episode 480: reward = 200.00, steps = 200
    12:01:28 [INFO] train episode 481: reward = 138.00, steps = 138
    12:01:42 [INFO] train episode 482: reward = 155.00, steps = 155
    12:02:00 [INFO] train episode 483: reward = 200.00, steps = 200
    12:02:13 [INFO] train episode 484: reward = 128.00, steps = 128
    12:02:31 [INFO] train episode 485: reward = 200.00, steps = 200
    12:02:49 [INFO] train episode 486: reward = 200.00, steps = 200
    12:03:07 [INFO] train episode 487: reward = 200.00, steps = 200
    12:03:22 [INFO] train episode 488: reward = 153.00, steps = 153
    12:03:28 [INFO] train episode 489: reward = 64.00, steps = 64
    12:03:46 [INFO] train episode 490: reward = 200.00, steps = 200
    12:04:05 [INFO] train episode 491: reward = 200.00, steps = 200
    12:04:10 [INFO] train episode 492: reward = 62.00, steps = 62
    12:04:19 [INFO] train episode 493: reward = 87.00, steps = 87
    12:04:32 [INFO] train episode 494: reward = 144.00, steps = 144
    12:04:50 [INFO] train episode 495: reward = 200.00, steps = 200
    12:05:06 [INFO] train episode 496: reward = 168.00, steps = 168
    12:05:24 [INFO] train episode 497: reward = 200.00, steps = 200
    12:05:36 [INFO] train episode 498: reward = 121.00, steps = 121
    12:05:46 [INFO] train episode 499: reward = 112.00, steps = 112
    12:06:00 [INFO] train episode 500: reward = 151.00, steps = 151
    12:06:15 [INFO] train episode 501: reward = 155.00, steps = 155
    12:06:33 [INFO] train episode 502: reward = 200.00, steps = 200
    12:06:46 [INFO] train episode 503: reward = 141.00, steps = 141
    12:07:04 [INFO] train episode 504: reward = 200.00, steps = 200
    12:07:18 [INFO] train episode 505: reward = 150.00, steps = 150
    12:07:31 [INFO] train episode 506: reward = 139.00, steps = 139
    12:07:49 [INFO] train episode 507: reward = 200.00, steps = 200
    12:08:07 [INFO] train episode 508: reward = 200.00, steps = 200
    12:08:19 [INFO] train episode 509: reward = 126.00, steps = 126
    12:08:38 [INFO] train episode 510: reward = 200.00, steps = 200
    12:08:56 [INFO] train episode 511: reward = 200.00, steps = 200
    12:09:14 [INFO] train episode 512: reward = 200.00, steps = 200
    12:09:27 [INFO] train episode 513: reward = 136.00, steps = 136
    12:09:32 [INFO] train episode 514: reward = 57.00, steps = 57
    12:09:43 [INFO] train episode 515: reward = 109.00, steps = 109
    12:10:00 [INFO] train episode 516: reward = 196.00, steps = 196
    12:10:16 [INFO] train episode 517: reward = 166.00, steps = 166
    12:10:34 [INFO] train episode 518: reward = 200.00, steps = 200
    12:10:45 [INFO] train episode 519: reward = 117.00, steps = 117
    12:11:00 [INFO] train episode 520: reward = 163.00, steps = 163
    12:11:15 [INFO] train episode 521: reward = 164.00, steps = 164
    12:11:29 [INFO] train episode 522: reward = 143.00, steps = 143
    12:11:38 [INFO] train episode 523: reward = 105.00, steps = 105
    12:11:56 [INFO] train episode 524: reward = 196.00, steps = 196
    12:12:10 [INFO] train episode 525: reward = 154.00, steps = 154
    12:12:19 [INFO] train episode 526: reward = 99.00, steps = 99
    12:12:31 [INFO] train episode 527: reward = 124.00, steps = 124
    12:12:42 [INFO] train episode 528: reward = 120.00, steps = 120
    12:12:55 [INFO] train episode 529: reward = 140.00, steps = 140
    12:13:11 [INFO] train episode 530: reward = 180.00, steps = 180
    12:13:28 [INFO] train episode 531: reward = 180.00, steps = 180
    12:13:39 [INFO] train episode 532: reward = 115.00, steps = 115
    12:13:49 [INFO] train episode 533: reward = 106.00, steps = 106
    12:14:04 [INFO] train episode 534: reward = 165.00, steps = 165
    12:14:19 [INFO] train episode 535: reward = 159.00, steps = 159
    12:14:24 [INFO] train episode 536: reward = 56.00, steps = 56
    12:14:34 [INFO] train episode 537: reward = 108.00, steps = 108
    12:14:48 [INFO] train episode 538: reward = 152.00, steps = 152
    12:14:59 [INFO] train episode 539: reward = 112.00, steps = 112
    12:15:15 [INFO] train episode 540: reward = 175.00, steps = 175
    12:15:30 [INFO] train episode 541: reward = 166.00, steps = 166
    12:15:44 [INFO] train episode 542: reward = 149.00, steps = 149
    12:15:56 [INFO] train episode 543: reward = 126.00, steps = 126
    12:16:09 [INFO] train episode 544: reward = 145.00, steps = 145
    12:16:27 [INFO] train episode 545: reward = 200.00, steps = 200
    12:16:42 [INFO] train episode 546: reward = 157.00, steps = 157
    12:17:00 [INFO] train episode 547: reward = 200.00, steps = 200
    12:17:12 [INFO] train episode 548: reward = 125.00, steps = 125
    12:17:21 [INFO] train episode 549: reward = 98.00, steps = 98
    12:17:33 [INFO] train episode 550: reward = 131.00, steps = 131
    12:17:50 [INFO] train episode 551: reward = 189.00, steps = 189
    12:18:09 [INFO] train episode 552: reward = 200.00, steps = 200
    12:18:27 [INFO] train episode 553: reward = 200.00, steps = 200
    12:18:42 [INFO] train episode 554: reward = 161.00, steps = 161
    12:19:00 [INFO] train episode 555: reward = 200.00, steps = 200
    12:19:19 [INFO] train episode 556: reward = 200.00, steps = 200
    12:19:34 [INFO] train episode 557: reward = 163.00, steps = 163
    12:19:45 [INFO] train episode 558: reward = 119.00, steps = 119
    12:20:03 [INFO] train episode 559: reward = 200.00, steps = 200
    12:20:21 [INFO] train episode 560: reward = 200.00, steps = 200
    12:20:36 [INFO] train episode 561: reward = 157.00, steps = 157
    12:20:53 [INFO] train episode 562: reward = 187.00, steps = 187
    12:21:10 [INFO] train episode 563: reward = 183.00, steps = 183
    12:21:28 [INFO] train episode 564: reward = 200.00, steps = 200
    12:21:43 [INFO] train episode 565: reward = 157.00, steps = 157
    12:21:57 [INFO] train episode 566: reward = 158.00, steps = 158
    12:22:16 [INFO] train episode 567: reward = 200.00, steps = 200
    12:22:33 [INFO] train episode 568: reward = 183.00, steps = 183
    12:22:51 [INFO] train episode 569: reward = 199.00, steps = 199
    12:23:09 [INFO] train episode 570: reward = 200.00, steps = 200
    12:23:27 [INFO] train episode 571: reward = 200.00, steps = 200
    12:23:44 [INFO] train episode 572: reward = 180.00, steps = 180
    12:23:57 [INFO] train episode 573: reward = 142.00, steps = 142
    12:24:15 [INFO] train episode 574: reward = 200.00, steps = 200
    12:24:33 [INFO] train episode 575: reward = 200.00, steps = 200
    12:24:50 [INFO] train episode 576: reward = 183.00, steps = 183
    12:25:08 [INFO] train episode 577: reward = 200.00, steps = 200
    12:25:20 [INFO] train episode 578: reward = 126.00, steps = 126
    12:25:31 [INFO] train episode 579: reward = 119.00, steps = 119
    12:25:44 [INFO] train episode 580: reward = 137.00, steps = 137
    12:26:02 [INFO] train episode 581: reward = 200.00, steps = 200
    12:26:20 [INFO] train episode 582: reward = 200.00, steps = 200
    12:26:34 [INFO] train episode 583: reward = 154.00, steps = 154
    12:26:53 [INFO] train episode 584: reward = 200.00, steps = 200
    12:27:11 [INFO] train episode 585: reward = 200.00, steps = 200
    12:27:29 [INFO] train episode 586: reward = 200.00, steps = 200
    12:27:46 [INFO] train episode 587: reward = 185.00, steps = 185
    12:28:04 [INFO] train episode 588: reward = 200.00, steps = 200
    12:28:22 [INFO] train episode 589: reward = 200.00, steps = 200
    12:28:31 [INFO] train episode 590: reward = 92.00, steps = 92
    12:28:45 [INFO] train episode 591: reward = 149.00, steps = 149
    12:28:55 [INFO] train episode 592: reward = 111.00, steps = 111
    12:29:13 [INFO] train episode 593: reward = 200.00, steps = 200
    12:29:31 [INFO] train episode 594: reward = 200.00, steps = 200
    12:29:45 [INFO] train episode 595: reward = 151.00, steps = 151
    12:29:59 [INFO] train episode 596: reward = 159.00, steps = 159
    12:30:13 [INFO] train episode 597: reward = 147.00, steps = 147
    12:30:31 [INFO] train episode 598: reward = 200.00, steps = 200
    12:30:49 [INFO] train episode 599: reward = 200.00, steps = 200
    12:31:05 [INFO] train episode 600: reward = 171.00, steps = 171
    12:31:08 [INFO] train episode 601: reward = 27.00, steps = 27
    12:31:26 [INFO] train episode 602: reward = 200.00, steps = 200
    12:31:33 [INFO] train episode 603: reward = 73.00, steps = 73
    12:31:48 [INFO] train episode 604: reward = 166.00, steps = 166
    12:32:02 [INFO] train episode 605: reward = 152.00, steps = 152
    12:32:21 [INFO] train episode 606: reward = 200.00, steps = 200
    12:32:39 [INFO] train episode 607: reward = 200.00, steps = 200
    12:32:57 [INFO] train episode 608: reward = 200.00, steps = 200
    12:33:15 [INFO] train episode 609: reward = 200.00, steps = 200
    12:33:26 [INFO] train episode 610: reward = 119.00, steps = 119
    12:33:45 [INFO] train episode 611: reward = 200.00, steps = 200
    12:33:58 [INFO] train episode 612: reward = 148.00, steps = 148
    12:34:15 [INFO] train episode 613: reward = 182.00, steps = 182
    12:34:33 [INFO] train episode 614: reward = 200.00, steps = 200
    12:34:52 [INFO] train episode 615: reward = 200.00, steps = 200
    12:35:10 [INFO] train episode 616: reward = 200.00, steps = 200
    12:35:24 [INFO] train episode 617: reward = 149.00, steps = 149
    12:35:37 [INFO] train episode 618: reward = 135.00, steps = 135
    12:35:48 [INFO] train episode 619: reward = 128.00, steps = 128
    12:36:01 [INFO] train episode 620: reward = 131.00, steps = 131
    12:36:19 [INFO] train episode 621: reward = 200.00, steps = 200
    12:36:37 [INFO] train episode 622: reward = 200.00, steps = 200
    12:36:57 [INFO] train episode 623: reward = 200.00, steps = 200
    12:37:16 [INFO] train episode 624: reward = 200.00, steps = 200
    12:37:28 [INFO] train episode 625: reward = 133.00, steps = 133
    12:37:46 [INFO] train episode 626: reward = 200.00, steps = 200
    12:38:00 [INFO] train episode 627: reward = 152.00, steps = 152
    12:38:18 [INFO] train episode 628: reward = 200.00, steps = 200
    12:38:29 [INFO] train episode 629: reward = 112.00, steps = 112
    12:38:45 [INFO] train episode 630: reward = 159.00, steps = 159
    12:38:58 [INFO] train episode 631: reward = 140.00, steps = 140
    12:39:16 [INFO] train episode 632: reward = 200.00, steps = 200
    12:39:29 [INFO] train episode 633: reward = 144.00, steps = 144
    12:39:42 [INFO] train episode 634: reward = 140.00, steps = 140
    12:40:01 [INFO] train episode 635: reward = 200.00, steps = 200
    12:40:13 [INFO] train episode 636: reward = 130.00, steps = 130
    12:40:32 [INFO] train episode 637: reward = 200.00, steps = 200
    12:40:47 [INFO] train episode 638: reward = 162.00, steps = 162
    12:41:05 [INFO] train episode 639: reward = 200.00, steps = 200
    12:41:23 [INFO] train episode 640: reward = 200.00, steps = 200
    12:41:38 [INFO] train episode 641: reward = 161.00, steps = 161
    12:41:49 [INFO] train episode 642: reward = 117.00, steps = 117
    12:41:57 [INFO] train episode 643: reward = 75.00, steps = 75
    12:42:16 [INFO] train episode 644: reward = 200.00, steps = 200
    12:42:36 [INFO] train episode 645: reward = 200.00, steps = 200
    12:42:54 [INFO] train episode 646: reward = 200.00, steps = 200
    12:43:13 [INFO] train episode 647: reward = 200.00, steps = 200
    12:43:31 [INFO] train episode 648: reward = 200.00, steps = 200
    12:43:47 [INFO] train episode 649: reward = 170.00, steps = 170
    12:44:01 [INFO] train episode 650: reward = 140.00, steps = 140
    12:44:20 [INFO] train episode 651: reward = 200.00, steps = 200
    12:44:30 [INFO] train episode 652: reward = 99.00, steps = 99
    12:44:44 [INFO] train episode 653: reward = 159.00, steps = 159
    12:45:02 [INFO] train episode 654: reward = 195.00, steps = 195
    12:45:20 [INFO] train episode 655: reward = 200.00, steps = 200
    12:45:31 [INFO] train episode 656: reward = 120.00, steps = 120
    12:45:49 [INFO] train episode 657: reward = 200.00, steps = 200
    12:46:07 [INFO] train episode 658: reward = 200.00, steps = 200
    12:46:21 [INFO] train episode 659: reward = 147.00, steps = 147
    12:46:34 [INFO] train episode 660: reward = 145.00, steps = 145
    12:46:52 [INFO] train episode 661: reward = 200.00, steps = 200
    12:47:11 [INFO] train episode 662: reward = 200.00, steps = 200
    12:47:29 [INFO] train episode 663: reward = 200.00, steps = 200
    12:47:47 [INFO] train episode 664: reward = 200.00, steps = 200
    12:48:03 [INFO] train episode 665: reward = 176.00, steps = 176
    12:48:17 [INFO] train episode 666: reward = 148.00, steps = 148
    12:48:34 [INFO] train episode 667: reward = 190.00, steps = 190
    12:48:45 [INFO] train episode 668: reward = 106.00, steps = 106
    12:49:03 [INFO] train episode 669: reward = 200.00, steps = 200
    12:49:21 [INFO] train episode 670: reward = 200.00, steps = 200
    12:49:39 [INFO] train episode 671: reward = 200.00, steps = 200
    12:49:56 [INFO] train episode 672: reward = 183.00, steps = 183
    12:50:15 [INFO] train episode 673: reward = 200.00, steps = 200
    12:50:29 [INFO] train episode 674: reward = 162.00, steps = 162
    12:50:47 [INFO] train episode 675: reward = 200.00, steps = 200
    12:51:04 [INFO] train episode 676: reward = 188.00, steps = 188
    12:51:19 [INFO] train episode 677: reward = 158.00, steps = 158
    12:51:37 [INFO] train episode 678: reward = 200.00, steps = 200
    12:51:52 [INFO] train episode 679: reward = 162.00, steps = 162
    12:52:10 [INFO] train episode 680: reward = 200.00, steps = 200
    12:52:23 [INFO] train episode 681: reward = 149.00, steps = 149
    12:52:36 [INFO] train episode 682: reward = 140.00, steps = 140
    12:52:54 [INFO] train episode 683: reward = 200.00, steps = 200
    12:53:06 [INFO] train episode 684: reward = 139.00, steps = 139
    12:53:25 [INFO] train episode 685: reward = 200.00, steps = 200
    12:53:42 [INFO] train episode 686: reward = 187.00, steps = 187
    12:54:00 [INFO] train episode 687: reward = 200.00, steps = 200
    12:54:15 [INFO] train episode 688: reward = 172.00, steps = 172
    12:54:33 [INFO] train episode 689: reward = 200.00, steps = 200
    12:54:51 [INFO] train episode 690: reward = 200.00, steps = 200
    12:55:04 [INFO] train episode 691: reward = 142.00, steps = 142
    12:55:21 [INFO] train episode 692: reward = 180.00, steps = 180
    12:55:35 [INFO] train episode 693: reward = 162.00, steps = 162
    12:55:54 [INFO] train episode 694: reward = 200.00, steps = 200
    12:56:11 [INFO] train episode 695: reward = 200.00, steps = 200
    12:56:25 [INFO] train episode 696: reward = 157.00, steps = 157
    12:56:43 [INFO] train episode 697: reward = 200.00, steps = 200
    12:57:01 [INFO] train episode 698: reward = 196.00, steps = 196
    12:57:19 [INFO] train episode 699: reward = 200.00, steps = 200
    12:57:31 [INFO] train episode 700: reward = 139.00, steps = 139
    12:57:48 [INFO] train episode 701: reward = 185.00, steps = 185
    12:58:06 [INFO] train episode 702: reward = 200.00, steps = 200
    12:58:24 [INFO] train episode 703: reward = 200.00, steps = 200
    12:58:36 [INFO] train episode 704: reward = 136.00, steps = 136
    12:58:51 [INFO] train episode 705: reward = 161.00, steps = 161
    12:59:09 [INFO] train episode 706: reward = 200.00, steps = 200
    12:59:27 [INFO] train episode 707: reward = 200.00, steps = 200
    12:59:43 [INFO] train episode 708: reward = 173.00, steps = 173
    13:00:01 [INFO] train episode 709: reward = 200.00, steps = 200
    13:00:20 [INFO] train episode 710: reward = 200.00, steps = 200
    13:00:39 [INFO] train episode 711: reward = 200.00, steps = 200
    13:00:55 [INFO] train episode 712: reward = 162.00, steps = 162
    13:01:09 [INFO] train episode 713: reward = 148.00, steps = 148
    13:01:27 [INFO] train episode 714: reward = 200.00, steps = 200
    13:01:46 [INFO] train episode 715: reward = 200.00, steps = 200
    13:02:05 [INFO] train episode 716: reward = 200.00, steps = 200
    13:02:24 [INFO] train episode 717: reward = 200.00, steps = 200
    13:02:36 [INFO] train episode 718: reward = 123.00, steps = 123
    13:02:52 [INFO] train episode 719: reward = 169.00, steps = 169
    13:03:10 [INFO] train episode 720: reward = 200.00, steps = 200
    13:03:29 [INFO] train episode 721: reward = 200.00, steps = 200
    13:03:47 [INFO] train episode 722: reward = 194.00, steps = 194
    13:04:05 [INFO] train episode 723: reward = 200.00, steps = 200
    13:04:17 [INFO] train episode 724: reward = 129.00, steps = 129
    13:04:32 [INFO] train episode 725: reward = 155.00, steps = 155
    13:04:51 [INFO] train episode 726: reward = 200.00, steps = 200
    13:05:10 [INFO] train episode 727: reward = 200.00, steps = 200
    13:05:23 [INFO] train episode 728: reward = 146.00, steps = 146
    13:05:39 [INFO] train episode 729: reward = 170.00, steps = 170
    13:05:55 [INFO] train episode 730: reward = 165.00, steps = 165
    13:06:14 [INFO] train episode 731: reward = 200.00, steps = 200
    13:06:32 [INFO] train episode 732: reward = 200.00, steps = 200
    13:06:50 [INFO] train episode 733: reward = 200.00, steps = 200
    13:07:09 [INFO] train episode 734: reward = 194.00, steps = 194
    13:07:29 [INFO] train episode 735: reward = 200.00, steps = 200
    13:07:45 [INFO] train episode 736: reward = 163.00, steps = 163
    13:08:02 [INFO] train episode 737: reward = 170.00, steps = 170
    13:08:20 [INFO] train episode 738: reward = 200.00, steps = 200
    13:08:38 [INFO] train episode 739: reward = 198.00, steps = 198
    13:08:56 [INFO] train episode 740: reward = 200.00, steps = 200
    13:09:15 [INFO] train episode 741: reward = 200.00, steps = 200
    13:09:34 [INFO] train episode 742: reward = 200.00, steps = 200
    13:09:53 [INFO] train episode 743: reward = 200.00, steps = 200
    13:10:04 [INFO] train episode 744: reward = 122.00, steps = 122
    13:10:18 [INFO] train episode 745: reward = 145.00, steps = 145
    13:10:36 [INFO] train episode 746: reward = 200.00, steps = 200
    13:10:54 [INFO] train episode 747: reward = 200.00, steps = 200
    13:11:12 [INFO] train episode 748: reward = 200.00, steps = 200
    13:11:31 [INFO] train episode 749: reward = 200.00, steps = 200
    13:11:48 [INFO] train episode 750: reward = 200.00, steps = 200
    13:12:06 [INFO] train episode 751: reward = 200.00, steps = 200
    13:12:25 [INFO] train episode 752: reward = 200.00, steps = 200
    13:12:42 [INFO] train episode 753: reward = 200.00, steps = 200
    13:13:01 [INFO] train episode 754: reward = 200.00, steps = 200
    13:13:19 [INFO] train episode 755: reward = 200.00, steps = 200
    13:13:19 [INFO] ==== test ====
    13:13:38 [INFO] test episode 0: reward = 200.00, steps = 200
    13:13:56 [INFO] test episode 1: reward = 200.00, steps = 200
    13:14:15 [INFO] test episode 2: reward = 200.00, steps = 200
    13:14:34 [INFO] test episode 3: reward = 200.00, steps = 200
    13:14:52 [INFO] test episode 4: reward = 200.00, steps = 200
    13:15:10 [INFO] test episode 5: reward = 200.00, steps = 200
    13:15:29 [INFO] test episode 6: reward = 200.00, steps = 200
    13:15:47 [INFO] test episode 7: reward = 200.00, steps = 200
    13:16:05 [INFO] test episode 8: reward = 200.00, steps = 200
    13:16:24 [INFO] test episode 9: reward = 200.00, steps = 200
    13:16:42 [INFO] test episode 10: reward = 200.00, steps = 200
    13:17:00 [INFO] test episode 11: reward = 200.00, steps = 200
    13:17:19 [INFO] test episode 12: reward = 200.00, steps = 200
    13:17:35 [INFO] test episode 13: reward = 179.00, steps = 179
    13:17:46 [INFO] test episode 14: reward = 119.00, steps = 119
    13:18:05 [INFO] test episode 15: reward = 200.00, steps = 200
    13:18:24 [INFO] test episode 16: reward = 200.00, steps = 200
    13:18:43 [INFO] test episode 17: reward = 200.00, steps = 200
    13:18:56 [INFO] test episode 18: reward = 140.00, steps = 140
    13:19:16 [INFO] test episode 19: reward = 200.00, steps = 200
    13:19:24 [INFO] test episode 20: reward = 86.00, steps = 86
    13:19:43 [INFO] test episode 21: reward = 200.00, steps = 200
    13:20:04 [INFO] test episode 22: reward = 200.00, steps = 200
    13:20:24 [INFO] test episode 23: reward = 200.00, steps = 200
    13:20:44 [INFO] test episode 24: reward = 200.00, steps = 200
    13:21:04 [INFO] test episode 25: reward = 200.00, steps = 200
    13:21:24 [INFO] test episode 26: reward = 200.00, steps = 200
    13:21:44 [INFO] test episode 27: reward = 200.00, steps = 200
    13:22:03 [INFO] test episode 28: reward = 200.00, steps = 200
    13:22:23 [INFO] test episode 29: reward = 200.00, steps = 200
    13:22:42 [INFO] test episode 30: reward = 200.00, steps = 200
    13:23:02 [INFO] test episode 31: reward = 200.00, steps = 200
    13:23:21 [INFO] test episode 32: reward = 200.00, steps = 200
    13:23:41 [INFO] test episode 33: reward = 200.00, steps = 200
    13:24:00 [INFO] test episode 34: reward = 200.00, steps = 200
    13:24:20 [INFO] test episode 35: reward = 200.00, steps = 200
    13:24:39 [INFO] test episode 36: reward = 199.00, steps = 199
    13:24:56 [INFO] test episode 37: reward = 182.00, steps = 182
    13:25:15 [INFO] test episode 38: reward = 200.00, steps = 200
    13:25:34 [INFO] test episode 39: reward = 200.00, steps = 200
    13:25:53 [INFO] test episode 40: reward = 200.00, steps = 200
    13:26:13 [INFO] test episode 41: reward = 200.00, steps = 200
    13:26:32 [INFO] test episode 42: reward = 200.00, steps = 200
    13:26:51 [INFO] test episode 43: reward = 200.00, steps = 200
    13:27:04 [INFO] test episode 44: reward = 131.00, steps = 131
    13:27:23 [INFO] test episode 45: reward = 200.00, steps = 200
    13:27:35 [INFO] test episode 46: reward = 121.00, steps = 121
    13:27:54 [INFO] test episode 47: reward = 200.00, steps = 200
    13:28:13 [INFO] test episode 48: reward = 200.00, steps = 200
    13:28:32 [INFO] test episode 49: reward = 200.00, steps = 200
    13:28:52 [INFO] test episode 50: reward = 200.00, steps = 200
    13:29:11 [INFO] test episode 51: reward = 200.00, steps = 200
    13:29:30 [INFO] test episode 52: reward = 200.00, steps = 200
    13:29:44 [INFO] test episode 53: reward = 141.00, steps = 141
    13:30:03 [INFO] test episode 54: reward = 200.00, steps = 200
    13:30:23 [INFO] test episode 55: reward = 200.00, steps = 200
    13:30:42 [INFO] test episode 56: reward = 200.00, steps = 200
    13:31:01 [INFO] test episode 57: reward = 200.00, steps = 200
    13:31:20 [INFO] test episode 58: reward = 200.00, steps = 200
    13:31:39 [INFO] test episode 59: reward = 200.00, steps = 200
    13:31:58 [INFO] test episode 60: reward = 200.00, steps = 200
    13:32:18 [INFO] test episode 61: reward = 200.00, steps = 200
    13:32:37 [INFO] test episode 62: reward = 200.00, steps = 200
    13:32:56 [INFO] test episode 63: reward = 200.00, steps = 200
    13:33:16 [INFO] test episode 64: reward = 200.00, steps = 200
    13:33:32 [INFO] test episode 65: reward = 159.00, steps = 159
    13:33:51 [INFO] test episode 66: reward = 200.00, steps = 200
    13:34:11 [INFO] test episode 67: reward = 200.00, steps = 200
    13:34:30 [INFO] test episode 68: reward = 200.00, steps = 200
    13:34:49 [INFO] test episode 69: reward = 200.00, steps = 200
    13:35:09 [INFO] test episode 70: reward = 200.00, steps = 200
    13:35:30 [INFO] test episode 71: reward = 200.00, steps = 200
    13:35:50 [INFO] test episode 72: reward = 200.00, steps = 200
    13:36:10 [INFO] test episode 73: reward = 200.00, steps = 200
    13:36:30 [INFO] test episode 74: reward = 200.00, steps = 200
    13:36:49 [INFO] test episode 75: reward = 200.00, steps = 200
    13:37:09 [INFO] test episode 76: reward = 200.00, steps = 200
    13:37:28 [INFO] test episode 77: reward = 200.00, steps = 200
    13:37:48 [INFO] test episode 78: reward = 200.00, steps = 200
    13:38:08 [INFO] test episode 79: reward = 200.00, steps = 200
    13:38:27 [INFO] test episode 80: reward = 200.00, steps = 200
    13:38:47 [INFO] test episode 81: reward = 200.00, steps = 200
    13:39:06 [INFO] test episode 82: reward = 200.00, steps = 200
    13:39:26 [INFO] test episode 83: reward = 200.00, steps = 200
    13:39:45 [INFO] test episode 84: reward = 200.00, steps = 200
    13:40:05 [INFO] test episode 85: reward = 200.00, steps = 200
    13:40:24 [INFO] test episode 86: reward = 200.00, steps = 200
    13:40:43 [INFO] test episode 87: reward = 200.00, steps = 200
    13:41:01 [INFO] test episode 88: reward = 200.00, steps = 200
    13:41:18 [INFO] test episode 89: reward = 170.00, steps = 170
    13:41:38 [INFO] test episode 90: reward = 200.00, steps = 200
    13:41:58 [INFO] test episode 91: reward = 200.00, steps = 200
    13:42:17 [INFO] test episode 92: reward = 200.00, steps = 200
    13:42:30 [INFO] test episode 93: reward = 141.00, steps = 141
    13:42:49 [INFO] test episode 94: reward = 200.00, steps = 200
    13:43:08 [INFO] test episode 95: reward = 200.00, steps = 200
    13:43:28 [INFO] test episode 96: reward = 200.00, steps = 200
    13:43:44 [INFO] test episode 97: reward = 162.00, steps = 162
    13:44:03 [INFO] test episode 98: reward = 200.00, steps = 200
    13:44:22 [INFO] test episode 99: reward = 200.00, steps = 200
    13:44:22 [INFO] average episode reward = 193.30 ± 20.35

In [5]:

    env.close()