# Use Off-Policy VPG to Play CartPole-v0¶

TensorFlow version

In [1]:

    %matplotlib inline

    import sys
    import logging
    import itertools

    import numpy as np
    np.random.seed(0)
    import pandas as pd
    import gym
    import matplotlib.pyplot as plt
    import tensorflow.compat.v2 as tf
    tf.random.set_seed(0)
    from tensorflow import keras
    from tensorflow import nn
    from tensorflow import optimizers
    from tensorflow.keras import layers
    from tensorflow.keras import losses

    logging.basicConfig(level=logging.INFO,
            format='%(asctime)s [%(levelname)s] %(message)s',
            stream=sys.stdout, datefmt='%H:%M:%S')

In [2]:

    env = gym.make('CartPole-v0')
    for key in vars(env):
        logging.info('%s: %s', key, vars(env)[key])
    for key in vars(env.spec):
        logging.info('%s: %s', key, vars(env.spec)[key])

    11:29:13 [INFO] env: <CartPoleEnv<CartPole-v0>>
    11:29:13 [INFO] action_space: Discrete(2)
    11:29:13 [INFO] observation_space: Box(-3.4028234663852886e+38, 3.4028234663852886e+38, (4,), float32)
    11:29:13 [INFO] reward_range: (-inf, inf)
    11:29:13 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 50}
    11:29:13 [INFO] _max_episode_steps: 200
    11:29:13 [INFO] _elapsed_steps: None
    11:29:13 [INFO] id: CartPole-v0
    11:29:13 [INFO] entry_point: gym.envs.classic_control:CartPoleEnv
    11:29:13 [INFO] reward_threshold: 195.0
    11:29:13 [INFO] nondeterministic: False
    11:29:13 [INFO] max_episode_steps: 200
    11:29:13 [INFO] _kwargs: {}
    11:29:13 [INFO] _env_name: CartPole

In [3]:

    class OffPolicyVPGAgent:
        def __init__(self, env):
            self.action_n = env.action_space.n
            self.gamma = 0.99

            def dot(y_true, y_pred):
                return -tf.reduce_sum(y_true * y_pred, axis=-1)

            self.policy_net = self.build_net(hidden_sizes=[],
                    output_size=self.action_n,
                    output_activation=nn.softmax,
                    loss=dot, learning_rate=0.06)

        def build_net(self, hidden_sizes, output_size,
                activation=nn.relu, output_activation=None,
                use_bias=False, loss=losses.mse, learning_rate=0.001):
            model = keras.Sequential()
            for hidden_size in hidden_sizes:
                model.add(layers.Dense(units=hidden_size,
                        activation=activation, use_bias=use_bias))
            model.add(layers.Dense(units=output_size,
                    activation=output_activation, use_bias=use_bias))
            optimizer = optimizers.Adam(learning_rate)
            model.compile(optimizer=optimizer, loss=loss)
            return model

        def reset(self, mode=None):
            self.mode = mode
            if self.mode == 'train':
                self.trajectory = []

        def step(self, observation, reward, terminated):
            if self.mode == 'train':
                action = np.random.choice(self.action_n)  # use random policy
                self.trajectory += [observation, reward, terminated, action]
            else:
                probs = self.policy_net.predict(observation[np.newaxis], verbose=0)[0]
                action = np.random.choice(self.action_n, p=probs)
            return action

        def close(self):
            if self.mode == 'train':
                self.learn()

        def learn(self):
            df = pd.DataFrame(np.array(self.trajectory, dtype=object).reshape(-1, 4),
                    columns=['state', 'reward', 'terminated', 'action'])
            df['discount'] = self.gamma ** df.index.to_series()
            df['discounted_reward'] = df['discount'] * df['reward'].astype(float)
            df['discounted_return'] = df['discounted_reward'][::-1].cumsum()
            states = np.stack(df['state'])
            actions = np.eye(self.action_n)[df['action'].astype(int)]
            df['behavior_prob'] = 1. / self.action_n
            df['sample_weight'] = df['discounted_return'] / df['behavior_prob']
            sample_weight = df[['sample_weight',]].values
            self.policy_net.fit(states, actions, sample_weight=sample_weight,
                    verbose=0)

    agent = OffPolicyVPGAgent(env)

In [4]:

    def play_episode(env, agent, seed=None, mode=None, render=False):
        observation, _ = env.reset(seed=seed)
        reward, terminated, truncated = 0., False, False
        agent.reset(mode=mode)
        episode_reward, elapsed_steps = 0., 0
        while True:
            action = agent.step(observation, reward, terminated)
            if render:
                env.render()
            if terminated or truncated:
                break
            observation, reward, terminated, truncated, _ = env.step(action)
            episode_reward += reward
            elapsed_steps += 1
        agent.close()
        return episode_reward, elapsed_steps

    logging.info('==== train ====')
    episode_rewards = []
    for episode in itertools.count():
        play_episode(env, agent, seed=episode,
                mode='train')
        episode_reward, elapsed_steps = play_episode(env, agent)
        episode_rewards.append(episode_reward)
        logging.info('train episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
        if np.mean(episode_rewards[-10:]) > 195:
            break
    plt.plot(episode_rewards)

    logging.info('==== test ====')
    episode_rewards = []
    for episode in range(100):
        episode_reward, elapsed_steps = play_episode(env, agent)
        episode_rewards.append(episode_reward)
        logging.info('test episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
    logging.info('average episode reward = %.2f ± %.2f',
            np.mean(episode_rewards), np.std(episode_rewards))

    11:29:14 [INFO] ==== train ====
    11:29:15 [INFO] NumExpr defaulting to 4 threads.
    11:29:20 [INFO] train episode 0: reward = 34.00, steps = 34
    11:29:21 [INFO] train episode 1: reward = 16.00, steps = 16
    11:29:22 [INFO] train episode 2: reward = 20.00, steps = 20
    11:29:23 [INFO] train episode 3: reward = 17.00, steps = 17
    11:29:25 [INFO] train episode 4: reward = 26.00, steps = 26
    11:29:25 [INFO] train episode 5: reward = 12.00, steps = 12
    11:29:27 [INFO] train episode 6: reward = 25.00, steps = 25
    11:29:28 [INFO] train episode 7: reward = 19.00, steps = 19
    11:29:30 [INFO] train episode 8: reward = 37.00, steps = 37
    11:29:31 [INFO] train episode 9: reward = 27.00, steps = 27
    11:29:32 [INFO] train episode 10: reward = 16.00, steps = 16
    11:29:33 [INFO] train episode 11: reward = 23.00, steps = 23
    11:29:34 [INFO] train episode 12: reward = 12.00, steps = 12
    11:29:35 [INFO] train episode 13: reward = 24.00, steps = 24
    11:29:36 [INFO] train episode 14: reward = 10.00, steps = 10
    11:29:37 [INFO] train episode 15: reward = 25.00, steps = 25
    11:29:38 [INFO] train episode 16: reward = 25.00, steps = 25
    11:29:40 [INFO] train episode 17: reward = 34.00, steps = 34
    11:29:41 [INFO] train episode 18: reward = 13.00, steps = 13
    11:29:42 [INFO] train episode 19: reward = 17.00, steps = 17
    11:29:43 [INFO] train episode 20: reward = 11.00, steps = 11
    11:29:44 [INFO] train episode 21: reward = 19.00, steps = 19
    11:29:48 [INFO] train episode 22: reward = 75.00, steps = 75
    11:29:49 [INFO] train episode 23: reward = 20.00, steps = 20
    11:29:52 [INFO] train episode 24: reward = 55.00, steps = 55
    11:29:53 [INFO] train episode 25: reward = 19.00, steps = 19
    11:29:56 [INFO] train episode 26: reward = 70.00, steps = 70
    11:29:56 [INFO] train episode 27: reward = 11.00, steps = 11
    11:29:58 [INFO] train episode 28: reward = 24.00, steps = 24
    11:30:02 [INFO] train episode 29: reward = 81.00, steps = 81
    11:30:05 [INFO] train episode 30: reward = 68.00, steps = 68
    11:30:10 [INFO] train episode 31: reward = 111.00, steps = 111
    11:30:12 [INFO] train episode 32: reward = 44.00, steps = 44
    11:30:14 [INFO] train episode 33: reward = 42.00, steps = 42
    11:30:15 [INFO] train episode 34: reward = 24.00, steps = 24
    11:30:18 [INFO] train episode 35: reward = 65.00, steps = 65
    11:30:20 [INFO] train episode 36: reward = 46.00, steps = 46
    11:30:23 [INFO] train episode 37: reward = 59.00, steps = 59
    11:30:27 [INFO] train episode 38: reward = 96.00, steps = 96
    11:30:30 [INFO] train episode 39: reward = 61.00, steps = 61
    11:30:33 [INFO] train episode 40: reward = 49.00, steps = 49
    11:30:37 [INFO] train episode 41: reward = 105.00, steps = 105
    11:30:41 [INFO] train episode 42: reward = 83.00, steps = 83
    11:30:46 [INFO] train episode 43: reward = 102.00, steps = 102
    11:30:48 [INFO] train episode 44: reward = 41.00, steps = 41
    11:30:51 [INFO] train episode 45: reward = 58.00, steps = 58
    11:30:53 [INFO] train episode 46: reward = 55.00, steps = 55
    11:30:56 [INFO] train episode 47: reward = 51.00, steps = 51
    11:30:58 [INFO] train episode 48: reward = 55.00, steps = 55
    11:31:03 [INFO] train episode 49: reward = 102.00, steps = 102
    11:31:06 [INFO] train episode 50: reward = 58.00, steps = 58
    11:31:09 [INFO] train episode 51: reward = 81.00, steps = 81
    11:31:11 [INFO] train episode 52: reward = 24.00, steps = 24
    11:31:13 [INFO] train episode 53: reward = 50.00, steps = 50
    11:31:16 [INFO] train episode 54: reward = 76.00, steps = 76
    11:31:19 [INFO] train episode 55: reward = 62.00, steps = 62
    11:31:22 [INFO] train episode 56: reward = 51.00, steps = 51
    11:31:26 [INFO] train episode 57: reward = 88.00, steps = 88
    11:31:28 [INFO] train episode 58: reward = 40.00, steps = 40
    11:31:31 [INFO] train episode 59: reward = 71.00, steps = 71
    11:31:34 [INFO] train episode 60: reward = 70.00, steps = 70
    11:31:36 [INFO] train episode 61: reward = 39.00, steps = 39
    11:31:38 [INFO] train episode 62: reward = 51.00, steps = 51
    11:31:41 [INFO] train episode 63: reward = 56.00, steps = 56
    11:31:44 [INFO] train episode 64: reward = 54.00, steps = 54
    11:31:47 [INFO] train episode 65: reward = 70.00, steps = 70
    11:31:50 [INFO] train episode 66: reward = 68.00, steps = 68
    11:31:53 [INFO] train episode 67: reward = 61.00, steps = 61
    11:31:55 [INFO] train episode 68: reward = 43.00, steps = 43
    11:31:57 [INFO] train episode 69: reward = 51.00, steps = 51
    11:32:00 [INFO] train episode 70: reward = 58.00, steps = 58
    11:32:03 [INFO] train episode 71: reward = 64.00, steps = 64
    11:32:05 [INFO] train episode 72: reward = 38.00, steps = 38
    11:32:07 [INFO] train episode 73: reward = 34.00, steps = 34
    11:32:11 [INFO] train episode 74: reward = 88.00, steps = 88
    11:32:16 [INFO] train episode 75: reward = 125.00, steps = 125
    11:32:21 [INFO] train episode 76: reward = 92.00, steps = 92
    11:32:25 [INFO] train episode 77: reward = 103.00, steps = 103
    11:32:29 [INFO] train episode 78: reward = 82.00, steps = 82
    11:32:31 [INFO] train episode 79: reward = 37.00, steps = 37
    11:32:33 [INFO] train episode 80: reward = 38.00, steps = 38
    11:32:36 [INFO] train episode 81: reward = 61.00, steps = 61
    11:32:38 [INFO] train episode 82: reward = 60.00, steps = 60
    11:32:41 [INFO] train episode 83: reward = 63.00, steps = 63
    11:32:45 [INFO] train episode 84: reward = 81.00, steps = 81
    11:32:48 [INFO] train episode 85: reward = 54.00, steps = 54
    11:32:50 [INFO] train episode 86: reward = 50.00, steps = 50
    11:32:53 [INFO] train episode 87: reward = 72.00, steps = 72
    11:32:56 [INFO] train episode 88: reward = 59.00, steps = 59
    11:33:01 [INFO] train episode 89: reward = 96.00, steps = 96
    11:33:03 [INFO] train episode 90: reward = 48.00, steps = 48
    11:33:07 [INFO] train episode 91: reward = 83.00, steps = 83
    11:33:09 [INFO] train episode 92: reward = 50.00, steps = 50
    11:33:12 [INFO] train episode 93: reward = 54.00, steps = 54
    11:33:14 [INFO] train episode 94: reward = 47.00, steps = 47
    11:33:16 [INFO] train episode 95: reward = 45.00, steps = 45
    11:33:19 [INFO] train episode 96: reward = 68.00, steps = 68
    11:33:22 [INFO] train episode 97: reward = 66.00, steps = 66
    11:33:25 [INFO] train episode 98: reward = 53.00, steps = 53
    11:33:28 [INFO] train episode 99: reward = 58.00, steps = 58
    11:33:32 [INFO] train episode 100: reward = 90.00, steps = 90
    11:33:35 [INFO] train episode 101: reward = 74.00, steps = 74
    11:33:38 [INFO] train episode 102: reward = 61.00, steps = 61
    11:33:41 [INFO] train episode 103: reward = 50.00, steps = 50
    11:33:43 [INFO] train episode 104: reward = 51.00, steps = 51
    11:33:46 [INFO] train episode 105: reward = 58.00, steps = 58
    11:33:50 [INFO] train episode 106: reward = 100.00, steps = 100
    11:33:53 [INFO] train episode 107: reward = 49.00, steps = 49
    11:33:55 [INFO] train episode 108: reward = 50.00, steps = 50
    11:33:57 [INFO] train episode 109: reward = 32.00, steps = 32
    11:34:01 [INFO] train episode 110: reward = 81.00, steps = 81
    11:34:04 [INFO] train episode 111: reward = 75.00, steps = 75
    11:34:08 [INFO] train episode 112: reward = 76.00, steps = 76
    11:34:11 [INFO] train episode 113: reward = 71.00, steps = 71
    11:34:14 [INFO] train episode 114: reward = 72.00, steps = 72
    11:34:17 [INFO] train episode 115: reward = 63.00, steps = 63
    11:34:21 [INFO] train episode 116: reward = 87.00, steps = 87
    11:34:26 [INFO] train episode 117: reward = 104.00, steps = 104
    11:34:30 [INFO] train episode 118: reward = 91.00, steps = 91
    11:34:33 [INFO] train episode 119: reward = 53.00, steps = 53
    11:34:36 [INFO] train episode 120: reward = 61.00, steps = 61
    11:34:39 [INFO] train episode 121: reward = 61.00, steps = 61
    11:34:41 [INFO] train episode 122: reward = 57.00, steps = 57
    11:34:46 [INFO] train episode 123: reward = 97.00, steps = 97
    11:34:50 [INFO] train episode 124: reward = 70.00, steps = 70
    11:34:52 [INFO] train episode 125: reward = 41.00, steps = 41
    11:34:55 [INFO] train episode 126: reward = 66.00, steps = 66
    11:34:58 [INFO] train episode 127: reward = 74.00, steps = 74
    11:35:02 [INFO] train episode 128: reward = 84.00, steps = 84
    11:35:05 [INFO] train episode 129: reward = 59.00, steps = 59
    11:35:07 [INFO] train episode 130: reward = 53.00, steps = 53
    11:35:14 [INFO] train episode 131: reward = 145.00, steps = 145
    11:35:18 [INFO] train episode 132: reward = 93.00, steps = 93
    11:35:22 [INFO] train episode 133: reward = 85.00, steps = 85
    11:35:26 [INFO] train episode 134: reward = 81.00, steps = 81
    11:35:28 [INFO] train episode 135: reward = 40.00, steps = 40
    11:35:31 [INFO] train episode 136: reward = 56.00, steps = 56
    11:35:33 [INFO] train episode 137: reward = 59.00, steps = 59
    11:35:36 [INFO] train episode 138: reward = 52.00, steps = 52
    11:35:38 [INFO] train episode 139: reward = 39.00, steps = 39
    11:35:45 [INFO] train episode 140: reward = 143.00, steps = 143
    11:35:50 [INFO] train episode 141: reward = 104.00, steps = 104
    11:35:53 [INFO] train episode 142: reward = 65.00, steps = 65
    11:35:55 [INFO] train episode 143: reward = 50.00, steps = 50
    11:35:58 [INFO] train episode 144: reward = 72.00, steps = 72
    11:36:02 [INFO] train episode 145: reward = 67.00, steps = 67
    11:36:07 [INFO] train episode 146: reward = 127.00, steps = 127
    11:36:16 [INFO] train episode 147: reward = 188.00, steps = 188
    11:36:20 [INFO] train episode 148: reward = 90.00, steps = 90
    11:36:23 [INFO] train episode 149: reward = 66.00, steps = 66
    11:36:26 [INFO] train episode 150: reward = 71.00, steps = 71
    11:36:30 [INFO] train episode 151: reward = 71.00, steps = 71
    11:36:33 [INFO] train episode 152: reward = 81.00, steps = 81
    11:36:37 [INFO] train episode 153: reward = 72.00, steps = 72
    11:36:40 [INFO] train episode 154: reward = 68.00, steps = 68
    11:36:46 [INFO] train episode 155: reward = 126.00, steps = 126
    11:36:47 [INFO] train episode 156: reward = 38.00, steps = 38
    11:36:50 [INFO] train episode 157: reward = 60.00, steps = 60
    11:36:53 [INFO] train episode 158: reward = 68.00, steps = 68
    11:36:58 [INFO] train episode 159: reward = 89.00, steps = 89
    11:37:02 [INFO] train episode 160: reward = 107.00, steps = 107
    11:37:08 [INFO] train episode 161: reward = 122.00, steps = 122
    11:37:10 [INFO] train episode 162: reward = 49.00, steps = 49
    11:37:14 [INFO] train episode 163: reward = 71.00, steps = 71
    11:37:16 [INFO] train episode 164: reward = 48.00, steps = 48
    11:37:21 [INFO] train episode 165: reward = 125.00, steps = 125
    11:37:26 [INFO] train episode 166: reward = 110.00, steps = 110
    11:37:30 [INFO] train episode 167: reward = 85.00, steps = 85
    11:37:34 [INFO] train episode 168: reward = 86.00, steps = 86
    11:37:38 [INFO] train episode 169: reward = 75.00, steps = 75
    11:37:41 [INFO] train episode 170: reward = 63.00, steps = 63
    11:37:44 [INFO] train episode 171: reward = 74.00, steps = 74
    11:37:48 [INFO] train episode 172: reward = 78.00, steps = 78
    11:37:51 [INFO] train episode 173: reward = 64.00, steps = 64
    11:37:52 [INFO] train episode 174: reward = 39.00, steps = 39
    11:37:56 [INFO] train episode 175: reward = 69.00, steps = 69
    11:37:59 [INFO] train episode 176: reward = 73.00, steps = 73
    11:38:03 [INFO] train episode 177: reward = 80.00, steps = 80
    11:38:05 [INFO] train episode 178: reward = 56.00, steps = 56
    11:38:07 [INFO] train episode 179: reward = 43.00, steps = 43
    11:38:12 [INFO] train episode 180: reward = 100.00, steps = 100
    11:38:15 [INFO] train episode 181: reward = 67.00, steps = 67
    11:38:17 [INFO] train episode 182: reward = 46.00, steps = 46
    11:38:21 [INFO] train episode 183: reward = 77.00, steps = 77
    11:38:24 [INFO] train episode 184: reward = 70.00, steps = 70
    11:38:27 [INFO] train episode 185: reward = 58.00, steps = 58
    11:38:29 [INFO] train episode 186: reward = 55.00, steps = 55
    11:38:34 [INFO] train episode 187: reward = 96.00, steps = 96
    11:38:40 [INFO] train episode 188: reward = 138.00, steps = 138
    11:38:42 [INFO] train episode 189: reward = 52.00, steps = 52
    11:38:47 [INFO] train episode 190: reward = 91.00, steps = 91
    11:38:50 [INFO] train episode 191: reward = 70.00, steps = 70
    11:38:53 [INFO] train episode 192: reward = 73.00, steps = 73
    11:38:57 [INFO] train episode 193: reward = 81.00, steps = 81
    11:39:03 [INFO] train episode 194: reward = 139.00, steps = 139
    11:39:06 [INFO] train episode 195: reward = 63.00, steps = 63
    11:39:13 [INFO] train episode 196: reward = 132.00, steps = 132
    11:39:17 [INFO] train episode 197: reward = 82.00, steps = 82
    11:39:20 [INFO] train episode 198: reward = 81.00, steps = 81
    11:39:24 [INFO] train episode 199: reward = 78.00, steps = 78
    11:39:28 [INFO] train episode 200: reward = 93.00, steps = 93
    11:39:31 [INFO] train episode 201: reward = 70.00, steps = 70
    11:39:34 [INFO] train episode 202: reward = 63.00, steps = 63
    11:39:39 [INFO] train episode 203: reward = 95.00, steps = 95
    11:39:42 [INFO] train episode 204: reward = 70.00, steps = 70
    11:39:47 [INFO] train episode 205: reward = 104.00, steps = 104
    11:39:50 [INFO] train episode 206: reward = 66.00, steps = 66
    11:39:57 [INFO] train episode 207: reward = 160.00, steps = 160
    11:40:01 [INFO] train episode 208: reward = 95.00, steps = 95
    11:40:04 [INFO] train episode 209: reward = 54.00, steps = 54
    11:40:06 [INFO] train episode 210: reward = 53.00, steps = 53
    11:40:09 [INFO] train episode 211: reward = 58.00, steps = 58
    11:40:13 [INFO] train episode 212: reward = 95.00, steps = 95
    11:40:18 [INFO] train episode 213: reward = 89.00, steps = 89
    11:40:23 [INFO] train episode 214: reward = 115.00, steps = 115
    11:40:28 [INFO] train episode 215: reward = 112.00, steps = 112
    11:40:32 [INFO] train episode 216: reward = 92.00, steps = 92
    11:40:37 [INFO] train episode 217: reward = 104.00, steps = 104
    11:40:40 [INFO] train episode 218: reward = 71.00, steps = 71
    11:40:45 [INFO] train episode 219: reward = 104.00, steps = 104
    11:40:48 [INFO] train episode 220: reward = 67.00, steps = 67
    11:40:51 [INFO] train episode 221: reward = 68.00, steps = 68
    11:40:56 [INFO] train episode 222: reward = 100.00, steps = 100
    11:40:59 [INFO] train episode 223: reward = 63.00, steps = 63
    11:41:01 [INFO] train episode 224: reward = 44.00, steps = 44
    11:41:03 [INFO] train episode 225: reward = 51.00, steps = 51
    11:41:06 [INFO] train episode 226: reward = 63.00, steps = 63
    11:41:10 [INFO] train episode 227: reward = 83.00, steps = 83
    11:41:16 [INFO] train episode 228: reward = 122.00, steps = 122
    11:41:18 [INFO] train episode 229: reward = 60.00, steps = 60
    11:41:21 [INFO] train episode 230: reward = 66.00, steps = 66
    11:41:25 [INFO] train episode 231: reward = 82.00, steps = 82
    11:41:29 [INFO] train episode 232: reward = 75.00, steps = 75
    11:41:32 [INFO] train episode 233: reward = 67.00, steps = 67
    11:41:35 [INFO] train episode 234: reward = 77.00, steps = 77
    11:41:39 [INFO] train episode 235: reward = 74.00, steps = 74
    11:41:44 [INFO] train episode 236: reward = 113.00, steps = 113
    11:41:47 [INFO] train episode 237: reward = 66.00, steps = 66
    11:41:51 [INFO] train episode 238: reward = 83.00, steps = 83
    11:41:54 [INFO] train episode 239: reward = 66.00, steps = 66
    11:41:57 [INFO] train episode 240: reward = 69.00, steps = 69
    11:42:02 [INFO] train episode 241: reward = 101.00, steps = 101
    11:42:08 [INFO] train episode 242: reward = 133.00, steps = 133
    11:42:11 [INFO] train episode 243: reward = 72.00, steps = 72
    11:42:17 [INFO] train episode 244: reward = 120.00, steps = 120
    11:42:21 [INFO] train episode 245: reward = 104.00, steps = 104
    11:42:26 [INFO] train episode 246: reward = 97.00, steps = 97
    11:42:29 [INFO] train episode 247: reward = 73.00, steps = 73
    11:42:32 [INFO] train episode 248: reward = 55.00, steps = 55
    11:42:34 [INFO] train episode 249: reward = 62.00, steps = 62
    11:42:39 [INFO] train episode 250: reward = 88.00, steps = 88
    11:42:42 [INFO] train episode 251: reward = 73.00, steps = 73
    11:42:44 [INFO] train episode 252: reward = 50.00, steps = 50
    11:42:48 [INFO] train episode 253: reward = 79.00, steps = 79
    11:42:52 [INFO] train episode 254: reward = 88.00, steps = 88
    11:42:57 [INFO] train episode 255: reward = 100.00, steps = 100
    11:43:04 [INFO] train episode 256: reward = 174.00, steps = 174
    11:43:10 [INFO] train episode 257: reward = 121.00, steps = 121
    11:43:14 [INFO] train episode 258: reward = 75.00, steps = 75
    11:43:16 [INFO] train episode 259: reward = 42.00, steps = 42
    11:43:17 [INFO] train episode 260: reward = 41.00, steps = 41
    11:43:23 [INFO] train episode 261: reward = 123.00, steps = 123
    11:43:26 [INFO] train episode 262: reward = 58.00, steps = 58
    11:43:28 [INFO] train episode 263: reward = 58.00, steps = 58
    11:43:33 [INFO] train episode 264: reward = 105.00, steps = 105
    11:43:36 [INFO] train episode 265: reward = 61.00, steps = 61
    11:43:39 [INFO] train episode 266: reward = 63.00, steps = 63
    11:43:42 [INFO] train episode 267: reward = 74.00, steps = 74
    11:43:46 [INFO] train episode 268: reward = 71.00, steps = 71
    11:43:48 [INFO] train episode 269: reward = 56.00, steps = 56
    11:43:53 [INFO] train episode 270: reward = 113.00, steps = 113
    11:43:56 [INFO] train episode 271: reward = 60.00, steps = 60
    11:44:00 [INFO] train episode 272: reward = 77.00, steps = 77
    11:44:03 [INFO] train episode 273: reward = 67.00, steps = 67
    11:44:07 [INFO] train episode 274: reward = 78.00, steps = 78
    11:44:10 [INFO] train episode 275: reward = 68.00, steps = 68
    11:44:14 [INFO] train episode 276: reward = 91.00, steps = 91
    11:44:18 [INFO] train episode 277: reward = 84.00, steps = 84
    11:44:23 [INFO] train episode 278: reward = 112.00, steps = 112
    11:44:26 [INFO] train episode 279: reward = 74.00, steps = 74
    11:44:31 [INFO] train episode 280: reward = 94.00, steps = 94
    11:44:34 [INFO] train episode 281: reward = 75.00, steps = 75
    11:44:37 [INFO] train episode 282: reward = 75.00, steps = 75
    11:44:40 [INFO] train episode 283: reward = 56.00, steps = 56
    11:44:44 [INFO] train episode 284: reward = 81.00, steps = 81
    11:44:48 [INFO] train episode 285: reward = 79.00, steps = 79
    11:44:51 [INFO] train episode 286: reward = 64.00, steps = 64
    11:44:53 [INFO] train episode 287: reward = 46.00, steps = 46
    11:44:56 [INFO] train episode 288: reward = 61.00, steps = 61
    11:44:58 [INFO] train episode 289: reward = 52.00, steps = 52
    11:45:02 [INFO] train episode 290: reward = 79.00, steps = 79
    11:45:06 [INFO] train episode 291: reward = 97.00, steps = 97
    11:45:09 [INFO] train episode 292: reward = 71.00, steps = 71
    11:45:13 [INFO] train episode 293: reward = 81.00, steps = 81
    11:45:16 [INFO] train episode 294: reward = 63.00, steps = 63
    11:45:20 [INFO] train episode 295: reward = 80.00, steps = 80
    11:45:24 [INFO] train episode 296: reward = 100.00, steps = 100
    11:45:28 [INFO] train episode 297: reward = 80.00, steps = 80
    11:45:35 [INFO] train episode 298: reward = 150.00, steps = 150
    11:45:42 [INFO] train episode 299: reward = 163.00, steps = 163
    11:45:44 [INFO] train episode 300: reward = 47.00, steps = 47
    11:45:48 [INFO] train episode 301: reward = 86.00, steps = 86
    11:45:53 [INFO] train episode 302: reward = 106.00, steps = 106
    11:45:56 [INFO] train episode 303: reward = 68.00, steps = 68
    11:46:00 [INFO] train episode 304: reward = 82.00, steps = 82
    11:46:03 [INFO] train episode 305: reward = 66.00, steps = 66
    11:46:05 [INFO] train episode 306: reward = 46.00, steps = 46
    11:46:08 [INFO] train episode 307: reward = 69.00, steps = 69
    11:46:11 [INFO] train episode 308: reward = 58.00, steps = 58
    11:46:14 [INFO] train episode 309: reward = 60.00, steps = 60
    11:46:18 [INFO] train episode 310: reward = 87.00, steps = 87
    11:46:22 [INFO] train episode 311: reward = 88.00, steps = 88
    11:46:27 [INFO] train episode 312: reward = 117.00, steps = 117
    11:46:30 [INFO] train episode 313: reward = 52.00, steps = 52
    11:46:33 [INFO] train episode 314: reward = 82.00, steps = 82
    11:46:38 [INFO] train episode 315: reward = 95.00, steps = 95
    11:46:42 [INFO] train episode 316: reward = 85.00, steps = 85
    11:46:46 [INFO] train episode 317: reward = 86.00, steps = 86
    11:46:52 [INFO] train episode 318: reward = 145.00, steps = 145
    11:46:55 [INFO] train episode 319: reward = 62.00, steps = 62
    11:46:59 [INFO] train episode 320: reward = 88.00, steps = 88
    11:47:02 [INFO] train episode 321: reward = 53.00, steps = 53
    11:47:05 [INFO] train episode 322: reward = 64.00, steps = 64
    11:47:10 [INFO] train episode 323: reward = 112.00, steps = 112
    11:47:13 [INFO] train episode 324: reward = 74.00, steps = 74
    11:47:16 [INFO] train episode 325: reward = 58.00, steps = 58
    11:47:19 [INFO] train episode 326: reward = 65.00, steps = 65
    11:47:22 [INFO] train episode 327: reward = 71.00, steps = 71
    11:47:26 [INFO] train episode 328: reward = 87.00, steps = 87
    11:47:29 [INFO] train episode 329: reward = 61.00, steps = 61
    11:47:33 [INFO] train episode 330: reward = 86.00, steps = 86
    11:47:36 [INFO] train episode 331: reward = 73.00, steps = 73
    11:47:40 [INFO] train episode 332: reward = 86.00, steps = 86
    11:47:43 [INFO] train episode 333: reward = 54.00, steps = 54
    11:47:46 [INFO] train episode 334: reward = 62.00, steps = 62
    11:47:49 [INFO] train episode 335: reward = 70.00, steps = 70
    11:47:52 [INFO] train episode 336: reward = 60.00, steps = 60
    11:47:56 [INFO] train episode 337: reward = 82.00, steps = 82
    11:48:00 [INFO] train episode 338: reward = 97.00, steps = 97
    11:48:03 [INFO] train episode 339: reward = 70.00, steps = 70
    11:48:07 [INFO] train episode 340: reward = 83.00, steps = 83
    11:48:11 [INFO] train episode 341: reward = 91.00, steps = 91
    11:48:15 [INFO] train episode 342: reward = 84.00, steps = 84
    11:48:18 [INFO] train episode 343: reward = 72.00, steps = 72
    11:48:24 [INFO] train episode 344: reward = 119.00, steps = 119
    11:48:27 [INFO] train episode 345: reward = 59.00, steps = 59
    11:48:30 [INFO] train episode 346: reward = 82.00, steps = 82
    11:48:35 [INFO] train episode 347: reward = 94.00, steps = 94
    11:48:39 [INFO] train episode 348: reward = 103.00, steps = 103
    11:48:46 [INFO] train episode 349: reward = 149.00, steps = 149
    11:48:50 [INFO] train episode 350: reward = 82.00, steps = 82
    11:48:54 [INFO] train episode 351: reward = 77.00, steps = 77
    11:48:57 [INFO] train episode 352: reward = 78.00, steps = 78
    11:49:01 [INFO] train episode 353: reward = 75.00, steps = 75
    11:49:05 [INFO] train episode 354: reward = 89.00, steps = 89
    11:49:09 [INFO] train episode 355: reward = 81.00, steps = 81
    11:49:15 [INFO] train episode 356: reward = 137.00, steps = 137
    11:49:19 [INFO] train episode 357: reward = 105.00, steps = 105
    11:49:23 [INFO] train episode 358: reward = 88.00, steps = 88
    11:49:28 [INFO] train episode 359: reward = 102.00, steps = 102
    11:49:32 [INFO] train episode 360: reward = 85.00, steps = 85
    11:49:37 [INFO] train episode 361: reward = 106.00, steps = 106
    11:49:40 [INFO] train episode 362: reward = 74.00, steps = 74
    11:49:44 [INFO] train episode 363: reward = 79.00, steps = 79
    11:49:48 [INFO] train episode 364: reward = 78.00, steps = 78
    11:49:52 [INFO] train episode 365: reward = 93.00, steps = 93
    11:49:59 [INFO] train episode 366: reward = 147.00, steps = 147
    11:50:05 [INFO] train episode 367: reward = 149.00, steps = 149
    11:50:12 [INFO] train episode 368: reward = 140.00, steps = 140
    11:50:15 [INFO] train episode 369: reward = 71.00, steps = 71
    11:50:21 [INFO] train episode 370: reward = 123.00, steps = 123
    11:50:25 [INFO] train episode 371: reward = 99.00, steps = 99
    11:50:28 [INFO] train episode 372: reward = 71.00, steps = 71
    11:50:33 [INFO] train episode 373: reward = 110.00, steps = 110
    11:50:36 [INFO] train episode 374: reward = 65.00, steps = 65
    11:50:39 [INFO] train episode 375: reward = 61.00, steps = 61
    11:50:44 [INFO] train episode 376: reward = 96.00, steps = 96
    11:50:48 [INFO] train episode 377: reward = 100.00, steps = 100
    11:50:55 [INFO] train episode 378: reward = 148.00, steps = 148
    11:51:03 [INFO] train episode 379: reward = 166.00, steps = 166
    11:51:05 [INFO] train episode 380: reward = 59.00, steps = 59
    11:51:09 [INFO] train episode 381: reward = 84.00, steps = 84
    11:51:16 [INFO] train episode 382: reward = 150.00, steps = 150
    11:51:21 [INFO] train episode 383: reward = 100.00, steps = 100
    11:51:23 [INFO] train episode 384: reward = 65.00, steps = 65
    11:51:28 [INFO] train episode 385: reward = 92.00, steps = 92
    11:51:31 [INFO] train episode 386: reward = 66.00, steps = 66
    11:51:35 [INFO] train episode 387: reward = 101.00, steps = 101
    11:51:39 [INFO] train episode 388: reward = 86.00, steps = 86
    11:51:44 [INFO] train episode 389: reward = 92.00, steps = 92
    11:51:47 [INFO] train episode 390: reward = 73.00, steps = 73
    11:51:50 [INFO] train episode 391: reward = 60.00, steps = 60
    11:51:53 [INFO] train episode 392: reward = 71.00, steps = 71
    11:51:58 [INFO] train episode 393: reward = 109.00, steps = 109
    11:52:02 [INFO] train episode 394: reward = 95.00, steps = 95
    11:52:06 [INFO] train episode 395: reward = 74.00, steps = 74
    11:52:11 [INFO] train episode 396: reward = 117.00, steps = 117
    11:52:16 [INFO] train episode 397: reward = 110.00, steps = 110
    11:52:20 [INFO] train episode 398: reward = 89.00, steps = 89
    11:52:25 [INFO] train episode 399: reward = 103.00, steps = 103
    11:52:30 [INFO] train episode 400: reward = 107.00, steps = 107
    11:52:35 [INFO] train episode 401: reward = 111.00, steps = 111
    11:52:38 [INFO] train episode 402: reward = 79.00, steps = 79
    11:52:42 [INFO] train episode 403: reward = 75.00, steps = 75
    11:52:46 [INFO] train episode 404: reward = 95.00, steps = 95
    11:52:52 [INFO] train episode 405: reward = 133.00, steps = 133
    11:52:57 [INFO] train episode 406: reward = 100.00, steps = 100
    11:53:02 [INFO] train episode 407: reward = 101.00, steps = 101
    11:53:06 [INFO] train episode 408: reward = 105.00, steps = 105
    11:53:10 [INFO] train episode 409: reward = 88.00, steps = 88
    11:53:15 [INFO] train episode 410: reward = 110.00, steps = 110
    11:53:19 [INFO] train episode 411: reward = 81.00, steps = 81
    11:53:25 [INFO] train episode 412: reward = 121.00, steps = 121
    11:53:28 [INFO] train episode 413: reward = 76.00, steps = 76
    11:53:33 [INFO] train episode 414: reward = 103.00, steps = 103
    11:53:36 [INFO] train episode 415: reward = 66.00, steps = 66
    11:53:40 [INFO] train episode 416: reward = 91.00, steps = 91
    11:53:44 [INFO] train episode 417: reward = 83.00, steps = 83
    11:53:49 [INFO] train episode 418: reward = 110.00, steps = 110
    11:53:58 [INFO] train episode 419: reward = 200.00, steps = 200
    11:54:02 [INFO] train episode 420: reward = 94.00, steps = 94
    11:54:06 [INFO] train episode 421: reward = 76.00, steps = 76
    11:54:10 [INFO] train episode 422: reward = 92.00, steps = 92
    11:54:14 [INFO] train episode 423: reward = 82.00, steps = 82
    11:54:16 [INFO] train episode 424: reward = 58.00, steps = 58
    11:54:21 [INFO] train episode 425: reward = 99.00, steps = 99
    11:54:26 [INFO] train episode 426: reward = 104.00, steps = 104
    11:54:30 [INFO] train episode 427: reward = 74.00, steps = 74
    11:54:32 [INFO] train episode 428: reward = 59.00, steps = 59
    11:54:35 [INFO] train episode 429: reward = 58.00, steps = 58
    11:54:40 [INFO] train episode 430: reward = 97.00, steps = 97
    11:54:45 [INFO] train episode 431: reward = 109.00, steps = 109
    11:54:51 [INFO] train episode 432: reward = 126.00, steps = 126
    11:54:55 [INFO] train episode 433: reward = 88.00, steps = 88
    11:54:59 [INFO] train episode 434: reward = 95.00, steps = 95
    11:55:04 [INFO] train episode 435: reward = 116.00, steps = 116
    11:55:08 [INFO] train episode 436: reward = 77.00, steps = 77
    11:55:16 [INFO] train episode 437: reward = 172.00, steps = 172
    11:55:19 [INFO] train episode 438: reward = 82.00, steps = 82
    11:55:23 [INFO] train episode 439: reward = 86.00, steps = 86
    11:55:29 [INFO] train episode 440: reward = 128.00, steps = 128
    11:55:36 [INFO] train episode 441: reward = 142.00, steps = 142
    11:55:40 [INFO] train episode 442: reward = 99.00, steps = 99
    11:55:44 [INFO] train episode 443: reward = 84.00, steps = 84
    11:55:50 [INFO] train episode 444: reward = 121.00, steps = 121
    11:55:54 [INFO] train episode 445: reward = 95.00, steps = 95
    11:55:57 [INFO] train episode 446: reward = 65.00, steps = 65
    11:56:01 [INFO] train episode 447: reward = 75.00, steps = 75
    11:56:05 [INFO] train episode 448: reward = 104.00, steps = 104
    11:56:10 [INFO] train episode 449: reward = 100.00, steps = 100
    11:56:15 [INFO] train episode 450: reward = 106.00, steps = 106
    11:56:18 [INFO] train episode 451: reward = 67.00, steps = 67
    11:56:23 [INFO] train episode 452: reward = 105.00, steps = 105
    11:56:26 [INFO] train episode 453: reward = 63.00, steps = 63
    11:56:30 [INFO] train episode 454: reward = 93.00, steps = 93
    11:56:34 [INFO] train episode 455: reward = 91.00, steps = 91
    11:56:39 [INFO] train episode 456: reward = 106.00, steps = 106
    11:56:44 [INFO] train episode 457: reward = 114.00, steps = 114
    11:56:48 [INFO] train episode 458: reward = 80.00, steps = 80
    11:56:53 [INFO] train episode 459: reward = 116.00, steps = 116
    11:56:56 [INFO] train episode 460: reward = 76.00, steps = 76
    11:57:01 [INFO] train episode 461: reward = 97.00, steps = 97
    11:57:05 [INFO] train episode 462: reward = 87.00, steps = 87
    11:57:12 [INFO] train episode 463: reward = 144.00, steps = 144
    11:57:15 [INFO] train episode 464: reward = 73.00, steps = 73
    11:57:20 [INFO] train episode 465: reward = 118.00, steps = 118
    11:57:24 [INFO] train episode 466: reward = 82.00, steps = 82
    11:57:28 [INFO] train episode 467: reward = 88.00, steps = 88
    11:57:34 [INFO] train episode 468: reward = 119.00, steps = 119
    11:57:39 [INFO] train episode 469: reward = 116.00, steps = 116
    11:57:44 [INFO] train episode 470: reward = 109.00, steps = 109
    11:57:48 [INFO] train episode 471: reward = 84.00, steps = 84
    11:57:57 [INFO] train episode 472: reward = 200.00, steps = 200
    11:58:02 [INFO] train episode 473: reward = 107.00, steps = 107
    11:58:08 [INFO] train episode 474: reward = 130.00, steps = 130
    11:58:11 [INFO] train episode 475: reward = 57.00, steps = 57
    11:58:13 [INFO] train episode 476: reward = 56.00, steps = 56
    11:58:18 [INFO] train episode 477: reward = 100.00, steps = 100
    11:58:23 [INFO] train episode 478: reward = 110.00, steps = 110
    11:58:27 [INFO] train episode 479: reward = 92.00, steps = 92
    11:58:31 [INFO] train episode 480: reward = 86.00, steps = 86
    11:58:36 [INFO] train episode 481: reward = 121.00, steps = 121
    11:58:41 [INFO] train episode 482: reward = 97.00, steps = 97
    11:58:45 [INFO] train episode 483: reward = 86.00, steps = 86
    11:58:51 [INFO] train episode 484: reward = 131.00, steps = 131
    11:58:54 [INFO] train episode 485: reward = 65.00, steps = 65
    11:58:58 [INFO] train episode 486: reward = 95.00, steps = 95
    11:59:03 [INFO] train episode 487: reward = 96.00, steps = 96
    11:59:06 [INFO] train episode 488: reward = 73.00, steps = 73
    11:59:09 [INFO] train episode 489: reward = 66.00, steps = 66
    11:59:16 [INFO] train episode 490: reward = 137.00, steps = 137
    11:59:20 [INFO] train episode 491: reward = 86.00, steps = 86
    11:59:23 [INFO] train episode 492: reward = 69.00, steps = 69
    11:59:28 [INFO] train episode 493: reward = 119.00, steps = 119
    11:59:33 [INFO] train episode 494: reward = 112.00, steps = 112
    11:59:37 [INFO] train episode 495: reward = 89.00, steps = 89
    11:59:42 [INFO] train episode 496: reward = 92.00, steps = 92
    11:59:45 [INFO] train episode 497: reward = 80.00, steps = 80
    11:59:49 [INFO] train episode 498: reward = 80.00, steps = 80
    11:59:52 [INFO] train episode 499: reward = 74.00, steps = 74
    11:59:56 [INFO] train episode 500: reward = 72.00, steps = 72
    12:00:00 [INFO] train episode 501: reward = 99.00, steps = 99
    12:00:04 [INFO] train episode 502: reward = 83.00, steps = 83
    12:00:11 [INFO] train episode 503: reward = 155.00, steps = 155
    12:00:16 [INFO] train episode 504: reward = 100.00, steps = 100
    12:00:21 [INFO] train episode 505: reward = 107.00, steps = 107
    12:00:26 [INFO] train episode 506: reward = 99.00, steps = 99
    12:00:29 [INFO] train episode 507: reward = 66.00, steps = 66
    12:00:32 [INFO] train episode 508: reward = 81.00, steps = 81
    12:00:39 [INFO] train episode 509: reward = 131.00, steps = 131
    12:00:42 [INFO] train episode 510: reward = 74.00, steps = 74
    12:00:47 [INFO] train episode 511: reward = 111.00, steps = 111
    12:00:52 [INFO] train episode 512: reward = 90.00, steps = 90
    12:00:56 [INFO] train episode 513: reward = 88.00, steps = 88
    12:01:00 [INFO] train episode 514: reward = 92.00, steps = 92
    12:01:04 [INFO] train episode 515: reward = 89.00, steps = 89
    12:01:07 [INFO] train episode 516: reward = 59.00, steps = 59
    12:01:11 [INFO] train episode 517: reward = 82.00, steps = 82
    12:01:15 [INFO] train episode 518: reward = 77.00, steps = 77
    12:01:19 [INFO] train episode 519: reward = 90.00, steps = 90
    12:01:24 [INFO] train episode 520: reward = 116.00, steps = 116
    12:01:27 [INFO] train episode 521: reward = 69.00, steps = 69
    12:01:33 [INFO] train episode 522: reward = 124.00, steps = 124
    12:01:41 [INFO] train episode 523: reward = 169.00, steps = 169
    12:01:46 [INFO] train episode 524: reward = 104.00, steps = 104
    12:01:49 [INFO] train episode 525: reward = 76.00, steps = 76
    12:01:53 [INFO] train episode 526: reward = 75.00, steps = 75
    12:01:58 [INFO] train episode 527: reward = 119.00, steps = 119
    12:02:03 [INFO] train episode 528: reward = 96.00, steps = 96
    12:02:09 [INFO] train episode 529: reward = 135.00, steps = 135
    12:02:13 [INFO] train episode 530: reward = 86.00, steps = 86
    12:02:19 [INFO] train episode 531: reward = 149.00, steps = 149
    12:02:25 [INFO] train episode 532: reward = 131.00, steps = 131
    12:02:29 [INFO] train episode 533: reward = 76.00, steps = 76
    12:02:38 [INFO] train episode 534: reward = 200.00, steps = 200
    12:02:42 [INFO] train episode 535: reward = 89.00, steps = 89
    12:02:47 [INFO] train episode 536: reward = 102.00, steps = 102
    12:02:51 [INFO] train episode 537: reward = 101.00, steps = 101
    12:02:55 [INFO] train episode 538: reward = 79.00, steps = 79
    12:02:59 [INFO] train episode 539: reward = 91.00, steps = 91
    12:03:03 [INFO] train episode 540: reward = 79.00, steps = 79
    12:03:09 [INFO] train episode 541: reward = 129.00, steps = 129
    12:03:13 [INFO] train episode 542: reward = 101.00, steps = 101
    12:03:19 [INFO] train episode 543: reward = 136.00, steps = 136
    12:03:24 [INFO] train episode 544: reward = 97.00, steps = 97
    12:03:30 [INFO] train episode 545: reward = 132.00, steps = 132
    12:03:35 [INFO] train episode 546: reward = 118.00, steps = 118
    12:03:41 [INFO] train episode 547: reward = 136.00, steps = 136
    12:03:47 [INFO] train episode 548: reward = 121.00, steps = 121
    12:03:50 [INFO] train episode 549: reward = 75.00, steps = 75
    12:03:56 [INFO] train episode 550: reward = 130.00, steps = 130
    12:04:00 [INFO] train episode 551: reward = 84.00, steps = 84
    12:04:05 [INFO] train episode 552: reward = 105.00, steps = 105
    12:04:11 [INFO] train episode 553: reward = 147.00, steps = 147
    12:04:18 [INFO] train episode 554: reward = 141.00, steps = 141
    12:04:22 [INFO] train episode 555: reward = 81.00, steps = 81
    12:04:26 [INFO] train episode 556: reward = 95.00, steps = 95
    12:04:31 [INFO] train episode 557: reward = 106.00, steps = 106
    12:04:38 [INFO] train episode 558: reward = 150.00, steps = 150
    12:04:41 [INFO] train episode 559: reward = 81.00, steps = 81
    12:04:46 [INFO] train episode 560: reward = 100.00, steps = 100
    12:04:52 [INFO] train episode 561: reward = 138.00, steps = 138
    12:04:59 [INFO] train episode 562: reward = 136.00, steps = 136
    12:05:03 [INFO] train episode 563: reward = 96.00, steps = 96
    12:05:06 [INFO] train episode 564: reward = 73.00, steps = 73
    12:05:15 [INFO] train episode 565: reward = 200.00, steps = 200
    12:05:19 [INFO] train episode 566: reward = 78.00, steps = 78
    12:05:27 [INFO] train episode 567: reward = 183.00, steps = 183
    12:05:34 [INFO] train episode 568: reward = 158.00, steps = 158
    12:05:38 [INFO] train episode 569: reward = 82.00, steps = 82
    12:05:45 [INFO] train episode 570: reward = 136.00, steps = 136
    12:05:48 [INFO] train episode 571: reward = 69.00, steps = 69
    12:05:53 [INFO] train episode 572: reward = 101.00, steps = 101
    12:05:59 [INFO] train episode 573: reward = 128.00, steps = 128
    12:06:04 [INFO] train episode 574: reward = 114.00, steps = 114
    12:06:08 [INFO] train episode 575: reward = 84.00, steps = 84
    12:06:10 [INFO] train episode 576: reward = 60.00, steps = 60
    12:06:15 [INFO] train episode 577: reward = 99.00, steps = 99
    12:06:20 [INFO] train episode 578: reward = 117.00, steps = 117
    12:06:25 [INFO] train episode 579: reward = 95.00, steps = 95
    12:06:31 [INFO] train episode 580: reward = 143.00, steps = 143
    12:06:35 [INFO] train episode 581: reward = 82.00, steps = 82
    12:06:42 [INFO] train episode 582: reward = 163.00, steps = 163
    12:06:50 [INFO] train episode 583: reward = 176.00, steps = 176
    12:06:55 [INFO] train episode 584: reward = 98.00, steps = 98
    12:06:58 [INFO] train episode 585: reward = 79.00, steps = 79
    12:07:06 [INFO] train episode 586: reward = 166.00, steps = 166
    12:07:12 [INFO] train episode 587: reward = 136.00, steps = 136
    12:07:16 [INFO] train episode 588: reward = 83.00, steps = 83
    12:07:25 [INFO] train episode 589: reward = 200.00, steps = 200
    12:07:30 [INFO] train episode 590: reward = 101.00, steps = 101
    12:07:36 [INFO] train episode 591: reward = 129.00, steps = 129
    12:07:42 [INFO] train episode 592: reward = 134.00, steps = 134
    12:07:47 [INFO] train episode 593: reward = 124.00, steps = 124
    12:07:53 [INFO] train episode 594: reward = 122.00, steps = 122
    12:07:58 [INFO] train episode 595: reward = 106.00, steps = 106
    12:08:02 [INFO] train episode 596: reward = 103.00, steps = 103
    12:08:07 [INFO] train episode 597: reward = 100.00, steps = 100
    12:08:12 [INFO] train episode 598: reward = 104.00, steps = 104
    12:08:16 [INFO] train episode 599: reward = 89.00, steps = 89
    12:08:21 [INFO] train episode 600: reward = 105.00, steps = 105
    12:08:25 [INFO] train episode 601: reward = 96.00, steps = 96
    12:08:31 [INFO] train episode 602: reward = 123.00, steps = 123
    12:08:35 [INFO] train episode 603: reward = 96.00, steps = 96
    12:08:39 [INFO] train episode 604: reward = 96.00, steps = 96
    12:08:46 [INFO] train episode 605: reward = 142.00, steps = 142
    12:08:52 [INFO] train episode 606: reward = 138.00, steps = 138
    12:08:57 [INFO] train episode 607: reward = 101.00, steps = 101
    12:09:05 [INFO] train episode 608: reward = 184.00, steps = 184
    12:09:09 [INFO] train episode 609: reward = 76.00, steps = 76
    12:09:13 [INFO] train episode 610: reward = 93.00, steps = 93
    12:09:19 [INFO] train episode 611: reward = 130.00, steps = 130
    12:09:25 [INFO] train episode 612: reward = 129.00, steps = 129
    12:09:31 [INFO] train episode 613: reward = 133.00, steps = 133
    12:09:35 [INFO] train episode 614: reward = 98.00, steps = 98
    12:09:39 [INFO] train episode 615: reward = 86.00, steps = 86
    12:09:46 [INFO] train episode 616: reward = 146.00, steps = 146
    12:09:50 [INFO] train episode 617: reward = 71.00, steps = 71
    12:09:54 [INFO] train episode 618: reward = 82.00, steps = 82
    12:10:00 [INFO] train episode 619: reward = 142.00, steps = 142
    12:10:04 [INFO] train episode 620: reward = 87.00, steps = 87
    12:10:12 [INFO] train episode 621: reward = 165.00, steps = 165
    12:10:15 [INFO] train episode 622: reward = 86.00, steps = 86
    12:10:19 [INFO] train episode 623: reward = 84.00, steps = 84
    12:10:25 [INFO] train episode 624: reward = 117.00, steps = 117
    12:10:29 [INFO] train episode 625: reward = 88.00, steps = 88
    12:10:32 [INFO] train episode 626: reward = 79.00, steps = 79
    12:10:38 [INFO] train episode 627: reward = 119.00, steps = 119
    12:10:42 [INFO] train episode 628: reward = 88.00, steps = 88
    12:10:48 [INFO] train episode 629: reward = 125.00, steps = 125
    12:10:51 [INFO] train episode 630: reward = 70.00, steps = 70
    12:10:56 [INFO] train episode 631: reward = 113.00, steps = 113
    12:11:05 [INFO] train episode 632: reward = 188.00, steps = 188
    12:11:10 [INFO] train episode 633: reward = 117.00, steps = 117
    12:11:14 [INFO] train episode 634: reward = 94.00, steps = 94
    12:11:21 [INFO] train episode 635: reward = 148.00, steps = 148
    12:11:26 [INFO] train episode 636: reward = 112.00, steps = 112
    12:11:31 [INFO] train episode 637: reward = 117.00, steps = 117
    12:11:37 [INFO] train episode 638: reward = 125.00, steps = 125
    12:11:41 [INFO] train episode 639: reward = 87.00, steps = 87
    12:11:45 [INFO] train episode 640: reward = 95.00, steps = 95
    12:11:49 [INFO] train episode 641: reward = 81.00, steps = 81
    12:11:54 [INFO] train episode 642: reward = 100.00, steps = 100
    12:12:00 [INFO] train episode 643: reward = 143.00, steps = 143
    12:12:04 [INFO] train episode 644: reward = 79.00, steps = 79
    12:12:11 [INFO] train episode 645: reward = 153.00, steps = 153
    12:12:18 [INFO] train episode 646: reward = 166.00, steps = 166
    12:12:22 [INFO] train episode 647: reward = 92.00, steps = 92
    12:12:28 [INFO] train episode 648: reward = 129.00, steps = 129
    12:12:33 [INFO] train episode 649: reward = 112.00, steps = 112
    12:12:37 [INFO] train episode 650: reward = 90.00, steps = 90
    12:12:41 [INFO] train episode 651: reward = 75.00, steps = 75
    12:12:48 [INFO] train episode 652: reward = 143.00, steps = 143
    12:12:53 [INFO] train episode 653: reward = 108.00, steps = 108
    12:12:58 [INFO] train episode 654: reward = 118.00, steps = 118
    12:13:03 [INFO] train episode 655: reward = 101.00, steps = 101
    12:13:11 [INFO] train episode 656: reward = 160.00, steps = 160
    12:13:17 [INFO] train episode 657: reward = 147.00, steps = 147
    12:13:22 [INFO] train episode 658: reward = 96.00, steps = 96
    12:13:26 [INFO] train episode 659: reward = 93.00, steps = 93
    12:13:33 [INFO] train episode 660: reward = 160.00, steps = 160
    12:13:38 [INFO] train episode 661: reward = 95.00, steps = 95
    12:13:44 [INFO] train episode 662: reward = 145.00, steps = 145
    12:13:53 [INFO] train episode 663: reward = 200.00, steps = 200
    12:13:57 [INFO] train episode 664: reward = 92.00, steps = 92
    12:14:02 [INFO] train episode 665: reward = 95.00, steps = 95
    12:14:05 [INFO] train episode 666: reward = 79.00, steps = 79
    12:14:10 [INFO] train episode 667: reward = 106.00, steps = 106
    12:14:15 [INFO] train episode 668: reward = 98.00, steps = 98
    12:14:22 [INFO] train episode 669: reward = 163.00, steps = 163
    12:14:25 [INFO] train episode 670: reward = 71.00, steps = 71
    12:14:31 [INFO] train episode 671: reward = 134.00, steps = 134
    12:14:37 [INFO] train episode 672: reward = 117.00, steps = 117
    12:14:43 [INFO] train episode 673: reward = 138.00, steps = 138
    12:14:48 [INFO] train episode 674: reward = 115.00, steps = 115
    12:14:55 [INFO] train episode 675: reward = 141.00, steps = 141
    12:14:59 [INFO] train episode 676: reward = 84.00, steps = 84
    12:15:04 [INFO] train episode 677: reward = 126.00, steps = 126
    12:15:10 [INFO] train episode 678: reward = 121.00, steps = 121
    12:15:15 [INFO] train episode 679: reward = 105.00, steps = 105
    12:15:19 [INFO] train episode 680: reward = 84.00, steps = 84
    12:15:24 [INFO] train episode 681: reward = 112.00, steps = 112
    12:15:28 [INFO] train episode 682: reward = 101.00, steps = 101
    12:15:32 [INFO] train episode 683: reward = 78.00, steps = 78
    12:15:38 [INFO] train episode 684: reward = 139.00, steps = 139
    12:15:44 [INFO] train episode 685: reward = 135.00, steps = 135
    12:15:51 [INFO] train episode 686: reward = 148.00, steps = 148
    12:15:58 [INFO] train episode 687: reward = 152.00, steps = 152
    12:16:03 [INFO] train episode 688: reward = 119.00, steps = 119
    12:16:08 [INFO] train episode 689: reward = 108.00, steps = 108
    12:16:15 [INFO] train episode 690: reward = 145.00, steps = 145
    12:16:20 [INFO] train episode 691: reward = 113.00, steps = 113
    12:16:26 [INFO] train episode 692: reward = 128.00, steps = 128
    12:16:31 [INFO] train episode 693: reward = 104.00, steps = 104
    12:16:35 [INFO] train episode 694: reward = 93.00, steps = 93
    12:16:38 [INFO] train episode 695: reward = 78.00, steps = 78
    12:16:42 [INFO] train episode 696: reward = 75.00, steps = 75
    12:16:46 [INFO] train episode 697: reward = 93.00, steps = 93
    12:16:54 [INFO] train episode 698: reward = 179.00, steps = 179
    12:16:59 [INFO] train episode 699: reward = 115.00, steps = 115
    12:17:07 [INFO] train episode 700: reward = 167.00, steps = 167
    12:17:16 [INFO] train episode 701: reward = 192.00, steps = 192
    12:17:20 [INFO] train episode 702: reward = 100.00, steps = 100
    12:17:29 [INFO] train episode 703: reward = 200.00, steps = 200
    12:17:35 [INFO] train episode 704: reward = 122.00, steps = 122
    12:17:44 [INFO] train episode 705: reward = 200.00, steps = 200
    12:17:49 [INFO] train episode 706: reward = 118.00, steps = 118
    12:17:54 [INFO] train episode 707: reward = 117.00, steps = 117
    12:17:59 [INFO] train episode 708: reward = 91.00, steps = 91
    12:18:03 [INFO] train episode 709: reward = 89.00, steps = 89
    12:18:06 [INFO] train episode 710: reward = 80.00, steps = 80
    12:18:15 [INFO] train episode 711: reward = 186.00, steps = 186
    12:18:20 [INFO] train episode 712: reward = 125.00, steps = 125
    12:18:30 [INFO] train episode 713: reward = 200.00, steps = 200
    12:18:33 [INFO] train episode 714: reward = 89.00, steps = 89
    12:18:41 [INFO] train episode 715: reward = 170.00, steps = 170
    12:18:46 [INFO] train episode 716: reward = 103.00, steps = 103
    12:18:55 [INFO] train episode 717: reward = 193.00, steps = 193
    12:19:01 [INFO] train episode 718: reward = 132.00, steps = 132
    12:19:07 [INFO] train episode 719: reward = 138.00, steps = 138
    12:19:12 [INFO] train episode 720: reward = 119.00, steps = 119
    12:19:20 [INFO] train episode 721: reward = 173.00, steps = 173
    12:19:27 [INFO] train episode 722: reward = 158.00, steps = 158
    12:19:33 [INFO] train episode 723: reward = 122.00, steps = 122
    12:19:38 [INFO] train episode 724: reward = 100.00, steps = 100
    12:19:44 [INFO] train episode 725: reward = 133.00, steps = 133
    12:19:49 [INFO] train episode 726: reward = 124.00, steps = 124
    12:19:56 [INFO] train episode 727: reward = 148.00, steps = 148
    12:20:03 [INFO] train episode 728: reward = 163.00, steps = 163
    12:20:08 [INFO] train episode 729: reward = 104.00, steps = 104
    12:20:17 [INFO] train episode 730: reward = 193.00, steps = 193
    12:20:22 [INFO] train episode 731: reward = 106.00, steps = 106
    12:20:27 [INFO] train episode 732: reward = 125.00, steps = 125
    12:20:34 [INFO] train episode 733: reward = 154.00, steps = 154
    12:20:39 [INFO] train episode 734: reward = 101.00, steps = 101
    12:20:47 [INFO] train episode 735: reward = 166.00, steps = 166
    12:20:54 [INFO] train episode 736: reward = 155.00, steps = 155
    12:20:58 [INFO] train episode 737: reward = 105.00, steps = 105
    12:21:03 [INFO] train episode 738: reward = 106.00, steps = 106
    12:21:08 [INFO] train episode 739: reward = 113.00, steps = 113
    12:21:12 [INFO] train episode 740: reward = 86.00, steps = 86
    12:21:19 [INFO] train episode 741: reward = 140.00, steps = 140
    12:21:26 [INFO] train episode 742: reward = 155.00, steps = 155
    12:21:33 [INFO] train episode 743: reward = 172.00, steps = 172
    12:21:39 [INFO] train episode 744: reward = 125.00, steps = 125
    12:21:47 [INFO] train episode 745: reward = 180.00, steps = 180
    12:21:53 [INFO] train episode 746: reward = 120.00, steps = 120
    12:22:00 [INFO] train episode 747: reward = 163.00, steps = 163
    12:22:06 [INFO] train episode 748: reward = 138.00, steps = 138
    12:22:11 [INFO] train episode 749: reward = 93.00, steps = 93
    12:22:18 [INFO] train episode 750: reward = 165.00, steps = 165
    12:22:24 [INFO] train episode 751: reward = 135.00, steps = 135
    12:22:33 [INFO] train episode 752: reward = 200.00, steps = 200
    12:22:41 [INFO] train episode 753: reward = 159.00, steps = 159
    12:22:49 [INFO] train episode 754: reward = 185.00, steps = 185
    12:22:58 [INFO] train episode 755: reward = 200.00, steps = 200
    12:23:06 [INFO] train episode 756: reward = 180.00, steps = 180
    12:23:12 [INFO] train episode 757: reward = 125.00, steps = 125
    12:23:16 [INFO] train episode 758: reward = 101.00, steps = 101
    12:23:21 [INFO] train episode 759: reward = 105.00, steps = 105
    12:23:29 [INFO] train episode 760: reward = 168.00, steps = 168
    12:23:33 [INFO] train episode 761: reward = 102.00, steps = 102
    12:23:39 [INFO] train episode 762: reward = 128.00, steps = 128
    12:23:48 [INFO] train episode 763: reward = 200.00, steps = 200
    12:23:55 [INFO] train episode 764: reward = 144.00, steps = 144
    12:24:02 [INFO] train episode 765: reward = 154.00, steps = 154
    12:24:11 [INFO] train episode 766: reward = 195.00, steps = 195
    12:24:17 [INFO] train episode 767: reward = 148.00, steps = 148
    12:24:24 [INFO] train episode 768: reward = 149.00, steps = 149
    12:24:30 [INFO] train episode 769: reward = 128.00, steps = 128
    12:24:36 [INFO] train episode 770: reward = 126.00, steps = 126
    12:24:42 [INFO] train episode 771: reward = 150.00, steps = 150
    12:24:48 [INFO] train episode 772: reward = 127.00, steps = 127
    12:24:58 [INFO] train episode 773: reward = 200.00, steps = 200
    12:25:05 [INFO] train episode 774: reward = 162.00, steps = 162
    12:25:15 [INFO] train episode 775: reward = 200.00, steps = 200
    12:25:16 [INFO] train episode 776: reward = 23.00, steps = 23
    12:25:22 [INFO] train episode 777: reward = 146.00, steps = 146
    12:25:30 [INFO] train episode 778: reward = 177.00, steps = 177
    12:25:35 [INFO] train episode 779: reward = 111.00, steps = 111
    12:25:43 [INFO] train episode 780: reward = 170.00, steps = 170
    12:25:50 [INFO] train episode 781: reward = 155.00, steps = 155
    12:25:56 [INFO] train episode 782: reward = 128.00, steps = 128
    12:26:03 [INFO] train episode 783: reward = 165.00, steps = 165
    12:26:09 [INFO] train episode 784: reward = 129.00, steps = 129
    12:26:18 [INFO] train episode 785: reward = 200.00, steps = 200
    12:26:26 [INFO] train episode 786: reward = 182.00, steps = 182
    12:26:34 [INFO] train episode 787: reward = 168.00, steps = 168
    12:26:39 [INFO] train episode 788: reward = 118.00, steps = 118
    12:26:47 [INFO] train episode 789: reward = 162.00, steps = 162
    12:26:50 [INFO] train episode 790: reward = 63.00, steps = 63
    12:26:55 [INFO] train episode 791: reward = 112.00, steps = 112
    12:26:57 [INFO] train episode 792: reward = 39.00, steps = 39
    12:27:03 [INFO] train episode 793: reward = 143.00, steps = 143
    12:27:09 [INFO] train episode 794: reward = 131.00, steps = 131
    12:27:14 [INFO] train episode 795: reward = 115.00, steps = 115
    12:27:20 [INFO] train episode 796: reward = 134.00, steps = 134
    12:27:24 [INFO] train episode 797: reward = 72.00, steps = 72
    12:27:30 [INFO] train episode 798: reward = 145.00, steps = 145
    12:27:32 [INFO] train episode 799: reward = 43.00, steps = 43
    12:27:40 [INFO] train episode 800: reward = 177.00, steps = 177
    12:27:43 [INFO] train episode 801: reward = 50.00, steps = 50
    12:27:51 [INFO] train episode 802: reward = 196.00, steps = 196
    12:28:00 [INFO] train episode 803: reward = 200.00, steps = 200
    12:28:10 [INFO] train episode 804: reward = 200.00, steps = 200
    12:28:15 [INFO] train episode 805: reward = 110.00, steps = 110
    12:28:22 [INFO] train episode 806: reward = 167.00, steps = 167
    12:28:27 [INFO] train episode 807: reward = 98.00, steps = 98
    12:28:32 [INFO] train episode 808: reward = 116.00, steps = 116
    12:28:39 [INFO] train episode 809: reward = 146.00, steps = 146
    12:28:47 [INFO] train episode 810: reward = 194.00, steps = 194
    12:28:54 [INFO] train episode 811: reward = 141.00, steps = 141
    12:29:00 [INFO] train episode 812: reward = 147.00, steps = 147
    12:29:07 [INFO] train episode 813: reward = 155.00, steps = 155
    12:29:12 [INFO] train episode 814: reward = 107.00, steps = 107
    12:29:17 [INFO] train episode 815: reward = 109.00, steps = 109
    12:29:25 [INFO] train episode 816: reward = 160.00, steps = 160
    12:29:32 [INFO] train episode 817: reward = 165.00, steps = 165
    12:29:41 [INFO] train episode 818: reward = 194.00, steps = 194
    12:29:47 [INFO] train episode 819: reward = 135.00, steps = 135
    12:29:55 [INFO] train episode 820: reward = 183.00, steps = 183
    12:30:04 [INFO] train episode 821: reward = 200.00, steps = 200
    12:30:10 [INFO] train episode 822: reward = 120.00, steps = 120
    12:30:15 [INFO] train episode 823: reward = 115.00, steps = 115
    12:30:23 [INFO] train episode 824: reward = 178.00, steps = 178
    12:30:30 [INFO] train episode 825: reward = 147.00, steps = 147
    12:30:36 [INFO] train episode 826: reward = 133.00, steps = 133
    12:30:41 [INFO] train episode 827: reward = 125.00, steps = 125
    12:30:46 [INFO] train episode 828: reward = 106.00, steps = 106
    12:30:52 [INFO] train episode 829: reward = 136.00, steps = 136
    12:30:59 [INFO] train episode 830: reward = 137.00, steps = 137
    12:31:06 [INFO] train episode 831: reward = 168.00, steps = 168
    12:31:12 [INFO] train episode 832: reward = 123.00, steps = 123
    12:31:20 [INFO] train episode 833: reward = 186.00, steps = 186
    12:31:26 [INFO] train episode 834: reward = 130.00, steps = 130
    12:31:32 [INFO] train episode 835: reward = 126.00, steps = 126
    12:31:41 [INFO] train episode 836: reward = 200.00, steps = 200
    12:31:50 [INFO] train episode 837: reward = 200.00, steps = 200
    12:31:57 [INFO] train episode 838: reward = 151.00, steps = 151
    12:32:06 [INFO] train episode 839: reward = 200.00, steps = 200
    12:32:12 [INFO] train episode 840: reward = 141.00, steps = 141
    12:32:17 [INFO] train episode 841: reward = 108.00, steps = 108
    12:32:22 [INFO] train episode 842: reward = 105.00, steps = 105
    12:32:28 [INFO] train episode 843: reward = 135.00, steps = 135
    12:32:33 [INFO] train episode 844: reward = 109.00, steps = 109
    12:32:42 [INFO] train episode 845: reward = 198.00, steps = 198
    12:32:47 [INFO] train episode 846: reward = 105.00, steps = 105
    12:32:51 [INFO] train episode 847: reward = 103.00, steps = 103
    12:32:59 [INFO] train episode 848: reward = 163.00, steps = 163
    12:33:04 [INFO] train episode 849: reward = 115.00, steps = 115
    12:33:11 [INFO] train episode 850: reward = 154.00, steps = 154
    12:33:17 [INFO] train episode 851: reward = 137.00, steps = 137
    12:33:22 [INFO] train episode 852: reward = 99.00, steps = 99
    12:33:30 [INFO] train episode 853: reward = 171.00, steps = 171
    12:33:38 [INFO] train episode 854: reward = 184.00, steps = 184
    12:33:43 [INFO] train episode 855: reward = 121.00, steps = 121
    12:33:49 [INFO] train episode 856: reward = 132.00, steps = 132
    12:33:54 [INFO] train episode 857: reward = 111.00, steps = 111
    12:34:01 [INFO] train episode 858: reward = 154.00, steps = 154
    12:34:07 [INFO] train episode 859: reward = 131.00, steps = 131
    12:34:13 [INFO] train episode 860: reward = 124.00, steps = 124
    12:34:19 [INFO] train episode 861: reward = 128.00, steps = 128
    12:34:28 [INFO] train episode 862: reward = 200.00, steps = 200
    12:34:34 [INFO] train episode 863: reward = 137.00, steps = 137
    12:34:42 [INFO] train episode 864: reward = 183.00, steps = 183
    12:34:47 [INFO] train episode 865: reward = 101.00, steps = 101
    12:34:52 [INFO] train episode 866: reward = 115.00, steps = 115
    12:34:58 [INFO] train episode 867: reward = 123.00, steps = 123
    12:35:03 [INFO] train episode 868: reward = 110.00, steps = 110
    12:35:11 [INFO] train episode 869: reward = 171.00, steps = 171
    12:35:16 [INFO] train episode 870: reward = 123.00, steps = 123
    12:35:22 [INFO] train episode 871: reward = 113.00, steps = 113
    12:35:29 [INFO] train episode 872: reward = 169.00, steps = 169
    12:35:36 [INFO] train episode 873: reward = 148.00, steps = 148
    12:35:42 [INFO] train episode 874: reward = 132.00, steps = 132
    12:35:47 [INFO] train episode 875: reward = 111.00, steps = 111
    12:35:54 [INFO] train episode 876: reward = 162.00, steps = 162
    12:36:03 [INFO] train episode 877: reward = 200.00, steps = 200
    12:36:11 [INFO] train episode 878: reward = 163.00, steps = 163
    12:36:18 [INFO] train episode 879: reward = 153.00, steps = 153
    12:36:25 [INFO] train episode 880: reward = 162.00, steps = 162
    12:36:31 [INFO] train episode 881: reward = 141.00, steps = 141
    12:36:37 [INFO] train episode 882: reward = 114.00, steps = 114
    12:36:46 [INFO] train episode 883: reward = 200.00, steps = 200
    12:36:54 [INFO] train episode 884: reward = 174.00, steps = 174
    12:36:59 [INFO] train episode 885: reward = 109.00, steps = 109
    12:37:05 [INFO] train episode 886: reward = 131.00, steps = 131
    12:37:10 [INFO] train episode 887: reward = 112.00, steps = 112
    12:37:20 [INFO] train episode 888: reward = 200.00, steps = 200
    12:37:26 [INFO] train episode 889: reward = 152.00, steps = 152
    12:37:33 [INFO] train episode 890: reward = 140.00, steps = 140
    12:37:39 [INFO] train episode 891: reward = 131.00, steps = 131
    12:37:45 [INFO] train episode 892: reward = 141.00, steps = 141
    12:37:51 [INFO] train episode 893: reward = 126.00, steps = 126
    12:37:55 [INFO] train episode 894: reward = 97.00, steps = 97
    12:38:02 [INFO] train episode 895: reward = 152.00, steps = 152
    12:38:09 [INFO] train episode 896: reward = 148.00, steps = 148
    12:38:14 [INFO] train episode 897: reward = 109.00, steps = 109
    12:38:23 [INFO] train episode 898: reward = 200.00, steps = 200
    12:38:28 [INFO] train episode 899: reward = 101.00, steps = 101
    12:38:34 [INFO] train episode 900: reward = 140.00, steps = 140
    12:38:40 [INFO] train episode 901: reward = 134.00, steps = 134
    12:38:47 [INFO] train episode 902: reward = 166.00, steps = 166
    12:38:56 [INFO] train episode 903: reward = 200.00, steps = 200
    12:39:03 [INFO] train episode 904: reward = 152.00, steps = 152
    12:39:10 [INFO] train episode 905: reward = 149.00, steps = 149
    12:39:15 [INFO] train episode 906: reward = 110.00, steps = 110
    12:39:20 [INFO] train episode 907: reward = 111.00, steps = 111
    12:39:27 [INFO] train episode 908: reward = 151.00, steps = 151
    12:39:34 [INFO] train episode 909: reward = 149.00, steps = 149
    12:39:41 [INFO] train episode 910: reward = 151.00, steps = 151
    12:39:46 [INFO] train episode 911: reward = 110.00, steps = 110
    12:39:53 [INFO] train episode 912: reward = 165.00, steps = 165
    12:40:01 [INFO] train episode 913: reward = 165.00, steps = 165
    12:40:09 [INFO] train episode 914: reward = 183.00, steps = 183
    12:40:16 [INFO] train episode 915: reward = 147.00, steps = 147
    12:40:23 [INFO] train episode 916: reward = 164.00, steps = 164
    12:40:30 [INFO] train episode 917: reward = 144.00, steps = 144
    12:40:39 [INFO] train episode 918: reward = 197.00, steps = 197
    12:40:45 [INFO] train episode 919: reward = 118.00, steps = 118
    12:40:52 [INFO] train episode 920: reward = 165.00, steps = 165
    12:40:59 [INFO] train episode 921: reward = 140.00, steps = 140
    12:41:04 [INFO] train episode 922: reward = 116.00, steps = 116
    12:41:09 [INFO] train episode 923: reward = 117.00, steps = 117
    12:41:15 [INFO] train episode 924: reward = 132.00, steps = 132
    12:41:20 [INFO] train episode 925: reward = 94.00, steps = 94
    12:41:25 [INFO] train episode 926: reward = 122.00, steps = 122
    12:41:32 [INFO] train episode 927: reward = 150.00, steps = 150
    12:41:40 [INFO] train episode 928: reward = 172.00, steps = 172
    12:41:46 [INFO] train episode 929: reward = 126.00, steps = 126
    12:41:51 [INFO] train episode 930: reward = 130.00, steps = 130
    12:41:58 [INFO] train episode 931: reward = 140.00, steps = 140
    12:42:03 [INFO] train episode 932: reward = 107.00, steps = 107
    12:42:12 [INFO] train episode 933: reward = 200.00, steps = 200
    12:42:18 [INFO] train episode 934: reward = 129.00, steps = 129
    12:42:27 [INFO] train episode 935: reward = 200.00, steps = 200
    12:42:33 [INFO] train episode 936: reward = 148.00, steps = 148
    12:42:41 [INFO] train episode 937: reward = 164.00, steps = 164
    12:42:47 [INFO] train episode 938: reward = 132.00, steps = 132
    12:42:52 [INFO] train episode 939: reward = 108.00, steps = 108
    12:42:59 [INFO] train episode 940: reward = 149.00, steps = 149
    12:43:04 [INFO] train episode 941: reward = 121.00, steps = 121
    12:43:11 [INFO] train episode 942: reward = 148.00, steps = 148
    12:43:17 [INFO] train episode 943: reward = 138.00, steps = 138
    12:43:26 [INFO] train episode 944: reward = 200.00, steps = 200
    12:43:31 [INFO] train episode 945: reward = 112.00, steps = 112
    12:43:37 [INFO] train episode 946: reward = 123.00, steps = 123
    12:43:46 [INFO] train episode 947: reward = 200.00, steps = 200
    12:43:52 [INFO] train episode 948: reward = 140.00, steps = 140
    12:44:01 [INFO] train episode 949: reward = 185.00, steps = 185
    12:44:06 [INFO] train episode 950: reward = 112.00, steps = 112
    12:44:11 [INFO] train episode 951: reward = 127.00, steps = 127
    12:44:18 [INFO] train episode 952: reward = 154.00, steps = 154
    12:44:23 [INFO] train episode 953: reward = 100.00, steps = 100
    12:44:30 [INFO] train episode 954: reward = 154.00, steps = 154
    12:44:38 [INFO] train episode 955: reward = 178.00, steps = 178
    12:44:46 [INFO] train episode 956: reward = 163.00, steps = 163
    12:44:51 [INFO] train episode 957: reward = 104.00, steps = 104
    12:45:00 [INFO] train episode 958: reward = 200.00, steps = 200
    12:45:09 [INFO] train episode 959: reward = 196.00, steps = 196
    12:45:17 [INFO] train episode 960: reward = 177.00, steps = 177
    12:45:22 [INFO] train episode 961: reward = 119.00, steps = 119
    12:45:29 [INFO] train episode 962: reward = 148.00, steps = 148
    12:45:35 [INFO] train episode 963: reward = 122.00, steps = 122
    12:45:40 [INFO] train episode 964: reward = 113.00, steps = 113
    12:45:48 [INFO] train episode 965: reward = 175.00, steps = 175
    12:45:54 [INFO] train episode 966: reward = 147.00, steps = 147
    12:46:01 [INFO] train episode 967: reward = 147.00, steps = 147
    12:46:07 [INFO] train episode 968: reward = 130.00, steps = 130
    12:46:14 [INFO] train episode 969: reward = 162.00, steps = 162
    12:46:22 [INFO] train episode 970: reward = 179.00, steps = 179
    12:46:28 [INFO] train episode 971: reward = 130.00, steps = 130
    12:46:37 [INFO] train episode 972: reward = 199.00, steps = 199
    12:46:46 [INFO] train episode 973: reward = 200.00, steps = 200
    12:46:51 [INFO] train episode 974: reward = 109.00, steps = 109
    12:47:00 [INFO] train episode 975: reward = 193.00, steps = 193
    12:47:04 [INFO] train episode 976: reward = 97.00, steps = 97
    12:47:12 [INFO] train episode 977: reward = 165.00, steps = 165
    12:47:17 [INFO] train episode 978: reward = 115.00, steps = 115
    12:47:24 [INFO] train episode 979: reward = 159.00, steps = 159
    12:47:31 [INFO] train episode 980: reward = 139.00, steps = 139
    12:47:38 [INFO] train episode 981: reward = 173.00, steps = 173
    12:47:47 [INFO] train episode 982: reward = 195.00, steps = 195
    12:47:53 [INFO] train episode 983: reward = 129.00, steps = 129
    12:48:01 [INFO] train episode 984: reward = 171.00, steps = 171
    12:48:08 [INFO] train episode 985: reward = 157.00, steps = 157
    12:48:13 [INFO] train episode 986: reward = 110.00, steps = 110
    12:48:18 [INFO] train episode 987: reward = 113.00, steps = 113
    12:48:22 [INFO] train episode 988: reward = 90.00, steps = 90
    12:48:28 [INFO] train episode 989: reward = 125.00, steps = 125
    12:48:33 [INFO] train episode 990: reward = 104.00, steps = 104
    12:48:38 [INFO] train episode 991: reward = 119.00, steps = 119
    12:48:44 [INFO] train episode 992: reward = 127.00, steps = 127
    12:48:51 [INFO] train episode 993: reward = 157.00, steps = 157
    12:48:57 [INFO] train episode 994: reward = 116.00, steps = 116
    12:49:01 [INFO] train episode 995: reward = 104.00, steps = 104
    12:49:05 [INFO] train episode 996: reward = 88.00, steps = 88
    12:49:13 [INFO] train episode 997: reward = 179.00, steps = 179
    12:49:21 [INFO] train episode 998: reward = 159.00, steps = 159
    12:49:26 [INFO] train episode 999: reward = 125.00, steps = 125
    12:49:31 [INFO] train episode 1000: reward = 98.00, steps = 98
    12:49:35 [INFO] train episode 1001: reward = 96.00, steps = 96
    12:49:41 [INFO] train episode 1002: reward = 123.00, steps = 123
    12:49:45 [INFO] train episode 1003: reward = 93.00, steps = 93
    12:49:50 [INFO] train episode 1004: reward = 111.00, steps = 111
    12:49:57 [INFO] train episode 1005: reward = 151.00, steps = 151
    12:50:02 [INFO] train episode 1006: reward = 111.00, steps = 111
    12:50:10 [INFO] train episode 1007: reward = 170.00, steps = 170
    12:50:14 [INFO] train episode 1008: reward = 95.00, steps = 95
    12:50:19 [INFO] train episode 1009: reward = 105.00, steps = 105
    12:50:25 [INFO] train episode 1010: reward = 130.00, steps = 130
    12:50:30 [INFO] train episode 1011: reward = 101.00, steps = 101
    12:50:36 [INFO] train episode 1012: reward = 130.00, steps = 130
    12:50:39 [INFO] train episode 1013: reward = 83.00, steps = 83
    12:50:45 [INFO] train episode 1014: reward = 127.00, steps = 127
    12:50:53 [INFO] train episode 1015: reward = 163.00, steps = 163
    12:51:00 [INFO] train episode 1016: reward = 166.00, steps = 166
    12:51:06 [INFO] train episode 1017: reward = 130.00, steps = 130
    12:51:10 [INFO] train episode 1018: reward = 95.00, steps = 95
    12:51:16 [INFO] train episode 1019: reward = 134.00, steps = 134
    12:51:23 [INFO] train episode 1020: reward = 139.00, steps = 139
    12:51:27 [INFO] train episode 1021: reward = 96.00, steps = 96
    12:51:34 [INFO] train episode 1022: reward = 155.00, steps = 155
    12:51:39 [INFO] train episode 1023: reward = 107.00, steps = 107
    12:51:44 [INFO] train episode 1024: reward = 107.00, steps = 107
    12:51:49 [INFO] train episode 1025: reward = 105.00, steps = 105
    12:51:56 [INFO] train episode 1026: reward = 147.00, steps = 147
    12:52:01 [INFO] train episode 1027: reward = 125.00, steps = 125
    12:52:06 [INFO] train episode 1028: reward = 108.00, steps = 108
    12:52:11 [INFO] train episode 1029: reward = 108.00, steps = 108
    12:52:17 [INFO] train episode 1030: reward = 129.00, steps = 129
    12:52:22 [INFO] train episode 1031: reward = 113.00, steps = 113
    12:52:26 [INFO] train episode 1032: reward = 80.00, steps = 80
    12:52:31 [INFO] train episode 1033: reward = 106.00, steps = 106
    12:52:35 [INFO] train episode 1034: reward = 91.00, steps = 91
    12:52:40 [INFO] train episode 1035: reward = 106.00, steps = 106
    12:52:45 [INFO] train episode 1036: reward = 121.00, steps = 121
    12:52:49 [INFO] train episode 1037: reward = 91.00, steps = 91
    12:52:55 [INFO] train episode 1038: reward = 114.00, steps = 114
    12:52:59 [INFO] train episode 1039: reward = 103.00, steps = 103
    12:53:05 [INFO] train episode 1040: reward = 111.00, steps = 111
    12:53:11 [INFO] train episode 1041: reward = 154.00, steps = 154
    12:53:16 [INFO] train episode 1042: reward = 93.00, steps = 93
    12:53:22 [INFO] train episode 1043: reward = 129.00, steps = 129
    12:53:31 [INFO] train episode 1044: reward = 200.00, steps = 200
    12:53:37 [INFO] train episode 1045: reward = 128.00, steps = 128
    12:53:42 [INFO] train episode 1046: reward = 121.00, steps = 121
    12:53:51 [INFO] train episode 1047: reward = 200.00, steps = 200
    12:53:57 [INFO] train episode 1048: reward = 130.00, steps = 130
    12:54:01 [INFO] train episode 1049: reward = 93.00, steps = 93
    12:54:06 [INFO] train episode 1050: reward = 94.00, steps = 94
    12:54:11 [INFO] train episode 1051: reward = 118.00, steps = 118
    12:54:19 [INFO] train episode 1052: reward = 171.00, steps = 171
    12:54:23 [INFO] train episode 1053: reward = 101.00, steps = 101
    12:54:28 [INFO] train episode 1054: reward = 95.00, steps = 95
    12:54:36 [INFO] train episode 1055: reward = 180.00, steps = 180
    12:54:40 [INFO] train episode 1056: reward = 98.00, steps = 98
    12:54:45 [INFO] train episode 1057: reward = 109.00, steps = 109
    12:54:51 [INFO] train episode 1058: reward = 122.00, steps = 122
    12:54:56 [INFO] train episode 1059: reward = 100.00, steps = 100
    12:55:02 [INFO] train episode 1060: reward = 142.00, steps = 142
    12:55:06 [INFO] train episode 1061: reward = 96.00, steps = 96
    12:55:12 [INFO] train episode 1062: reward = 130.00, steps = 130
    12:55:18 [INFO] train episode 1063: reward = 126.00, steps = 126
    12:55:28 [INFO] train episode 1064: reward = 200.00, steps = 200
    12:55:34 [INFO] train episode 1065: reward = 132.00, steps = 132
    12:55:38 [INFO] train episode 1066: reward = 98.00, steps = 98
    12:55:43 [INFO] train episode 1067: reward = 90.00, steps = 90
    12:55:48 [INFO] train episode 1068: reward = 128.00, steps = 128
    12:55:57 [INFO] train episode 1069: reward = 182.00, steps = 182
    12:56:01 [INFO] train episode 1070: reward = 105.00, steps = 105
    12:56:07 [INFO] train episode 1071: reward = 131.00, steps = 131
    12:56:12 [INFO] train episode 1072: reward = 100.00, steps = 100
    12:56:20 [INFO] train episode 1073: reward = 186.00, steps = 186
    12:56:29 [INFO] train episode 1074: reward = 193.00, steps = 193
    12:56:34 [INFO] train episode 1075: reward = 100.00, steps = 100
    12:56:43 [INFO] train episode 1076: reward = 200.00, steps = 200
    12:56:49 [INFO] train episode 1077: reward = 146.00, steps = 146
    12:56:58 [INFO] train episode 1078: reward = 200.00, steps = 200
    12:57:04 [INFO] train episode 1079: reward = 116.00, steps = 116
    12:57:10 [INFO] train episode 1080: reward = 131.00, steps = 131
    12:57:16 [INFO] train episode 1081: reward = 144.00, steps = 144
    12:57:22 [INFO] train episode 1082: reward = 118.00, steps = 118
    12:57:28 [INFO] train episode 1083: reward = 130.00, steps = 130
    12:57:37 [INFO] train episode 1084: reward = 200.00, steps = 200
    12:57:43 [INFO] train episode 1085: reward = 154.00, steps = 154
    12:57:50 [INFO] train episode 1086: reward = 155.00, steps = 155
    12:57:59 [INFO] train episode 1087: reward = 179.00, steps = 179
    12:58:04 [INFO] train episode 1088: reward = 119.00, steps = 119
    12:58:08 [INFO] train episode 1089: reward = 93.00, steps = 93
    12:58:13 [INFO] train episode 1090: reward = 111.00, steps = 111
    12:58:18 [INFO] train episode 1091: reward = 101.00, steps = 101
    12:58:23 [INFO] train episode 1092: reward = 97.00, steps = 97
    12:58:27 [INFO] train episode 1093: reward = 102.00, steps = 102
    12:58:33 [INFO] train episode 1094: reward = 133.00, steps = 133
    12:58:37 [INFO] train episode 1095: reward = 88.00, steps = 88
    12:58:43 [INFO] train episode 1096: reward = 126.00, steps = 126
    12:58:50 [INFO] train episode 1097: reward = 154.00, steps = 154
    12:58:56 [INFO] train episode 1098: reward = 142.00, steps = 142
    12:59:02 [INFO] train episode 1099: reward = 115.00, steps = 115
    12:59:06 [INFO] train episode 1100: reward = 95.00, steps = 95
    12:59:12 [INFO] train episode 1101: reward = 123.00, steps = 123
    12:59:17 [INFO] train episode 1102: reward = 119.00, steps = 119
    12:59:22 [INFO] train episode 1103: reward = 119.00, steps = 119
    12:59:29 [INFO] train episode 1104: reward = 140.00, steps = 140
    12:59:34 [INFO] train episode 1105: reward = 119.00, steps = 119
    12:59:39 [INFO] train episode 1106: reward = 112.00, steps = 112
    12:59:47 [INFO] train episode 1107: reward = 171.00, steps = 171
    12:59:53 [INFO] train episode 1108: reward = 141.00, steps = 141
    12:59:58 [INFO] train episode 1109: reward = 109.00, steps = 109
    13:00:04 [INFO] train episode 1110: reward = 130.00, steps = 130
    13:00:08 [INFO] train episode 1111: reward = 84.00, steps = 84
    13:00:14 [INFO] train episode 1112: reward = 121.00, steps = 121
    13:00:20 [INFO] train episode 1113: reward = 143.00, steps = 143
    13:00:25 [INFO] train episode 1114: reward = 114.00, steps = 114
    13:00:30 [INFO] train episode 1115: reward = 98.00, steps = 98
    13:00:39 [INFO] train episode 1116: reward = 194.00, steps = 194
    13:00:43 [INFO] train episode 1117: reward = 99.00, steps = 99
    13:00:49 [INFO] train episode 1118: reward = 134.00, steps = 134
    13:00:55 [INFO] train episode 1119: reward = 130.00, steps = 130
    13:01:00 [INFO] train episode 1120: reward = 95.00, steps = 95
    13:01:04 [INFO] train episode 1121: reward = 101.00, steps = 101
    13:01:11 [INFO] train episode 1122: reward = 146.00, steps = 146
    13:01:19 [INFO] train episode 1123: reward = 183.00, steps = 183
    13:01:27 [INFO] train episode 1124: reward = 181.00, steps = 181
    13:01:33 [INFO] train episode 1125: reward = 128.00, steps = 128
    13:01:39 [INFO] train episode 1126: reward = 138.00, steps = 138
    13:01:45 [INFO] train episode 1127: reward = 111.00, steps = 111
    13:01:53 [INFO] train episode 1128: reward = 189.00, steps = 189
    13:01:59 [INFO] train episode 1129: reward = 124.00, steps = 124
    13:02:03 [INFO] train episode 1130: reward = 94.00, steps = 94
    13:02:09 [INFO] train episode 1131: reward = 122.00, steps = 122
    13:02:18 [INFO] train episode 1132: reward = 200.00, steps = 200
    13:02:22 [INFO] train episode 1133: reward = 102.00, steps = 102
    13:02:31 [INFO] train episode 1134: reward = 200.00, steps = 200
    13:02:38 [INFO] train episode 1135: reward = 139.00, steps = 139
    13:02:43 [INFO] train episode 1136: reward = 122.00, steps = 122
    13:02:48 [INFO] train episode 1137: reward = 106.00, steps = 106
    13:02:53 [INFO] train episode 1138: reward = 101.00, steps = 101
    13:02:58 [INFO] train episode 1139: reward = 110.00, steps = 110
    13:03:06 [INFO] train episode 1140: reward = 168.00, steps = 168
    13:03:15 [INFO] train episode 1141: reward = 200.00, steps = 200
    13:03:20 [INFO] train episode 1142: reward = 123.00, steps = 123
    13:03:26 [INFO] train episode 1143: reward = 138.00, steps = 138
    13:03:32 [INFO] train episode 1144: reward = 116.00, steps = 116
    13:03:39 [INFO] train episode 1145: reward = 162.00, steps = 162
    13:03:45 [INFO] train episode 1146: reward = 137.00, steps = 137
    13:03:53 [INFO] train episode 1147: reward = 173.00, steps = 173
    13:03:58 [INFO] train episode 1148: reward = 116.00, steps = 116
    13:04:03 [INFO] train episode 1149: reward = 106.00, steps = 106
    13:04:08 [INFO] train episode 1150: reward = 110.00, steps = 110
    13:04:13 [INFO] train episode 1151: reward = 110.00, steps = 110
    13:04:21 [INFO] train episode 1152: reward = 158.00, steps = 158
    13:04:28 [INFO] train episode 1153: reward = 164.00, steps = 164
    13:04:36 [INFO] train episode 1154: reward = 175.00, steps = 175
    13:04:40 [INFO] train episode 1155: reward = 88.00, steps = 88
    13:04:45 [INFO] train episode 1156: reward = 111.00, steps = 111
    13:04:54 [INFO] train episode 1157: reward = 182.00, steps = 182
    13:04:59 [INFO] train episode 1158: reward = 120.00, steps = 120
    13:05:03 [INFO] train episode 1159: reward = 89.00, steps = 89
    13:05:08 [INFO] train episode 1160: reward = 118.00, steps = 118
    13:05:15 [INFO] train episode 1161: reward = 155.00, steps = 155
    13:05:19 [INFO] train episode 1162: reward = 82.00, steps = 82
    13:05:24 [INFO] train episode 1163: reward = 111.00, steps = 111
    13:05:32 [INFO] train episode 1164: reward = 173.00, steps = 173
    13:05:37 [INFO] train episode 1165: reward = 101.00, steps = 101
    13:05:43 [INFO] train episode 1166: reward = 128.00, steps = 128
    13:05:49 [INFO] train episode 1167: reward = 133.00, steps = 133
    13:05:55 [INFO] train episode 1168: reward = 136.00, steps = 136
    13:06:01 [INFO] train episode 1169: reward = 137.00, steps = 137
    13:06:05 [INFO] train episode 1170: reward = 94.00, steps = 94
    13:06:13 [INFO] train episode 1171: reward = 163.00, steps = 163
    13:06:17 [INFO] train episode 1172: reward = 102.00, steps = 102
    13:06:25 [INFO] train episode 1173: reward = 173.00, steps = 173
    13:06:34 [INFO] train episode 1174: reward = 200.00, steps = 200
    13:06:42 [INFO] train episode 1175: reward = 180.00, steps = 180
    13:06:48 [INFO] train episode 1176: reward = 125.00, steps = 125
    13:06:53 [INFO] train episode 1177: reward = 104.00, steps = 104
    13:06:59 [INFO] train episode 1178: reward = 123.00, steps = 123
    13:07:03 [INFO] train episode 1179: reward = 102.00, steps = 102
    13:07:08 [INFO] train episode 1180: reward = 105.00, steps = 105
    13:07:12 [INFO] train episode 1181: reward = 89.00, steps = 89
    13:07:17 [INFO] train episode 1182: reward = 101.00, steps = 101
    13:07:22 [INFO] train episode 1183: reward = 106.00, steps = 106
    13:07:26 [INFO] train episode 1184: reward = 98.00, steps = 98
    13:07:32 [INFO] train episode 1185: reward = 119.00, steps = 119
    13:07:37 [INFO] train episode 1186: reward = 106.00, steps = 106
    13:07:42 [INFO] train episode 1187: reward = 114.00, steps = 114
    13:07:47 [INFO] train episode 1188: reward = 95.00, steps = 95
    13:07:53 [INFO] train episode 1189: reward = 127.00, steps = 127
    13:07:57 [INFO] train episode 1190: reward = 101.00, steps = 101
    13:08:02 [INFO] train episode 1191: reward = 106.00, steps = 106
    13:08:08 [INFO] train episode 1192: reward = 120.00, steps = 120
    13:08:16 [INFO] train episode 1193: reward = 182.00, steps = 182
    13:08:22 [INFO] train episode 1194: reward = 125.00, steps = 125
    13:08:27 [INFO] train episode 1195: reward = 127.00, steps = 127
    13:08:34 [INFO] train episode 1196: reward = 125.00, steps = 125
    13:08:40 [INFO] train episode 1197: reward = 128.00, steps = 128
    13:08:44 [INFO] train episode 1198: reward = 93.00, steps = 93
    13:08:49 [INFO] train episode 1199: reward = 116.00, steps = 116
    13:08:57 [INFO] train episode 1200: reward = 155.00, steps = 155
    13:09:02 [INFO] train episode 1201: reward = 114.00, steps = 114
    13:09:10 [INFO] train episode 1202: reward = 153.00, steps = 153
    13:09:17 [INFO] train episode 1203: reward = 135.00, steps = 135
    13:09:27 [INFO] train episode 1204: reward = 200.00, steps = 200
    13:09:32 [INFO] train episode 1205: reward = 93.00, steps = 93
    13:09:37 [INFO] train episode 1206: reward = 96.00, steps = 96
    13:09:43 [INFO] train episode 1207: reward = 128.00, steps = 128
    13:09:48 [INFO] train episode 1208: reward = 97.00, steps = 97
    13:09:55 [INFO] train episode 1209: reward = 133.00, steps = 133
    13:10:03 [INFO] train episode 1210: reward = 166.00, steps = 166
    13:10:08 [INFO] train episode 1211: reward = 92.00, steps = 92
    13:10:14 [INFO] train episode 1212: reward = 113.00, steps = 113
    13:10:20 [INFO] train episode 1213: reward = 107.00, steps = 107
    13:10:27 [INFO] train episode 1214: reward = 126.00, steps = 126
    13:10:34 [INFO] train episode 1215: reward = 136.00, steps = 136
    13:10:41 [INFO] train episode 1216: reward = 127.00, steps = 127
    13:10:49 [INFO] train episode 1217: reward = 162.00, steps = 162
    13:10:54 [INFO] train episode 1218: reward = 93.00, steps = 93
    13:10:59 [INFO] train episode 1219: reward = 99.00, steps = 99
    13:11:05 [INFO] train episode 1220: reward = 97.00, steps = 97
    13:11:13 [INFO] train episode 1221: reward = 124.00, steps = 124
    13:11:22 [INFO] train episode 1222: reward = 153.00, steps = 153
    13:11:33 [INFO] train episode 1223: reward = 200.00, steps = 200
    13:11:39 [INFO] train episode 1224: reward = 109.00, steps = 109
    13:11:46 [INFO] train episode 1225: reward = 129.00, steps = 129
    13:11:56 [INFO] train episode 1226: reward = 133.00, steps = 133
    13:12:03 [INFO] train episode 1227: reward = 116.00, steps = 116
    13:12:11 [INFO] train episode 1228: reward = 136.00, steps = 136
    13:12:18 [INFO] train episode 1229: reward = 122.00, steps = 122
    13:12:27 [INFO] train episode 1230: reward = 166.00, steps = 166
    13:12:34 [INFO] train episode 1231: reward = 128.00, steps = 128
    13:12:40 [INFO] train episode 1232: reward = 103.00, steps = 103
    13:12:46 [INFO] train episode 1233: reward = 103.00, steps = 103
    13:12:53 [INFO] train episode 1234: reward = 124.00, steps = 124
    13:13:02 [INFO] train episode 1235: reward = 147.00, steps = 147
    13:13:10 [INFO] train episode 1236: reward = 141.00, steps = 141
    13:13:17 [INFO] train episode 1237: reward = 128.00, steps = 128
    13:13:25 [INFO] train episode 1238: reward = 131.00, steps = 131
    13:13:31 [INFO] train episode 1239: reward = 107.00, steps = 107
    13:13:38 [INFO] train episode 1240: reward = 125.00, steps = 125
    13:13:47 [INFO] train episode 1241: reward = 160.00, steps = 160
    13:13:54 [INFO] train episode 1242: reward = 119.00, steps = 119
    13:14:00 [INFO] train episode 1243: reward = 107.00, steps = 107
    13:14:06 [INFO] train episode 1244: reward = 97.00, steps = 97
    13:14:12 [INFO] train episode 1245: reward = 112.00, steps = 112
    13:14:18 [INFO] train episode 1246: reward = 107.00, steps = 107
    13:14:27 [INFO] train episode 1247: reward = 151.00, steps = 151
    13:14:34 [INFO] train episode 1248: reward = 124.00, steps = 124
    13:14:44 [INFO] train episode 1249: reward = 172.00, steps = 172
    13:14:51 [INFO] train episode 1250: reward = 131.00, steps = 131
    13:14:58 [INFO] train episode 1251: reward = 115.00, steps = 115
    13:15:05 [INFO] train episode 1252: reward = 127.00, steps = 127
    13:15:11 [INFO] train episode 1253: reward = 119.00, steps = 119
    13:15:17 [INFO] train episode 1254: reward = 86.00, steps = 86
    13:15:23 [INFO] train episode 1255: reward = 120.00, steps = 120
    13:15:30 [INFO] train episode 1256: reward = 119.00, steps = 119
    13:15:39 [INFO] train episode 1257: reward = 146.00, steps = 146
    13:15:45 [INFO] train episode 1258: reward = 104.00, steps = 104
    13:15:53 [INFO] train episode 1259: reward = 146.00, steps = 146
    13:16:01 [INFO] train episode 1260: reward = 139.00, steps = 139
    13:16:10 [INFO] train episode 1261: reward = 167.00, steps = 167
    13:16:17 [INFO] train episode 1262: reward = 123.00, steps = 123
    13:16:24 [INFO] train episode 1263: reward = 127.00, steps = 127
    13:16:35 [INFO] train episode 1264: reward = 200.00, steps = 200
    13:16:47 [INFO] train episode 1265: reward = 197.00, steps = 197
    13:16:53 [INFO] train episode 1266: reward = 111.00, steps = 111
    13:17:00 [INFO] train episode 1267: reward = 122.00, steps = 122
    13:17:06 [INFO] train episode 1268: reward = 109.00, steps = 109
    13:17:17 [INFO] train episode 1269: reward = 183.00, steps = 183
    13:17:23 [INFO] train episode 1270: reward = 111.00, steps = 111
    13:17:30 [INFO] train episode 1271: reward = 135.00, steps = 135
    13:17:36 [INFO] train episode 1272: reward = 102.00, steps = 102
    13:17:41 [INFO] train episode 1273: reward = 91.00, steps = 91
    13:17:47 [INFO] train episode 1274: reward = 108.00, steps = 108
    13:17:52 [INFO] train episode 1275: reward = 80.00, steps = 80
    13:17:59 [INFO] train episode 1276: reward = 127.00, steps = 127
    13:18:07 [INFO] train episode 1277: reward = 132.00, steps = 132
    13:18:16 [INFO] train episode 1278: reward = 156.00, steps = 156
    13:18:22 [INFO] train episode 1279: reward = 110.00, steps = 110
    13:18:28 [INFO] train episode 1280: reward = 107.00, steps = 107
    13:18:34 [INFO] train episode 1281: reward = 104.00, steps = 104
    13:18:39 [INFO] train episode 1282: reward = 94.00, steps = 94
    13:18:46 [INFO] train episode 1283: reward = 123.00, steps = 123
    13:18:53 [INFO] train episode 1284: reward = 131.00, steps = 131
    13:19:00 [INFO] train episode 1285: reward = 113.00, steps = 113
    13:19:05 [INFO] train episode 1286: reward = 106.00, steps = 106
    13:19:16 [INFO] train episode 1287: reward = 160.00, steps = 160
    13:19:24 [INFO] train episode 1288: reward = 104.00, steps = 104
    13:19:29 [INFO] train episode 1289: reward = 86.00, steps = 86
    13:19:36 [INFO] train episode 1290: reward = 96.00, steps = 96
    13:19:43 [INFO] train episode 1291: reward = 120.00, steps = 120
    13:19:49 [INFO] train episode 1292: reward = 111.00, steps = 111
    13:19:57 [INFO] train episode 1293: reward = 129.00, steps = 129
    13:20:05 [INFO] train episode 1294: reward = 131.00, steps = 131
    13:20:13 [INFO] train episode 1295: reward = 143.00, steps = 143
    13:20:21 [INFO] train episode 1296: reward = 126.00, steps = 126
    13:20:31 [INFO] train episode 1297: reward = 167.00, steps = 167
    13:20:39 [INFO] train episode 1298: reward = 135.00, steps = 135
    13:20:46 [INFO] train episode 1299: reward = 117.00, steps = 117
    13:20:53 [INFO] train episode 1300: reward = 126.00, steps = 126
    13:21:00 [INFO] train episode 1301: reward = 106.00, steps = 106
    13:21:06 [INFO] train episode 1302: reward = 107.00, steps = 107
    13:21:14 [INFO] train episode 1303: reward = 141.00, steps = 141
    13:21:21 [INFO] train episode 1304: reward = 123.00, steps = 123
    13:21:28 [INFO] train episode 1305: reward = 117.00, steps = 117
    13:21:36 [INFO] train episode 1306: reward = 136.00, steps = 136
    13:21:46 [INFO] train episode 1307: reward = 172.00, steps = 172
    13:21:56 [INFO] train episode 1308: reward = 172.00, steps = 172
    13:22:03 [INFO] train episode 1309: reward = 113.00, steps = 113
    13:22:11 [INFO] train episode 1310: reward = 142.00, steps = 142
    13:22:18 [INFO] train episode 1311: reward = 132.00, steps = 132
    13:22:23 [INFO] train episode 1312: reward = 92.00, steps = 92
    13:22:31 [INFO] train episode 1313: reward = 133.00, steps = 133
    13:22:41 [INFO] train episode 1314: reward = 148.00, steps = 148
    13:22:48 [INFO] train episode 1315: reward = 120.00, steps = 120
    13:22:58 [INFO] train episode 1316: reward = 142.00, steps = 142
    13:23:07 [INFO] train episode 1317: reward = 147.00, steps = 147
    13:23:14 [INFO] train episode 1318: reward = 133.00, steps = 133
    13:23:27 [INFO] train episode 1319: reward = 200.00, steps = 200
    13:23:39 [INFO] train episode 1320: reward = 187.00, steps = 187
    13:23:49 [INFO] train episode 1321: reward = 187.00, steps = 187
    13:23:58 [INFO] train episode 1322: reward = 163.00, steps = 163
    13:24:08 [INFO] train episode 1323: reward = 187.00, steps = 187
    13:24:18 [INFO] train episode 1324: reward = 182.00, steps = 182
    13:24:28 [INFO] train episode 1325: reward = 195.00, steps = 195
    13:24:37 [INFO] train episode 1326: reward = 151.00, steps = 151
    13:24:43 [INFO] train episode 1327: reward = 117.00, steps = 117
    13:24:53 [INFO] train episode 1328: reward = 181.00, steps = 181
    13:25:01 [INFO] train episode 1329: reward = 147.00, steps = 147
    13:25:08 [INFO] train episode 1330: reward = 122.00, steps = 122
    13:25:19 [INFO] train episode 1331: reward = 200.00, steps = 200
    13:25:30 [INFO] train episode 1332: reward = 200.00, steps = 200
    13:25:38 [INFO] train episode 1333: reward = 154.00, steps = 154
    13:25:46 [INFO] train episode 1334: reward = 144.00, steps = 144
    13:25:53 [INFO] train episode 1335: reward = 132.00, steps = 132
    13:26:04 [INFO] train episode 1336: reward = 200.00, steps = 200
    13:26:11 [INFO] train episode 1337: reward = 137.00, steps = 137
    13:26:21 [INFO] train episode 1338: reward = 166.00, steps = 166
    13:26:31 [INFO] train episode 1339: reward = 200.00, steps = 200
    13:26:38 [INFO] train episode 1340: reward = 125.00, steps = 125
    13:26:45 [INFO] train episode 1341: reward = 131.00, steps = 131
    13:26:53 [INFO] train episode 1342: reward = 153.00, steps = 153
    13:27:02 [INFO] train episode 1343: reward = 164.00, steps = 164
    13:27:08 [INFO] train episode 1344: reward = 107.00, steps = 107
    13:27:16 [INFO] train episode 1345: reward = 143.00, steps = 143
    13:27:25 [INFO] train episode 1346: reward = 160.00, steps = 160
    13:27:32 [INFO] train episode 1347: reward = 127.00, steps = 127
    13:27:41 [INFO] train episode 1348: reward = 165.00, steps = 165
    13:27:50 [INFO] train episode 1349: reward = 173.00, steps = 173
    13:28:00 [INFO] train episode 1350: reward = 173.00, steps = 173
    13:28:10 [INFO] train episode 1351: reward = 200.00, steps = 200
    13:28:17 [INFO] train episode 1352: reward = 120.00, steps = 120
    13:28:23 [INFO] train episode 1353: reward = 113.00, steps = 113
    13:28:30 [INFO] train episode 1354: reward = 132.00, steps = 132
    13:28:40 [INFO] train episode 1355: reward = 171.00, steps = 171
    13:28:46 [INFO] train episode 1356: reward = 112.00, steps = 112
    13:28:54 [INFO] train episode 1357: reward = 162.00, steps = 162
    13:29:03 [INFO] train episode 1358: reward = 162.00, steps = 162
    13:29:10 [INFO] train episode 1359: reward = 131.00, steps = 131
    13:29:19 [INFO] train episode 1360: reward = 153.00, steps = 153
    13:29:24 [INFO] train episode 1361: reward = 105.00, steps = 105
    13:29:32 [INFO] train episode 1362: reward = 145.00, steps = 145
    13:29:42 [INFO] train episode 1363: reward = 173.00, steps = 173
    13:29:50 [INFO] train episode 1364: reward = 147.00, steps = 147
    13:29:59 [INFO] train episode 1365: reward = 179.00, steps = 179
    13:30:10 [INFO] train episode 1366: reward = 200.00, steps = 200
    13:30:19 [INFO] train episode 1367: reward = 168.00, steps = 168
    13:30:27 [INFO] train episode 1368: reward = 144.00, steps = 144
    13:30:33 [INFO] train episode 1369: reward = 110.00, steps = 110
    13:30:39 [INFO] train episode 1370: reward = 102.00, steps = 102
    13:30:46 [INFO] train episode 1371: reward = 136.00, steps = 136
    13:30:57 [INFO] train episode 1372: reward = 200.00, steps = 200
    13:31:06 [INFO] train episode 1373: reward = 165.00, steps = 165
    13:31:16 [INFO] train episode 1374: reward = 187.00, steps = 187
    13:31:23 [INFO] train episode 1375: reward = 126.00, steps = 126
    13:31:30 [INFO] train episode 1376: reward = 124.00, steps = 124
    13:31:36 [INFO] train episode 1377: reward = 104.00, steps = 104
    13:31:44 [INFO] train episode 1378: reward = 163.00, steps = 163
    13:31:51 [INFO] train episode 1379: reward = 126.00, steps = 126
    13:31:58 [INFO] train episode 1380: reward = 120.00, steps = 120
    13:32:04 [INFO] train episode 1381: reward = 106.00, steps = 106
    13:32:10 [INFO] train episode 1382: reward = 115.00, steps = 115
    13:32:19 [INFO] train episode 1383: reward = 166.00, steps = 166
    13:32:26 [INFO] train episode 1384: reward = 123.00, steps = 123
    13:32:37 [INFO] train episode 1385: reward = 200.00, steps = 200
    13:32:45 [INFO] train episode 1386: reward = 154.00, steps = 154
    13:32:53 [INFO] train episode 1387: reward = 132.00, steps = 132
    13:33:03 [INFO] train episode 1388: reward = 200.00, steps = 200
    13:33:13 [INFO] train episode 1389: reward = 169.00, steps = 169
    13:33:23 [INFO] train episode 1390: reward = 200.00, steps = 200
    13:33:30 [INFO] train episode 1391: reward = 114.00, steps = 114
    13:33:37 [INFO] train episode 1392: reward = 132.00, steps = 132
    13:33:45 [INFO] train episode 1393: reward = 148.00, steps = 148
    13:33:52 [INFO] train episode 1394: reward = 132.00, steps = 132
    13:34:03 [INFO] train episode 1395: reward = 189.00, steps = 189
    13:34:13 [INFO] train episode 1396: reward = 152.00, steps = 152
    13:34:24 [INFO] train episode 1397: reward = 158.00, steps = 158
    13:34:29 [INFO] train episode 1398: reward = 90.00, steps = 90
    13:34:42 [INFO] train episode 1399: reward = 200.00, steps = 200
    13:34:52 [INFO] train episode 1400: reward = 149.00, steps = 149
    13:35:02 [INFO] train episode 1401: reward = 143.00, steps = 143
    13:35:12 [INFO] train episode 1402: reward = 163.00, steps = 163
    13:35:19 [INFO] train episode 1403: reward = 109.00, steps = 109
    13:35:29 [INFO] train episode 1404: reward = 149.00, steps = 149
    13:35:37 [INFO] train episode 1405: reward = 134.00, steps = 134
    13:35:43 [INFO] train episode 1406: reward = 98.00, steps = 98
    13:35:52 [INFO] train episode 1407: reward = 144.00, steps = 144
    13:36:01 [INFO] train episode 1408: reward = 146.00, steps = 146
    13:36:11 [INFO] train episode 1409: reward = 170.00, steps = 170
    13:36:21 [INFO] train episode 1410: reward = 157.00, steps = 157
    13:36:31 [INFO] train episode 1411: reward = 146.00, steps = 146
    13:36:44 [INFO] train episode 1412: reward = 189.00, steps = 189
    13:36:52 [INFO] train episode 1413: reward = 112.00, steps = 112
    13:37:01 [INFO] train episode 1414: reward = 162.00, steps = 162
    13:37:09 [INFO] train episode 1415: reward = 133.00, steps = 133
    13:37:17 [INFO] train episode 1416: reward = 153.00, steps = 153
    13:37:24 [INFO] train episode 1417: reward = 122.00, steps = 122
    13:37:31 [INFO] train episode 1418: reward = 119.00, steps = 119
    13:37:41 [INFO] train episode 1419: reward = 175.00, steps = 175
    13:37:50 [INFO] train episode 1420: reward = 181.00, steps = 181
    13:37:58 [INFO] train episode 1421: reward = 152.00, steps = 152
    13:38:04 [INFO] train episode 1422: reward = 134.00, steps = 134
    13:38:12 [INFO] train episode 1423: reward = 158.00, steps = 158
    13:38:20 [INFO] train episode 1424: reward = 149.00, steps = 149
    13:38:26 [INFO] train episode 1425: reward = 132.00, steps = 132
    13:38:34 [INFO] train episode 1426: reward = 156.00, steps = 156
    13:38:44 [INFO] train episode 1427: reward = 173.00, steps = 173
    13:38:51 [INFO] train episode 1428: reward = 122.00, steps = 122
    13:39:04 [INFO] train episode 1429: reward = 200.00, steps = 200
    13:39:14 [INFO] train episode 1430: reward = 135.00, steps = 135
    13:39:24 [INFO] train episode 1431: reward = 115.00, steps = 115
    13:39:33 [INFO] train episode 1432: reward = 119.00, steps = 119
    13:39:47 [INFO] train episode 1433: reward = 196.00, steps = 196
    13:39:56 [INFO] train episode 1434: reward = 157.00, steps = 157
    13:40:08 [INFO] train episode 1435: reward = 200.00, steps = 200
    13:40:16 [INFO] train episode 1436: reward = 152.00, steps = 152
    13:40:27 [INFO] train episode 1437: reward = 188.00, steps = 188
    13:40:37 [INFO] train episode 1438: reward = 167.00, steps = 167
    13:40:46 [INFO] train episode 1439: reward = 128.00, steps = 128
    13:40:54 [INFO] train episode 1440: reward = 156.00, steps = 156
    13:41:01 [INFO] train episode 1441: reward = 125.00, steps = 125
    13:41:08 [INFO] train episode 1442: reward = 128.00, steps = 128
    13:41:18 [INFO] train episode 1443: reward = 178.00, steps = 178
    13:41:28 [INFO] train episode 1444: reward = 181.00, steps = 181
    13:41:35 [INFO] train episode 1445: reward = 148.00, steps = 148
    13:41:41 [INFO] train episode 1446: reward = 104.00, steps = 104
    13:41:50 [INFO] train episode 1447: reward = 184.00, steps = 184
    13:41:55 [INFO] train episode 1448: reward = 117.00, steps = 117
    13:42:02 [INFO] train episode 1449: reward = 136.00, steps = 136
    13:42:09 [INFO] train episode 1450: reward = 140.00, steps = 140
    13:42:17 [INFO] train episode 1451: reward = 150.00, steps = 150
    13:42:24 [INFO] train episode 1452: reward = 134.00, steps = 134
    13:42:31 [INFO] train episode 1453: reward = 155.00, steps = 155
    13:42:40 [INFO] train episode 1454: reward = 170.00, steps = 170
    13:42:48 [INFO] train episode 1455: reward = 168.00, steps = 168
    13:42:57 [INFO] train episode 1456: reward = 188.00, steps = 188
    13:43:02 [INFO] train episode 1457: reward = 105.00, steps = 105
    13:43:11 [INFO] train episode 1458: reward = 169.00, steps = 169
    13:43:17 [INFO] train episode 1459: reward = 137.00, steps = 137
    13:43:24 [INFO] train episode 1460: reward = 132.00, steps = 132
    13:43:33 [INFO] train episode 1461: reward = 185.00, steps = 185
    13:43:41 [INFO] train episode 1462: reward = 160.00, steps = 160
    13:43:46 [INFO] train episode 1463: reward = 121.00, steps = 121
    13:43:56 [INFO] train episode 1464: reward = 200.00, steps = 200
    13:44:01 [INFO] train episode 1465: reward = 98.00, steps = 98
    13:44:09 [INFO] train episode 1466: reward = 159.00, steps = 159
    13:44:15 [INFO] train episode 1467: reward = 126.00, steps = 126
    13:44:21 [INFO] train episode 1468: reward = 128.00, steps = 128
    13:44:31 [INFO] train episode 1469: reward = 200.00, steps = 200
    13:44:38 [INFO] train episode 1470: reward = 140.00, steps = 140
    13:44:48 [INFO] train episode 1471: reward = 200.00, steps = 200
    13:44:54 [INFO] train episode 1472: reward = 124.00, steps = 124
    13:45:02 [INFO] train episode 1473: reward = 164.00, steps = 164
    13:45:08 [INFO] train episode 1474: reward = 121.00, steps = 121
    13:45:14 [INFO] train episode 1475: reward = 115.00, steps = 115
    13:45:21 [INFO] train episode 1476: reward = 152.00, steps = 152
    13:45:27 [INFO] train episode 1477: reward = 112.00, steps = 112
    13:45:35 [INFO] train episode 1478: reward = 173.00, steps = 173
    13:45:43 [INFO] train episode 1479: reward = 146.00, steps = 146
    13:45:50 [INFO] train episode 1480: reward = 141.00, steps = 141
    13:45:56 [INFO] train episode 1481: reward = 122.00, steps = 122
    13:46:05 [INFO] train episode 1482: reward = 200.00, steps = 200
    13:46:12 [INFO] train episode 1483: reward = 128.00, steps = 128
    13:46:22 [INFO] train episode 1484: reward = 198.00, steps = 198
    13:46:31 [INFO] train episode 1485: reward = 200.00, steps = 200
    13:46:39 [INFO] train episode 1486: reward = 165.00, steps = 165
    13:46:46 [INFO] train episode 1487: reward = 138.00, steps = 138
    13:46:56 [INFO] train episode 1488: reward = 200.00, steps = 200
    13:47:06 [INFO] train episode 1489: reward = 195.00, steps = 195
    13:47:15 [INFO] train episode 1490: reward = 193.00, steps = 193
    13:47:24 [INFO] train episode 1491: reward = 177.00, steps = 177
    13:47:34 [INFO] train episode 1492: reward = 187.00, steps = 187
    13:47:42 [INFO] train episode 1493: reward = 161.00, steps = 161
    13:47:48 [INFO] train episode 1494: reward = 111.00, steps = 111
    13:47:56 [INFO] train episode 1495: reward = 148.00, steps = 148
    13:48:06 [INFO] train episode 1496: reward = 200.00, steps = 200
    13:48:14 [INFO] train episode 1497: reward = 157.00, steps = 157
    13:48:22 [INFO] train episode 1498: reward = 152.00, steps = 152
    13:48:30 [INFO] train episode 1499: reward = 140.00, steps = 140
    13:48:37 [INFO] train episode 1500: reward = 136.00, steps = 136
    13:48:45 [INFO] train episode 1501: reward = 149.00, steps = 149
    13:48:55 [INFO] train episode 1502: reward = 200.00, steps = 200
    13:49:06 [INFO] train episode 1503: reward = 200.00, steps = 200
    13:49:15 [INFO] train episode 1504: reward = 174.00, steps = 174
    13:49:23 [INFO] train episode 1505: reward = 161.00, steps = 161
    13:49:30 [INFO] train episode 1506: reward = 137.00, steps = 137
    13:49:38 [INFO] train episode 1507: reward = 139.00, steps = 139
    13:49:44 [INFO] train episode 1508: reward = 122.00, steps = 122
    13:49:55 [INFO] train episode 1509: reward = 200.00, steps = 200
    13:50:05 [INFO] train episode 1510: reward = 200.00, steps = 200
    13:50:12 [INFO] train episode 1511: reward = 127.00, steps = 127
    13:50:18 [INFO] train episode 1512: reward = 125.00, steps = 125
    13:50:26 [INFO] train episode 1513: reward = 152.00, steps = 152
    13:50:33 [INFO] train episode 1514: reward = 118.00, steps = 118
    13:50:43 [INFO] train episode 1515: reward = 197.00, steps = 197
    13:50:52 [INFO] train episode 1516: reward = 169.00, steps = 169
    13:51:01 [INFO] train episode 1517: reward = 183.00, steps = 183
    13:51:11 [INFO] train episode 1518: reward = 189.00, steps = 189
    13:51:17 [INFO] train episode 1519: reward = 110.00, steps = 110
    13:51:25 [INFO] train episode 1520: reward = 152.00, steps = 152
    13:51:34 [INFO] train episode 1521: reward = 176.00, steps = 176
    13:51:43 [INFO] train episode 1522: reward = 162.00, steps = 162
    13:51:50 [INFO] train episode 1523: reward = 143.00, steps = 143
    13:51:56 [INFO] train episode 1524: reward = 113.00, steps = 113
    13:52:05 [INFO] train episode 1525: reward = 179.00, steps = 179
    13:52:14 [INFO] train episode 1526: reward = 163.00, steps = 163
    13:52:23 [INFO] train episode 1527: reward = 170.00, steps = 170
    13:52:34 [INFO] train episode 1528: reward = 200.00, steps = 200
    13:52:43 [INFO] train episode 1529: reward = 173.00, steps = 173
    13:52:51 [INFO] train episode 1530: reward = 160.00, steps = 160
    13:52:58 [INFO] train episode 1531: reward = 140.00, steps = 140
    13:53:05 [INFO] train episode 1532: reward = 126.00, steps = 126
    13:53:12 [INFO] train episode 1533: reward = 144.00, steps = 144
    13:53:19 [INFO] train episode 1534: reward = 132.00, steps = 132
    13:53:29 [INFO] train episode 1535: reward = 178.00, steps = 178
    13:53:39 [INFO] train episode 1536: reward = 189.00, steps = 189
    13:53:49 [INFO] train episode 1537: reward = 200.00, steps = 200
    13:53:59 [INFO] train episode 1538: reward = 200.00, steps = 200
    13:54:06 [INFO] train episode 1539: reward = 123.00, steps = 123
    13:54:12 [INFO] train episode 1540: reward = 113.00, steps = 113
    13:54:19 [INFO] train episode 1541: reward = 141.00, steps = 141
    13:54:27 [INFO] train episode 1542: reward = 146.00, steps = 146
    13:54:38 [INFO] train episode 1543: reward = 200.00, steps = 200
    13:54:45 [INFO] train episode 1544: reward = 137.00, steps = 137
    13:54:54 [INFO] train episode 1545: reward = 186.00, steps = 186
    13:55:03 [INFO] train episode 1546: reward = 159.00, steps = 159
    13:55:13 [INFO] train episode 1547: reward = 199.00, steps = 199
    13:55:21 [INFO] train episode 1548: reward = 144.00, steps = 144
    13:55:28 [INFO] train episode 1549: reward = 141.00, steps = 141
    13:55:39 [INFO] train episode 1550: reward = 200.00, steps = 200
    13:55:49 [INFO] train episode 1551: reward = 200.00, steps = 200
    13:56:00 [INFO] train episode 1552: reward = 200.00, steps = 200
    13:56:10 [INFO] train episode 1553: reward = 200.00, steps = 200
    13:56:19 [INFO] train episode 1554: reward = 174.00, steps = 174
    13:56:26 [INFO] train episode 1555: reward = 129.00, steps = 129
    13:56:33 [INFO] train episode 1556: reward = 140.00, steps = 140
    13:56:44 [INFO] train episode 1557: reward = 200.00, steps = 200
    13:56:50 [INFO] train episode 1558: reward = 122.00, steps = 122
    13:57:01 [INFO] train episode 1559: reward = 200.00, steps = 200
    13:57:11 [INFO] train episode 1560: reward = 200.00, steps = 200
    13:57:18 [INFO] train episode 1561: reward = 123.00, steps = 123
    13:57:27 [INFO] train episode 1562: reward = 178.00, steps = 178
    13:57:33 [INFO] train episode 1563: reward = 106.00, steps = 106
    13:57:40 [INFO] train episode 1564: reward = 141.00, steps = 141
    13:57:50 [INFO] train episode 1565: reward = 200.00, steps = 200
    13:57:58 [INFO] train episode 1566: reward = 145.00, steps = 145
    13:58:05 [INFO] train episode 1567: reward = 139.00, steps = 139
    13:58:13 [INFO] train episode 1568: reward = 148.00, steps = 148
    13:58:22 [INFO] train episode 1569: reward = 169.00, steps = 169
    13:58:29 [INFO] train episode 1570: reward = 140.00, steps = 140
    13:58:39 [INFO] train episode 1571: reward = 176.00, steps = 176
    13:58:46 [INFO] train episode 1572: reward = 139.00, steps = 139
    13:58:53 [INFO] train episode 1573: reward = 129.00, steps = 129
    13:59:01 [INFO] train episode 1574: reward = 146.00, steps = 146
    13:59:08 [INFO] train episode 1575: reward = 132.00, steps = 132
    13:59:14 [INFO] train episode 1576: reward = 121.00, steps = 121
    13:59:21 [INFO] train episode 1577: reward = 136.00, steps = 136
    13:59:32 [INFO] train episode 1578: reward = 200.00, steps = 200
    13:59:37 [INFO] train episode 1579: reward = 107.00, steps = 107
    13:59:44 [INFO] train episode 1580: reward = 132.00, steps = 132
    13:59:54 [INFO] train episode 1581: reward = 181.00, steps = 181
    13:59:59 [INFO] train episode 1582: reward = 109.00, steps = 109
    14:00:09 [INFO] train episode 1583: reward = 184.00, steps = 184
    14:00:19 [INFO] train episode 1584: reward = 193.00, steps = 193
    14:00:26 [INFO] train episode 1585: reward = 132.00, steps = 132
    14:00:34 [INFO] train episode 1586: reward = 157.00, steps = 157
    14:00:43 [INFO] train episode 1587: reward = 158.00, steps = 158
    14:00:53 [INFO] train episode 1588: reward = 184.00, steps = 184
    14:01:03 [INFO] train episode 1589: reward = 200.00, steps = 200
    14:01:10 [INFO] train episode 1590: reward = 129.00, steps = 129
    14:01:17 [INFO] train episode 1591: reward = 144.00, steps = 144
    14:01:25 [INFO] train episode 1592: reward = 148.00, steps = 148
    14:01:35 [INFO] train episode 1593: reward = 188.00, steps = 188
    14:01:42 [INFO] train episode 1594: reward = 123.00, steps = 123
    14:01:50 [INFO] train episode 1595: reward = 160.00, steps = 160
    14:01:56 [INFO] train episode 1596: reward = 111.00, steps = 111
    14:02:04 [INFO] train episode 1597: reward = 145.00, steps = 145
    14:02:14 [INFO] train episode 1598: reward = 195.00, steps = 195
    14:02:24 [INFO] train episode 1599: reward = 200.00, steps = 200
    14:02:33 [INFO] train episode 1600: reward = 164.00, steps = 164
    14:02:39 [INFO] train episode 1601: reward = 122.00, steps = 122
    14:02:47 [INFO] train episode 1602: reward = 151.00, steps = 151
    14:02:58 [INFO] train episode 1603: reward = 200.00, steps = 200
    14:03:04 [INFO] train episode 1604: reward = 118.00, steps = 118
    14:03:12 [INFO] train episode 1605: reward = 141.00, steps = 141
    14:03:20 [INFO] train episode 1606: reward = 162.00, steps = 162
    14:03:28 [INFO] train episode 1607: reward = 145.00, steps = 145
    14:03:37 [INFO] train episode 1608: reward = 178.00, steps = 178
    14:03:45 [INFO] train episode 1609: reward = 145.00, steps = 145
    14:03:55 [INFO] train episode 1610: reward = 200.00, steps = 200
    14:04:02 [INFO] train episode 1611: reward = 138.00, steps = 138
    14:04:10 [INFO] train episode 1612: reward = 144.00, steps = 144
    14:04:20 [INFO] train episode 1613: reward = 200.00, steps = 200
    14:04:28 [INFO] train episode 1614: reward = 138.00, steps = 138
    14:04:38 [INFO] train episode 1615: reward = 187.00, steps = 187
    14:04:44 [INFO] train episode 1616: reward = 115.00, steps = 115
    14:04:50 [INFO] train episode 1617: reward = 119.00, steps = 119
    14:04:58 [INFO] train episode 1618: reward = 150.00, steps = 150
    14:05:09 [INFO] train episode 1619: reward = 200.00, steps = 200
    14:05:19 [INFO] train episode 1620: reward = 200.00, steps = 200
    14:05:30 [INFO] train episode 1621: reward = 200.00, steps = 200
    14:05:38 [INFO] train episode 1622: reward = 155.00, steps = 155
    14:05:45 [INFO] train episode 1623: reward = 142.00, steps = 142
    14:05:53 [INFO] train episode 1624: reward = 140.00, steps = 140
    14:05:59 [INFO] train episode 1625: reward = 132.00, steps = 132
    14:06:06 [INFO] train episode 1626: reward = 120.00, steps = 120
    14:06:13 [INFO] train episode 1627: reward = 132.00, steps = 132
    14:06:23 [INFO] train episode 1628: reward = 200.00, steps = 200
    14:06:29 [INFO] train episode 1629: reward = 113.00, steps = 113
    14:06:40 [INFO] train episode 1630: reward = 195.00, steps = 195
    14:06:45 [INFO] train episode 1631: reward = 106.00, steps = 106
    14:06:51 [INFO] train episode 1632: reward = 105.00, steps = 105
    14:07:01 [INFO] train episode 1633: reward = 189.00, steps = 189
    14:07:09 [INFO] train episode 1634: reward = 166.00, steps = 166
    14:07:19 [INFO] train episode 1635: reward = 179.00, steps = 179
    14:07:27 [INFO] train episode 1636: reward = 154.00, steps = 154
    14:07:35 [INFO] train episode 1637: reward = 151.00, steps = 151
    14:07:45 [INFO] train episode 1638: reward = 200.00, steps = 200
    14:07:56 [INFO] train episode 1639: reward = 200.00, steps = 200
    14:08:02 [INFO] train episode 1640: reward = 118.00, steps = 118
    14:08:13 [INFO] train episode 1641: reward = 200.00, steps = 200
    14:08:23 [INFO] train episode 1642: reward = 191.00, steps = 191
    14:08:30 [INFO] train episode 1643: reward = 138.00, steps = 138
    14:08:37 [INFO] train episode 1644: reward = 133.00, steps = 133
    14:08:44 [INFO] train episode 1645: reward = 127.00, steps = 127
    14:08:53 [INFO] train episode 1646: reward = 178.00, steps = 178
    14:08:58 [INFO] train episode 1647: reward = 97.00, steps = 97
    14:09:08 [INFO] train episode 1648: reward = 186.00, steps = 186
    14:09:16 [INFO] train episode 1649: reward = 155.00, steps = 155
    14:09:22 [INFO] train episode 1650: reward = 117.00, steps = 117
    14:09:32 [INFO] train episode 1651: reward = 179.00, steps = 179
    14:09:40 [INFO] train episode 1652: reward = 154.00, steps = 154
    14:09:49 [INFO] train episode 1653: reward = 140.00, steps = 140
    14:09:58 [INFO] train episode 1654: reward = 175.00, steps = 175
    14:10:09 [INFO] train episode 1655: reward = 199.00, steps = 199
    14:10:18 [INFO] train episode 1656: reward = 186.00, steps = 186
    14:10:29 [INFO] train episode 1657: reward = 200.00, steps = 200
    14:10:36 [INFO] train episode 1658: reward = 136.00, steps = 136
    14:10:44 [INFO] train episode 1659: reward = 144.00, steps = 144
    14:10:52 [INFO] train episode 1660: reward = 157.00, steps = 157
    14:10:59 [INFO] train episode 1661: reward = 136.00, steps = 136
    14:11:07 [INFO] train episode 1662: reward = 132.00, steps = 132
    14:11:14 [INFO] train episode 1663: reward = 138.00, steps = 138
    14:11:23 [INFO] train episode 1664: reward = 165.00, steps = 165
    14:11:33 [INFO] train episode 1665: reward = 195.00, steps = 195
    14:11:43 [INFO] train episode 1666: reward = 200.00, steps = 200
    14:11:50 [INFO] train episode 1667: reward = 132.00, steps = 132
    14:12:01 [INFO] train episode 1668: reward = 200.00, steps = 200
    14:12:11 [INFO] train episode 1669: reward = 187.00, steps = 187
    14:12:18 [INFO] train episode 1670: reward = 139.00, steps = 139
    14:12:25 [INFO] train episode 1671: reward = 140.00, steps = 140
    14:12:33 [INFO] train episode 1672: reward = 156.00, steps = 156
    14:12:41 [INFO] train episode 1673: reward = 134.00, steps = 134
    14:12:51 [INFO] train episode 1674: reward = 200.00, steps = 200
    14:12:57 [INFO] train episode 1675: reward = 118.00, steps = 118
    14:13:04 [INFO] train episode 1676: reward = 120.00, steps = 120
    14:13:12 [INFO] train episode 1677: reward = 168.00, steps = 168
    14:13:22 [INFO] train episode 1678: reward = 163.00, steps = 163
    14:13:30 [INFO] train episode 1679: reward = 137.00, steps = 137
    14:13:38 [INFO] train episode 1680: reward = 158.00, steps = 158
    14:13:47 [INFO] train episode 1681: reward = 161.00, steps = 161
    14:13:55 [INFO] train episode 1682: reward = 134.00, steps = 134
    14:14:06 [INFO] train episode 1683: reward = 200.00, steps = 200
    14:14:14 [INFO] train episode 1684: reward = 153.00, steps = 153
    14:14:23 [INFO] train episode 1685: reward = 166.00, steps = 166
    14:14:36 [INFO] train episode 1686: reward = 200.00, steps = 200
    14:14:46 [INFO] train episode 1687: reward = 171.00, steps = 171
    14:14:52 [INFO] train episode 1688: reward = 110.00, steps = 110
    14:14:59 [INFO] train episode 1689: reward = 126.00, steps = 126
    14:15:07 [INFO] train episode 1690: reward = 148.00, steps = 148
    14:15:17 [INFO] train episode 1691: reward = 170.00, steps = 170
    14:15:24 [INFO] train episode 1692: reward = 121.00, steps = 121
    14:15:36 [INFO] train episode 1693: reward = 200.00, steps = 200
    14:15:48 [INFO] train episode 1694: reward = 178.00, steps = 178
    14:15:54 [INFO] train episode 1695: reward = 116.00, steps = 116
    14:16:01 [INFO] train episode 1696: reward = 126.00, steps = 126
    14:16:08 [INFO] train episode 1697: reward = 122.00, steps = 122
    14:16:15 [INFO] train episode 1698: reward = 124.00, steps = 124
    14:16:26 [INFO] train episode 1699: reward = 200.00, steps = 200
    14:16:37 [INFO] train episode 1700: reward = 193.00, steps = 193
    14:16:50 [INFO] train episode 1701: reward = 200.00, steps = 200
    14:17:02 [INFO] train episode 1702: reward = 189.00, steps = 189
    14:17:12 [INFO] train episode 1703: reward = 200.00, steps = 200
    14:17:22 [INFO] train episode 1704: reward = 177.00, steps = 177
    14:17:32 [INFO] train episode 1705: reward = 200.00, steps = 200
    14:17:41 [INFO] train episode 1706: reward = 161.00, steps = 161
    14:17:47 [INFO] train episode 1707: reward = 128.00, steps = 128
    14:17:55 [INFO] train episode 1708: reward = 133.00, steps = 133
    14:18:05 [INFO] train episode 1709: reward = 200.00, steps = 200
    14:18:13 [INFO] train episode 1710: reward = 150.00, steps = 150
    14:18:21 [INFO] train episode 1711: reward = 142.00, steps = 142
    14:18:27 [INFO] train episode 1712: reward = 125.00, steps = 125
    14:18:34 [INFO] train episode 1713: reward = 131.00, steps = 131
    14:18:43 [INFO] train episode 1714: reward = 160.00, steps = 160
    14:18:49 [INFO] train episode 1715: reward = 127.00, steps = 127
    14:18:55 [INFO] train episode 1716: reward = 113.00, steps = 113
    14:19:03 [INFO] train episode 1717: reward = 137.00, steps = 137
    14:19:10 [INFO] train episode 1718: reward = 136.00, steps = 136
    14:19:20 [INFO] train episode 1719: reward = 200.00, steps = 200
    14:19:27 [INFO] train episode 1720: reward = 128.00, steps = 128
    14:19:35 [INFO] train episode 1721: reward = 140.00, steps = 140
    14:19:42 [INFO] train episode 1722: reward = 135.00, steps = 135
    14:19:50 [INFO] train episode 1723: reward = 150.00, steps = 150
    14:20:00 [INFO] train episode 1724: reward = 200.00, steps = 200
    14:20:11 [INFO] train episode 1725: reward = 200.00, steps = 200
    14:20:21 [INFO] train episode 1726: reward = 200.00, steps = 200
    14:20:32 [INFO] train episode 1727: reward = 200.00, steps = 200
    14:20:39 [INFO] train episode 1728: reward = 127.00, steps = 127
    14:20:47 [INFO] train episode 1729: reward = 147.00, steps = 147
    14:20:54 [INFO] train episode 1730: reward = 134.00, steps = 134
    14:21:03 [INFO] train episode 1731: reward = 178.00, steps = 178
    14:21:10 [INFO] train episode 1732: reward = 133.00, steps = 133
    14:21:18 [INFO] train episode 1733: reward = 136.00, steps = 136
    14:21:27 [INFO] train episode 1734: reward = 173.00, steps = 173
    14:21:35 [INFO] train episode 1735: reward = 153.00, steps = 153
    14:21:42 [INFO] train episode 1736: reward = 134.00, steps = 134
    14:21:52 [INFO] train episode 1737: reward = 193.00, steps = 193
    14:21:59 [INFO] train episode 1738: reward = 136.00, steps = 136
    14:22:08 [INFO] train episode 1739: reward = 164.00, steps = 164
    14:22:14 [INFO] train episode 1740: reward = 104.00, steps = 104
    14:22:21 [INFO] train episode 1741: reward = 129.00, steps = 129
    14:22:28 [INFO] train episode 1742: reward = 139.00, steps = 139
    14:22:37 [INFO] train episode 1743: reward = 183.00, steps = 183
    14:22:44 [INFO] train episode 1744: reward = 124.00, steps = 124
    14:22:52 [INFO] train episode 1745: reward = 144.00, steps = 144
    14:22:58 [INFO] train episode 1746: reward = 111.00, steps = 111
    14:23:04 [INFO] train episode 1747: reward = 123.00, steps = 123
    14:23:15 [INFO] train episode 1748: reward = 195.00, steps = 195
    14:23:24 [INFO] train episode 1749: reward = 167.00, steps = 167
    14:23:30 [INFO] train episode 1750: reward = 113.00, steps = 113
    14:23:39 [INFO] train episode 1751: reward = 155.00, steps = 155
    14:23:45 [INFO] train episode 1752: reward = 110.00, steps = 110
    14:23:55 [INFO] train episode 1753: reward = 189.00, steps = 189
    14:24:04 [INFO] train episode 1754: reward = 156.00, steps = 156
    14:24:12 [INFO] train episode 1755: reward = 135.00, steps = 135
    14:24:22 [INFO] train episode 1756: reward = 188.00, steps = 188
    14:24:31 [INFO] train episode 1757: reward = 162.00, steps = 162
    14:24:39 [INFO] train episode 1758: reward = 157.00, steps = 157
    14:24:45 [INFO] train episode 1759: reward = 112.00, steps = 112
    14:24:53 [INFO] train episode 1760: reward = 158.00, steps = 158
    14:25:00 [INFO] train episode 1761: reward = 131.00, steps = 131
    14:25:07 [INFO] train episode 1762: reward = 136.00, steps = 136
    14:25:17 [INFO] train episode 1763: reward = 182.00, steps = 182
    14:25:25 [INFO] train episode 1764: reward = 158.00, steps = 158
    14:25:36 [INFO] train episode 1765: reward = 200.00, steps = 200
    14:25:43 [INFO] train episode 1766: reward = 134.00, steps = 134
    14:25:54 [INFO] train episode 1767: reward = 200.00, steps = 200
    14:26:00 [INFO] train episode 1768: reward = 120.00, steps = 120
    14:26:08 [INFO] train episode 1769: reward = 155.00, steps = 155
    14:26:19 [INFO] train episode 1770: reward = 200.00, steps = 200
    14:26:26 [INFO] train episode 1771: reward = 144.00, steps = 144
    14:26:34 [INFO] train episode 1772: reward = 127.00, steps = 127
    14:26:48 [INFO] train episode 1773: reward = 200.00, steps = 200
    14:27:00 [INFO] train episode 1774: reward = 200.00, steps = 200
    14:27:11 [INFO] train episode 1775: reward = 200.00, steps = 200
    14:27:18 [INFO] train episode 1776: reward = 116.00, steps = 116
    14:27:29 [INFO] train episode 1777: reward = 200.00, steps = 200
    14:27:40 [INFO] train episode 1778: reward = 187.00, steps = 187
    14:27:47 [INFO] train episode 1779: reward = 118.00, steps = 118
    14:27:59 [INFO] train episode 1780: reward = 200.00, steps = 200
    14:28:07 [INFO] train episode 1781: reward = 130.00, steps = 130
    14:28:17 [INFO] train episode 1782: reward = 161.00, steps = 161
    14:28:26 [INFO] train episode 1783: reward = 156.00, steps = 156
    14:28:38 [INFO] train episode 1784: reward = 200.00, steps = 200
    14:28:47 [INFO] train episode 1785: reward = 134.00, steps = 134
    14:28:58 [INFO] train episode 1786: reward = 200.00, steps = 200
    14:29:09 [INFO] train episode 1787: reward = 167.00, steps = 167
    14:29:22 [INFO] train episode 1788: reward = 200.00, steps = 200
    14:29:30 [INFO] train episode 1789: reward = 149.00, steps = 149
    14:29:39 [INFO] train episode 1790: reward = 141.00, steps = 141
    14:29:50 [INFO] train episode 1791: reward = 164.00, steps = 164
    14:30:03 [INFO] train episode 1792: reward = 200.00, steps = 200
    14:30:11 [INFO] train episode 1793: reward = 140.00, steps = 140
    14:30:23 [INFO] train episode 1794: reward = 200.00, steps = 200
    14:30:30 [INFO] train episode 1795: reward = 125.00, steps = 125
    14:30:40 [INFO] train episode 1796: reward = 194.00, steps = 194
    14:30:51 [INFO] train episode 1797: reward = 200.00, steps = 200
    14:30:59 [INFO] train episode 1798: reward = 149.00, steps = 149
    14:31:06 [INFO] train episode 1799: reward = 134.00, steps = 134
    14:31:13 [INFO] train episode 1800: reward = 131.00, steps = 131
    14:31:24 [INFO] train episode 1801: reward = 198.00, steps = 198
    14:31:34 [INFO] train episode 1802: reward = 148.00, steps = 148
    14:31:45 [INFO] train episode 1803: reward = 199.00, steps = 199
    14:31:54 [INFO] train episode 1804: reward = 142.00, steps = 142
    14:32:03 [INFO] train episode 1805: reward = 170.00, steps = 170
    14:32:10 [INFO] train episode 1806: reward = 127.00, steps = 127
    14:32:23 [INFO] train episode 1807: reward = 200.00, steps = 200
    14:32:33 [INFO] train episode 1808: reward = 160.00, steps = 160
    14:32:43 [INFO] train episode 1809: reward = 170.00, steps = 170
    14:32:54 [INFO] train episode 1810: reward = 200.00, steps = 200
    14:33:06 [INFO] train episode 1811: reward = 200.00, steps = 200
    14:33:17 [INFO] train episode 1812: reward = 182.00, steps = 182
    14:33:27 [INFO] train episode 1813: reward = 176.00, steps = 176
    14:33:37 [INFO] train episode 1814: reward = 179.00, steps = 179
    14:33:45 [INFO] train episode 1815: reward = 144.00, steps = 144
    14:33:57 [INFO] train episode 1816: reward = 200.00, steps = 200
    14:34:05 [INFO] train episode 1817: reward = 142.00, steps = 142
    14:34:14 [INFO] train episode 1818: reward = 161.00, steps = 161
    14:34:24 [INFO] train episode 1819: reward = 176.00, steps = 176
    14:34:35 [INFO] train episode 1820: reward = 192.00, steps = 192
    14:34:46 [INFO] train episode 1821: reward = 193.00, steps = 193
    14:34:55 [INFO] train episode 1822: reward = 159.00, steps = 159
    14:35:07 [INFO] train episode 1823: reward = 200.00, steps = 200
    14:35:16 [INFO] train episode 1824: reward = 158.00, steps = 158
    14:35:23 [INFO] train episode 1825: reward = 128.00, steps = 128
    14:35:31 [INFO] train episode 1826: reward = 135.00, steps = 135
    14:35:38 [INFO] train episode 1827: reward = 141.00, steps = 141
    14:35:50 [INFO] train episode 1828: reward = 200.00, steps = 200
    14:36:01 [INFO] train episode 1829: reward = 200.00, steps = 200
    14:36:12 [INFO] train episode 1830: reward = 193.00, steps = 193
    14:36:24 [INFO] train episode 1831: reward = 200.00, steps = 200
    14:36:32 [INFO] train episode 1832: reward = 150.00, steps = 150
    14:36:40 [INFO] train episode 1833: reward = 148.00, steps = 148
    14:36:49 [INFO] train episode 1834: reward = 156.00, steps = 156
    14:36:57 [INFO] train episode 1835: reward = 137.00, steps = 137
    14:37:08 [INFO] train episode 1836: reward = 186.00, steps = 186
    14:37:17 [INFO] train episode 1837: reward = 165.00, steps = 165
    14:37:25 [INFO] train episode 1838: reward = 141.00, steps = 141
    14:37:32 [INFO] train episode 1839: reward = 126.00, steps = 126
    14:37:41 [INFO] train episode 1840: reward = 165.00, steps = 165
    14:37:52 [INFO] train episode 1841: reward = 200.00, steps = 200
    14:38:04 [INFO] train episode 1842: reward = 200.00, steps = 200
    14:38:13 [INFO] train episode 1843: reward = 180.00, steps = 180
    14:38:25 [INFO] train episode 1844: reward = 198.00, steps = 198
    14:38:36 [INFO] train episode 1845: reward = 188.00, steps = 188
    14:38:43 [INFO] train episode 1846: reward = 125.00, steps = 125
    14:38:50 [INFO] train episode 1847: reward = 119.00, steps = 119
    14:38:58 [INFO] train episode 1848: reward = 151.00, steps = 151
    14:39:06 [INFO] train episode 1849: reward = 131.00, steps = 131
    14:39:15 [INFO] train episode 1850: reward = 174.00, steps = 174
    14:39:25 [INFO] train episode 1851: reward = 165.00, steps = 165
    14:39:33 [INFO] train episode 1852: reward = 158.00, steps = 158
    14:39:45 [INFO] train episode 1853: reward = 200.00, steps = 200
    14:39:53 [INFO] train episode 1854: reward = 151.00, steps = 151
    14:40:01 [INFO] train episode 1855: reward = 144.00, steps = 144
    14:40:12 [INFO] train episode 1856: reward = 200.00, steps = 200
    14:40:21 [INFO] train episode 1857: reward = 161.00, steps = 161
    14:40:29 [INFO] train episode 1858: reward = 141.00, steps = 141
    14:40:40 [INFO] train episode 1859: reward = 200.00, steps = 200
    14:40:49 [INFO] train episode 1860: reward = 150.00, steps = 150
    14:40:59 [INFO] train episode 1861: reward = 169.00, steps = 169
    14:41:08 [INFO] train episode 1862: reward = 144.00, steps = 144
    14:41:19 [INFO] train episode 1863: reward = 200.00, steps = 200
    14:41:28 [INFO] train episode 1864: reward = 168.00, steps = 168
    14:41:39 [INFO] train episode 1865: reward = 200.00, steps = 200
    14:41:49 [INFO] train episode 1866: reward = 200.00, steps = 200
    14:41:57 [INFO] train episode 1867: reward = 145.00, steps = 145
    14:42:07 [INFO] train episode 1868: reward = 191.00, steps = 191
    14:42:17 [INFO] train episode 1869: reward = 177.00, steps = 177
    14:42:25 [INFO] train episode 1870: reward = 149.00, steps = 149
    14:42:36 [INFO] train episode 1871: reward = 200.00, steps = 200
    14:42:45 [INFO] train episode 1872: reward = 164.00, steps = 164
    14:42:55 [INFO] train episode 1873: reward = 160.00, steps = 160
    14:43:04 [INFO] train episode 1874: reward = 165.00, steps = 165
    14:43:15 [INFO] train episode 1875: reward = 200.00, steps = 200
    14:43:26 [INFO] train episode 1876: reward = 200.00, steps = 200
    14:43:38 [INFO] train episode 1877: reward = 200.00, steps = 200
    14:43:49 [INFO] train episode 1878: reward = 196.00, steps = 196
    14:44:01 [INFO] train episode 1879: reward = 195.00, steps = 195
    14:44:13 [INFO] train episode 1880: reward = 200.00, steps = 200
    14:44:23 [INFO] train episode 1881: reward = 177.00, steps = 177
    14:44:31 [INFO] train episode 1882: reward = 144.00, steps = 144
    14:44:39 [INFO] train episode 1883: reward = 146.00, steps = 146
    14:44:50 [INFO] train episode 1884: reward = 179.00, steps = 179
    14:45:02 [INFO] train episode 1885: reward = 176.00, steps = 176
    14:45:13 [INFO] train episode 1886: reward = 198.00, steps = 198
    14:45:24 [INFO] train episode 1887: reward = 200.00, steps = 200
    14:45:34 [INFO] train episode 1888: reward = 171.00, steps = 171
    14:45:43 [INFO] train episode 1889: reward = 164.00, steps = 164
    14:45:54 [INFO] train episode 1890: reward = 179.00, steps = 179
    14:46:03 [INFO] train episode 1891: reward = 156.00, steps = 156
    14:46:12 [INFO] train episode 1892: reward = 150.00, steps = 150
    14:46:24 [INFO] train episode 1893: reward = 197.00, steps = 197
    14:46:31 [INFO] train episode 1894: reward = 139.00, steps = 139
    14:46:43 [INFO] train episode 1895: reward = 200.00, steps = 200
    14:46:54 [INFO] train episode 1896: reward = 196.00, steps = 196
    14:47:04 [INFO] train episode 1897: reward = 200.00, steps = 200
    14:47:15 [INFO] train episode 1898: reward = 200.00, steps = 200
    14:47:22 [INFO] train episode 1899: reward = 126.00, steps = 126
    14:47:29 [INFO] train episode 1900: reward = 134.00, steps = 134
    14:47:40 [INFO] train episode 1901: reward = 192.00, steps = 192
    14:47:50 [INFO] train episode 1902: reward = 168.00, steps = 168
    14:47:59 [INFO] train episode 1903: reward = 160.00, steps = 160
    14:48:10 [INFO] train episode 1904: reward = 193.00, steps = 193
    14:48:18 [INFO] train episode 1905: reward = 160.00, steps = 160
    14:48:28 [INFO] train episode 1906: reward = 186.00, steps = 186
    14:48:35 [INFO] train episode 1907: reward = 126.00, steps = 126
    14:48:44 [INFO] train episode 1908: reward = 168.00, steps = 168
    14:48:56 [INFO] train episode 1909: reward = 200.00, steps = 200
    14:49:07 [INFO] train episode 1910: reward = 177.00, steps = 177
    14:49:16 [INFO] train episode 1911: reward = 160.00, steps = 160
    14:49:27 [INFO] train episode 1912: reward = 200.00, steps = 200
    14:49:34 [INFO] train episode 1913: reward = 129.00, steps = 129
    14:49:43 [INFO] train episode 1914: reward = 161.00, steps = 161
    14:49:54 [INFO] train episode 1915: reward = 200.00, steps = 200
    14:50:03 [INFO] train episode 1916: reward = 144.00, steps = 144
    14:50:13 [INFO] train episode 1917: reward = 171.00, steps = 171
    14:50:24 [INFO] train episode 1918: reward = 169.00, steps = 169
    14:50:33 [INFO] train episode 1919: reward = 154.00, steps = 154
    14:50:42 [INFO] train episode 1920: reward = 155.00, steps = 155
    14:50:54 [INFO] train episode 1921: reward = 200.00, steps = 200
    14:51:03 [INFO] train episode 1922: reward = 150.00, steps = 150
    14:51:14 [INFO] train episode 1923: reward = 189.00, steps = 189
    14:51:28 [INFO] train episode 1924: reward = 200.00, steps = 200
    14:51:40 [INFO] train episode 1925: reward = 179.00, steps = 179
    14:51:52 [INFO] train episode 1926: reward = 159.00, steps = 159
    14:52:05 [INFO] train episode 1927: reward = 170.00, steps = 170
    14:52:20 [INFO] train episode 1928: reward = 200.00, steps = 200
    14:52:34 [INFO] train episode 1929: reward = 185.00, steps = 185
    14:52:46 [INFO] train episode 1930: reward = 164.00, steps = 164
    14:52:57 [INFO] train episode 1931: reward = 200.00, steps = 200
    14:53:08 [INFO] train episode 1932: reward = 186.00, steps = 186
    14:53:17 [INFO] train episode 1933: reward = 166.00, steps = 166
    14:53:28 [INFO] train episode 1934: reward = 192.00, steps = 192
    14:53:39 [INFO] train episode 1935: reward = 200.00, steps = 200
    14:53:47 [INFO] train episode 1936: reward = 142.00, steps = 142
    14:53:57 [INFO] train episode 1937: reward = 177.00, steps = 177
    14:54:08 [INFO] train episode 1938: reward = 191.00, steps = 191
    14:54:19 [INFO] train episode 1939: reward = 200.00, steps = 200
    14:54:30 [INFO] train episode 1940: reward = 200.00, steps = 200
    14:54:41 [INFO] train episode 1941: reward = 200.00, steps = 200
    14:54:50 [INFO] train episode 1942: reward = 174.00, steps = 174
    14:55:00 [INFO] train episode 1943: reward = 175.00, steps = 175
    14:55:11 [INFO] train episode 1944: reward = 200.00, steps = 200
    14:55:19 [INFO] train episode 1945: reward = 160.00, steps = 160
    14:55:29 [INFO] train episode 1946: reward = 167.00, steps = 167
    14:55:36 [INFO] train episode 1947: reward = 134.00, steps = 134
    14:55:46 [INFO] train episode 1948: reward = 174.00, steps = 174
    14:55:56 [INFO] train episode 1949: reward = 200.00, steps = 200
    14:56:07 [INFO] train episode 1950: reward = 200.00, steps = 200
    14:56:16 [INFO] train episode 1951: reward = 159.00, steps = 159
    14:56:24 [INFO] train episode 1952: reward = 153.00, steps = 153
    14:56:33 [INFO] train episode 1953: reward = 178.00, steps = 178
    14:56:43 [INFO] train episode 1954: reward = 162.00, steps = 162
    14:56:53 [INFO] train episode 1955: reward = 171.00, steps = 171
    14:57:04 [INFO] train episode 1956: reward = 200.00, steps = 200
    14:57:16 [INFO] train episode 1957: reward = 200.00, steps = 200
    14:57:25 [INFO] train episode 1958: reward = 160.00, steps = 160
    14:57:36 [INFO] train episode 1959: reward = 200.00, steps = 200
    14:57:44 [INFO] train episode 1960: reward = 145.00, steps = 145
    14:57:53 [INFO] train episode 1961: reward = 162.00, steps = 162
    14:58:03 [INFO] train episode 1962: reward = 200.00, steps = 200
    14:58:12 [INFO] train episode 1963: reward = 155.00, steps = 155
    14:58:20 [INFO] train episode 1964: reward = 159.00, steps = 159
    14:58:30 [INFO] train episode 1965: reward = 174.00, steps = 174
    14:58:42 [INFO] train episode 1966: reward = 200.00, steps = 200
    14:58:53 [INFO] train episode 1967: reward = 199.00, steps = 199
    14:59:03 [INFO] train episode 1968: reward = 200.00, steps = 200
    14:59:14 [INFO] train episode 1969: reward = 200.00, steps = 200
    14:59:22 [INFO] train episode 1970: reward = 143.00, steps = 143
    14:59:32 [INFO] train episode 1971: reward = 200.00, steps = 200
    14:59:43 [INFO] train episode 1972: reward = 200.00, steps = 200
    14:59:54 [INFO] train episode 1973: reward = 200.00, steps = 200
    15:00:05 [INFO] train episode 1974: reward = 200.00, steps = 200
    15:00:16 [INFO] train episode 1975: reward = 200.00, steps = 200
    15:00:26 [INFO] train episode 1976: reward = 156.00, steps = 156
    15:00:39 [INFO] train episode 1977: reward = 200.00, steps = 200
    15:00:47 [INFO] train episode 1978: reward = 141.00, steps = 141
    15:00:57 [INFO] train episode 1979: reward = 192.00, steps = 192
    15:01:09 [INFO] train episode 1980: reward = 199.00, steps = 199
    15:01:17 [INFO] train episode 1981: reward = 152.00, steps = 152
    15:01:26 [INFO] train episode 1982: reward = 166.00, steps = 166
    15:01:37 [INFO] train episode 1983: reward = 200.00, steps = 200
    15:01:48 [INFO] train episode 1984: reward = 200.00, steps = 200
    15:01:58 [INFO] train episode 1985: reward = 200.00, steps = 200
    15:02:09 [INFO] train episode 1986: reward = 198.00, steps = 198
    15:02:21 [INFO] train episode 1987: reward = 200.00, steps = 200
    15:02:31 [INFO] train episode 1988: reward = 169.00, steps = 169
    15:02:45 [INFO] train episode 1989: reward = 200.00, steps = 200
    15:02:59 [INFO] train episode 1990: reward = 200.00, steps = 200
    15:03:11 [INFO] train episode 1991: reward = 200.00, steps = 200
    15:03:22 [INFO] train episode 1992: reward = 194.00, steps = 194
    15:03:22 [INFO] ==== test ====
    15:03:34 [INFO] test episode 0: reward = 200.00, steps = 200
    15:03:45 [INFO] test episode 1: reward = 200.00, steps = 200
    15:03:58 [INFO] test episode 2: reward = 196.00, steps = 196
    15:04:08 [INFO] test episode 3: reward = 155.00, steps = 155
    15:04:21 [INFO] test episode 4: reward = 172.00, steps = 172
    15:04:34 [INFO] test episode 5: reward = 189.00, steps = 189
    15:04:46 [INFO] test episode 6: reward = 200.00, steps = 200
    15:04:58 [INFO] test episode 7: reward = 200.00, steps = 200
    15:05:10 [INFO] test episode 8: reward = 200.00, steps = 200
    15:05:22 [INFO] test episode 9: reward = 200.00, steps = 200
    15:05:33 [INFO] test episode 10: reward = 200.00, steps = 200
    15:05:45 [INFO] test episode 11: reward = 200.00, steps = 200
    15:05:56 [INFO] test episode 12: reward = 200.00, steps = 200
    15:06:07 [INFO] test episode 13: reward = 200.00, steps = 200
    15:06:18 [INFO] test episode 14: reward = 186.00, steps = 186
    15:06:29 [INFO] test episode 15: reward = 200.00, steps = 200
    15:06:40 [INFO] test episode 16: reward = 200.00, steps = 200
    15:06:56 [INFO] test episode 17: reward = 200.00, steps = 200
    15:07:09 [INFO] test episode 18: reward = 186.00, steps = 186
    15:07:22 [INFO] test episode 19: reward = 174.00, steps = 174
    15:07:34 [INFO] test episode 20: reward = 175.00, steps = 175
    15:07:48 [INFO] test episode 21: reward = 195.00, steps = 195
    15:08:01 [INFO] test episode 22: reward = 200.00, steps = 200
    15:08:12 [INFO] test episode 23: reward = 200.00, steps = 200
    15:08:24 [INFO] test episode 24: reward = 176.00, steps = 176
    15:08:34 [INFO] test episode 25: reward = 200.00, steps = 200
    15:08:46 [INFO] test episode 26: reward = 200.00, steps = 200
    15:08:57 [INFO] test episode 27: reward = 200.00, steps = 200
    15:09:08 [INFO] test episode 28: reward = 200.00, steps = 200
    15:09:19 [INFO] test episode 29: reward = 187.00, steps = 187
    15:09:31 [INFO] test episode 30: reward = 198.00, steps = 198
    15:09:41 [INFO] test episode 31: reward = 200.00, steps = 200
    15:09:51 [INFO] test episode 32: reward = 200.00, steps = 200
    15:10:02 [INFO] test episode 33: reward = 200.00, steps = 200
    15:10:12 [INFO] test episode 34: reward = 196.00, steps = 196
    15:10:21 [INFO] test episode 35: reward = 176.00, steps = 176
    15:10:32 [INFO] test episode 36: reward = 200.00, steps = 200
    15:10:42 [INFO] test episode 37: reward = 200.00, steps = 200
    15:10:53 [INFO] test episode 38: reward = 200.00, steps = 200
    15:11:03 [INFO] test episode 39: reward = 172.00, steps = 172
    15:11:13 [INFO] test episode 40: reward = 200.00, steps = 200
    15:11:24 [INFO] test episode 41: reward = 200.00, steps = 200
    15:11:34 [INFO] test episode 42: reward = 200.00, steps = 200
    15:11:45 [INFO] test episode 43: reward = 200.00, steps = 200
    15:11:53 [INFO] test episode 44: reward = 172.00, steps = 172
    15:12:04 [INFO] test episode 45: reward = 200.00, steps = 200
    15:12:15 [INFO] test episode 46: reward = 200.00, steps = 200
    15:12:25 [INFO] test episode 47: reward = 200.00, steps = 200
    15:12:34 [INFO] test episode 48: reward = 177.00, steps = 177
    15:12:46 [INFO] test episode 49: reward = 200.00, steps = 200
    15:12:56 [INFO] test episode 50: reward = 200.00, steps = 200
    15:13:07 [INFO] test episode 51: reward = 200.00, steps = 200
    15:13:17 [INFO] test episode 52: reward = 200.00, steps = 200
    15:13:28 [INFO] test episode 53: reward = 200.00, steps = 200
    15:13:38 [INFO] test episode 54: reward = 200.00, steps = 200
    15:13:48 [INFO] test episode 55: reward = 200.00, steps = 200
    15:13:59 [INFO] test episode 56: reward = 200.00, steps = 200
    15:14:09 [INFO] test episode 57: reward = 200.00, steps = 200
    15:14:20 [INFO] test episode 58: reward = 200.00, steps = 200
    15:14:30 [INFO] test episode 59: reward = 200.00, steps = 200
    15:14:41 [INFO] test episode 60: reward = 200.00, steps = 200
    15:14:51 [INFO] test episode 61: reward = 200.00, steps = 200
    15:15:02 [INFO] test episode 62: reward = 200.00, steps = 200
    15:15:12 [INFO] test episode 63: reward = 200.00, steps = 200
    15:15:23 [INFO] test episode 64: reward = 200.00, steps = 200
    15:15:33 [INFO] test episode 65: reward = 200.00, steps = 200
    15:15:44 [INFO] test episode 66: reward = 200.00, steps = 200
    15:15:54 [INFO] test episode 67: reward = 200.00, steps = 200
    15:16:04 [INFO] test episode 68: reward = 200.00, steps = 200
    15:16:15 [INFO] test episode 69: reward = 200.00, steps = 200
    15:16:24 [INFO] test episode 70: reward = 181.00, steps = 181
    15:16:35 [INFO] test episode 71: reward = 200.00, steps = 200
    15:16:45 [INFO] test episode 72: reward = 200.00, steps = 200
    15:16:56 [INFO] test episode 73: reward = 200.00, steps = 200
    15:17:06 [INFO] test episode 74: reward = 200.00, steps = 200
    15:17:15 [INFO] test episode 75: reward = 173.00, steps = 173
    15:17:26 [INFO] test episode 76: reward = 200.00, steps = 200
    15:17:36 [INFO] test episode 77: reward = 200.00, steps = 200
    15:17:46 [INFO] test episode 78: reward = 200.00, steps = 200
    15:17:57 [INFO] test episode 79: reward = 200.00, steps = 200
    15:18:07 [INFO] test episode 80: reward = 200.00, steps = 200
    15:18:17 [INFO] test episode 81: reward = 186.00, steps = 186
    15:18:27 [INFO] test episode 82: reward = 200.00, steps = 200
    15:18:37 [INFO] test episode 83: reward = 187.00, steps = 187
    15:18:48 [INFO] test episode 84: reward = 200.00, steps = 200
    15:18:58 [INFO] test episode 85: reward = 200.00, steps = 200
    15:19:09 [INFO] test episode 86: reward = 200.00, steps = 200
    15:19:19 [INFO] test episode 87: reward = 200.00, steps = 200
    15:19:30 [INFO] test episode 88: reward = 200.00, steps = 200
    15:19:40 [INFO] test episode 89: reward = 200.00, steps = 200
    15:19:51 [INFO] test episode 90: reward = 200.00, steps = 200
    15:20:01 [INFO] test episode 91: reward = 200.00, steps = 200
    15:20:11 [INFO] test episode 92: reward = 197.00, steps = 197
    15:20:22 [INFO] test episode 93: reward = 200.00, steps = 200
    15:20:32 [INFO] test episode 94: reward = 200.00, steps = 200
    15:20:43 [INFO] test episode 95: reward = 200.00, steps = 200
    15:20:53 [INFO] test episode 96: reward = 200.00, steps = 200
    15:21:03 [INFO] test episode 97: reward = 189.00, steps = 189
    15:21:14 [INFO] test episode 98: reward = 200.00, steps = 200
    15:21:24 [INFO] test episode 99: reward = 200.00, steps = 200
    15:21:24 [INFO] average episode reward = 195.95 ± 8.97

In [5]:

    env.close()