# Use QR-DQN to Play Pong¶

TensorFlow version

In [1]:

    %matplotlib inline

    import copy
    import logging
    import itertools
    import sys

    import numpy as np
    np.random.seed(0)
    import pandas as pd
    import gym
    from gym.wrappers.atari_preprocessing import AtariPreprocessing
    from gym.wrappers.frame_stack import FrameStack
    import matplotlib.pyplot as plt
    import tensorflow.compat.v2 as tf
    tf.random.set_seed(0)
    from tensorflow import keras
    from tensorflow import nn
    from tensorflow import optimizers
    from tensorflow import losses
    from tensorflow.keras import layers
    from tensorflow.keras import models

    logging.basicConfig(level=logging.INFO,
            format='%(asctime)s [%(levelname)s] %(message)s',
            stream=sys.stdout, datefmt='%H:%M:%S')

Environment

In [2]:

    env = gym.make('PongNoFrameskip-v4')
    env = FrameStack(AtariPreprocessing(env), num_stack=4)
    for key in vars(env):
        logging.info('%s: %s', key, vars(env)[key])
    for key in vars(env.spec):
        logging.info('%s: %s', key, vars(env.spec)[key])

    00:07:33 [INFO] env: <AtariPreprocessing<TimeLimit<AtariEnv<PongNoFrameskip-v4>>>>
    00:07:33 [INFO] action_space: Discrete(6)
    00:07:33 [INFO] observation_space: Box([[[0 0 0 ... 0 0 0]
      [0 0 0 ... 0 0 0]
      [0 0 0 ... 0 0 0]
      ...
      [0 0 0 ... 0 0 0]
      [0 0 0 ... 0 0 0]
      [0 0 0 ... 0 0 0]]

     [[0 0 0 ... 0 0 0]
      [0 0 0 ... 0 0 0]
      [0 0 0 ... 0 0 0]
      ...
      [0 0 0 ... 0 0 0]
      [0 0 0 ... 0 0 0]
      [0 0 0 ... 0 0 0]]

     [[0 0 0 ... 0 0 0]
      [0 0 0 ... 0 0 0]
      [0 0 0 ... 0 0 0]
      ...
      [0 0 0 ... 0 0 0]
      [0 0 0 ... 0 0 0]
      [0 0 0 ... 0 0 0]]

     [[0 0 0 ... 0 0 0]
      [0 0 0 ... 0 0 0]
      [0 0 0 ... 0 0 0]
      ...
      [0 0 0 ... 0 0 0]
      [0 0 0 ... 0 0 0]
      [0 0 0 ... 0 0 0]]], [[[255 255 255 ... 255 255 255]
      [255 255 255 ... 255 255 255]
      [255 255 255 ... 255 255 255]
      ...
      [255 255 255 ... 255 255 255]
      [255 255 255 ... 255 255 255]
      [255 255 255 ... 255 255 255]]

     [[255 255 255 ... 255 255 255]
      [255 255 255 ... 255 255 255]
      [255 255 255 ... 255 255 255]
      ...
      [255 255 255 ... 255 255 255]
      [255 255 255 ... 255 255 255]
      [255 255 255 ... 255 255 255]]

     [[255 255 255 ... 255 255 255]
      [255 255 255 ... 255 255 255]
      [255 255 255 ... 255 255 255]
      ...
      [255 255 255 ... 255 255 255]
      [255 255 255 ... 255 255 255]
      [255 255 255 ... 255 255 255]]

     [[255 255 255 ... 255 255 255]
      [255 255 255 ... 255 255 255]
      [255 255 255 ... 255 255 255]
      ...
      [255 255 255 ... 255 255 255]
      [255 255 255 ... 255 255 255]
      [255 255 255 ... 255 255 255]]], (4, 84, 84), uint8)
    00:07:33 [INFO] reward_range: (-inf, inf)
    00:07:33 [INFO] metadata: {'render.modes': ['human', 'rgb_array']}
    00:07:33 [INFO] num_stack: 4
    00:07:33 [INFO] lz4_compress: False
    00:07:33 [INFO] frames: deque([], maxlen=4)
    00:07:33 [INFO] id: PongNoFrameskip-v4
    00:07:33 [INFO] entry_point: gym.envs.atari:AtariEnv
    00:07:33 [INFO] reward_threshold: None
    00:07:33 [INFO] nondeterministic: False
    00:07:33 [INFO] max_episode_steps: 400000
    00:07:33 [INFO] _kwargs: {'game': 'pong', 'obs_type': 'image', 'frameskip': 1}
    00:07:33 [INFO] _env_name: PongNoFrameskip

Agent

In [3]:

    class DQNReplayer:
        def __init__(self, capacity):
            self.memory = pd.DataFrame(index=range(capacity),
                    columns=['state', 'action', 'reward', 'next_state', 'terminated'])
            self.i = 0
            self.count = 0
            self.capacity = capacity

        def store(self, *args):
            self.memory.loc[self.i] = np.asarray(args, dtype=object)
            self.i = (self.i + 1) % self.capacity
            self.count = min(self.count + 1, self.capacity)

        def sample(self, size):
            indices = np.random.choice(self.count, size=size)
            return (np.stack(self.memory.loc[indices, field]) for field in
                    self.memory.columns)

In [4]:

    class QRDQNAgent:
        def __init__(self, env):
            self.action_n = env.action_space.n
            self.gamma = 0.99
            self.epsilon = 1.

            self.replayer = DQNReplayer(capacity=100000)

            quantile_count = 64
            self.cumprob_tensor = tf.range(1 / (2 * quantile_count),
                    1, 1 / quantile_count)[np.newaxis, :, np.newaxis]

            self.evaluate_net = self.build_net(self.action_n, quantile_count)
            self.target_net = models.clone_model(self.evaluate_net)

        def build_net(self, action_n, quantile_count):
            net = keras.Sequential([
                    layers.Permute((2, 3, 1), input_shape=(4, 84, 84)),
                    layers.Conv2D(32, kernel_size=8, strides=4, activation=nn.relu),
                    layers.Conv2D(64, kernel_size=4, strides=2, activation=nn.relu),
                    layers.Conv2D(64, kernel_size=3, strides=1, activation=nn.relu),
                    layers.Flatten(),
                    layers.Dense(512, activation=nn.relu),
                    layers.Dense(action_n * quantile_count),
                    layers.Reshape((action_n, quantile_count))])
            optimizer = optimizers.Adam(0.0001)
            net.compile(optimizer=optimizer)
            return net

        def reset(self, mode=None):
            self.mode = mode
            if mode == 'train':
                self.trajectory = []

        def step(self, observation, reward, terminated):
            state_tensor = tf.convert_to_tensor(np.array(observation)[np.newaxis],
                    dtype=tf.float32)
            q_component_tensor = self.evaluate_net(state_tensor)
            q_tensor = tf.reduce_mean(q_component_tensor, axis=2)
            action_tensor = tf.math.argmax(q_tensor, axis=1)
            actions = action_tensor.numpy()
            action = actions[0]
            if self.mode == 'train':
                if np.random.rand() < self.epsilon:
                    action = np.random.randint(0, self.action_n)

                self.trajectory += [observation, reward, terminated, action]
                if len(self.trajectory) >= 8:
                    state, _, _, act, next_state, reward, terminated, _ = \
                            self.trajectory[-8:]
                    self.replayer.store(state, act, reward, next_state, terminated)
                if self.replayer.count >= 1024 and self.replayer.count % 10 == 0:
                    self.learn()
            return action

        def close(self):
            pass

        def update_net(self, target_net, evaluate_net, learning_rate=0.005):
            average_weights = [(1. - learning_rate) * t + learning_rate * e for t, e
                    in zip(target_net.get_weights(), evaluate_net.get_weights())]
            target_net.set_weights(average_weights)

        def learn(self):
            # replay
            batch_size = 32
            states, actions, rewards, next_states, terminateds = \
                    self.replayer.sample(batch_size)
            state_tensor = tf.convert_to_tensor(states, dtype=tf.float32)
            reward_tensor = tf.convert_to_tensor(rewards[:, np.newaxis],
                    dtype=tf.float32)
            terminated_tensor = tf.convert_to_tensor(terminateds[:, np.newaxis],
                    dtype=tf.float32)
            next_state_tensor = tf.convert_to_tensor(next_states, dtype=tf.float32)

            # compute target
            next_q_component_tensor = self.evaluate_net(next_state_tensor)
            next_q_tensor = tf.reduce_mean(next_q_component_tensor, axis=2)
            next_action_tensor = tf.math.argmax(next_q_tensor, axis=1)
            next_actions = next_action_tensor.numpy()
            all_next_q_quantile_tensor = self.target_net(next_state_tensor)
            indices = [[idx, next_action] for idx, next_action in
                    enumerate(next_actions)]
            next_q_quantile_tensor = tf.gather_nd(all_next_q_quantile_tensor,
                    indices)
            target_quantile_tensor = reward_tensor + self.gamma \
                    * next_q_quantile_tensor * (1. - terminated_tensor)

            with tf.GradientTape() as tape:
                all_q_quantile_tensor = self.evaluate_net(state_tensor)
                indices = [[idx, action] for idx, action in enumerate(actions)]
                q_quantile_tensor = tf.gather_nd(all_q_quantile_tensor, indices)

                target_quantile_tensor = target_quantile_tensor[:, np.newaxis, :]
                q_quantile_tensor = q_quantile_tensor[:, :, np.newaxis]
                td_error_tensor = target_quantile_tensor - q_quantile_tensor
                abs_td_error_tensor = tf.math.abs(td_error_tensor)
                hubor_delta = 1.
                hubor_loss_tensor = tf.where(abs_td_error_tensor < hubor_delta,
                        0.5 * tf.square(td_error_tensor),
                        hubor_delta * (abs_td_error_tensor - 0.5 * hubor_delta))
                comparison_tensor = tf.cast(td_error_tensor < 0, dtype=tf.float32)
                quantile_regression_tensor = tf.math.abs(self.cumprob_tensor -
                        comparison_tensor)
                quantile_huber_loss_tensor = tf.reduce_mean(tf.reduce_sum(
                        hubor_loss_tensor * quantile_regression_tensor, axis=-1),
                        axis=1)
                loss_tensor = tf.reduce_mean(quantile_huber_loss_tensor)
            grads = tape.gradient(loss_tensor, self.evaluate_net.variables)
            self.evaluate_net.optimizer.apply_gradients(
                    zip(grads, self.evaluate_net.variables))

            self.update_net(self.target_net, self.evaluate_net)

            self.epsilon = max(self.epsilon - 1e-5, 0.05)

    agent = QRDQNAgent(env)

Train & Test

In [5]:

    def play_episode(env, agent, seed=None, mode=None, render=False):
        observation, _ = env.reset(seed=seed)
        reward, terminated, truncated = 0., False, False
        agent.reset(mode=mode)
        episode_reward, elapsed_steps = 0., 0
        while True:
            action = agent.step(observation, reward, terminated)
            if render:
                env.render()
            if terminated or truncated:
                break
            observation, reward, terminated, truncated, _ = env.step(action)
            episode_reward += reward
            elapsed_steps += 1
        agent.close()
        return episode_reward, elapsed_steps

    logging.info('==== train ====')
    episode_rewards = []
    for episode in itertools.count():
        episode_reward, elapsed_steps = play_episode(env, agent, mode='train')
        episode_rewards.append(episode_reward)
        logging.info('train episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
        if np.mean(episode_rewards[-3:]) > 16.:
            break
    plt.plot(episode_rewards)

    logging.info('==== test ====')
    episode_rewards = []
    for episode in range(100):
        episode_reward, elapsed_steps = play_episode(env, agent)
        episode_rewards.append(episode_reward)
        logging.info('test episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
    logging.info('average episode reward = %.2f ± %.2f',
            np.mean(episode_rewards), np.std(episode_rewards))

    00:07:35 [INFO] ==== train ====
    00:07:58 [INFO] train episode 0: reward = -17.00, steps = 1265
    00:08:23 [INFO] train episode 1: reward = -21.00, steps = 762
    00:08:53 [INFO] train episode 2: reward = -20.00, steps = 960
    00:09:18 [INFO] train episode 3: reward = -21.00, steps = 791
    00:09:45 [INFO] train episode 4: reward = -21.00, steps = 875
    00:10:10 [INFO] train episode 5: reward = -21.00, steps = 820
    00:10:36 [INFO] train episode 6: reward = -21.00, steps = 878
    00:10:58 [INFO] train episode 7: reward = -19.00, steps = 934
    00:11:17 [INFO] train episode 8: reward = -20.00, steps = 837
    00:11:39 [INFO] train episode 9: reward = -20.00, steps = 956
    00:12:09 [INFO] train episode 10: reward = -21.00, steps = 987
    00:12:35 [INFO] train episode 11: reward = -20.00, steps = 861
    00:12:59 [INFO] train episode 12: reward = -21.00, steps = 758
    00:13:29 [INFO] train episode 13: reward = -19.00, steps = 956
    00:13:56 [INFO] train episode 14: reward = -21.00, steps = 846
    00:14:25 [INFO] train episode 15: reward = -21.00, steps = 968
    00:14:57 [INFO] train episode 16: reward = -19.00, steps = 1037
    00:15:21 [INFO] train episode 17: reward = -21.00, steps = 788
    00:15:52 [INFO] train episode 18: reward = -20.00, steps = 1007
    00:16:18 [INFO] train episode 19: reward = -21.00, steps = 843
    00:16:42 [INFO] train episode 20: reward = -21.00, steps = 789
    00:17:15 [INFO] train episode 21: reward = -21.00, steps = 1061
    00:17:40 [INFO] train episode 22: reward = -21.00, steps = 836
    00:18:04 [INFO] train episode 23: reward = -21.00, steps = 823
    00:18:34 [INFO] train episode 24: reward = -21.00, steps = 998
    00:19:01 [INFO] train episode 25: reward = -20.00, steps = 854
    00:19:26 [INFO] train episode 26: reward = -21.00, steps = 810
    00:19:52 [INFO] train episode 27: reward = -21.00, steps = 822
    00:20:17 [INFO] train episode 28: reward = -21.00, steps = 847
    00:20:43 [INFO] train episode 29: reward = -21.00, steps = 819
    00:21:09 [INFO] train episode 30: reward = -21.00, steps = 836
    00:21:41 [INFO] train episode 31: reward = -20.00, steps = 993
    00:22:08 [INFO] train episode 32: reward = -21.00, steps = 848
    00:22:37 [INFO] train episode 33: reward = -21.00, steps = 930
    00:23:07 [INFO] train episode 34: reward = -21.00, steps = 950
    00:23:43 [INFO] train episode 35: reward = -18.00, steps = 1161
    00:24:24 [INFO] train episode 36: reward = -19.00, steps = 1292
    00:24:57 [INFO] train episode 37: reward = -20.00, steps = 1066
    00:25:33 [INFO] train episode 38: reward = -19.00, steps = 1003
    00:26:03 [INFO] train episode 39: reward = -21.00, steps = 819
    00:26:32 [INFO] train episode 40: reward = -21.00, steps = 789
    00:27:03 [INFO] train episode 41: reward = -21.00, steps = 847
    00:27:36 [INFO] train episode 42: reward = -21.00, steps = 910
    00:28:07 [INFO] train episode 43: reward = -21.00, steps = 851
    00:28:42 [INFO] train episode 44: reward = -19.00, steps = 1036
    00:29:14 [INFO] train episode 45: reward = -20.00, steps = 930
    00:29:40 [INFO] train episode 46: reward = -21.00, steps = 851
    00:30:08 [INFO] train episode 47: reward = -20.00, steps = 921
    00:30:35 [INFO] train episode 48: reward = -20.00, steps = 924
    00:31:08 [INFO] train episode 49: reward = -18.00, steps = 1071
    00:31:38 [INFO] train episode 50: reward = -20.00, steps = 980
    00:32:10 [INFO] train episode 51: reward = -21.00, steps = 908
    00:32:36 [INFO] train episode 52: reward = -21.00, steps = 804
    00:33:06 [INFO] train episode 53: reward = -19.00, steps = 993
    00:33:36 [INFO] train episode 54: reward = -20.00, steps = 985
    00:34:03 [INFO] train episode 55: reward = -21.00, steps = 877
    00:34:32 [INFO] train episode 56: reward = -20.00, steps = 942
    00:34:59 [INFO] train episode 57: reward = -20.00, steps = 876
    00:35:30 [INFO] train episode 58: reward = -20.00, steps = 978
    00:36:03 [INFO] train episode 59: reward = -19.00, steps = 1033
    00:36:29 [INFO] train episode 60: reward = -21.00, steps = 818
    00:36:58 [INFO] train episode 61: reward = -20.00, steps = 898
    00:37:25 [INFO] train episode 62: reward = -21.00, steps = 761
    00:37:54 [INFO] train episode 63: reward = -21.00, steps = 898
    00:38:25 [INFO] train episode 64: reward = -21.00, steps = 974
    00:38:55 [INFO] train episode 65: reward = -21.00, steps = 914
    00:39:28 [INFO] train episode 66: reward = -21.00, steps = 1024
    00:40:00 [INFO] train episode 67: reward = -20.00, steps = 997
    00:40:30 [INFO] train episode 68: reward = -19.00, steps = 937
    00:40:58 [INFO] train episode 69: reward = -20.00, steps = 897
    00:41:27 [INFO] train episode 70: reward = -21.00, steps = 930
    00:41:53 [INFO] train episode 71: reward = -21.00, steps = 817
    00:42:25 [INFO] train episode 72: reward = -20.00, steps = 1012
    00:42:51 [INFO] train episode 73: reward = -21.00, steps = 846
    00:43:27 [INFO] train episode 74: reward = -20.00, steps = 1139
    00:43:59 [INFO] train episode 75: reward = -20.00, steps = 1021
    00:44:25 [INFO] train episode 76: reward = -21.00, steps = 825
    00:44:59 [INFO] train episode 77: reward = -21.00, steps = 1077
    00:45:27 [INFO] train episode 78: reward = -20.00, steps = 894
    00:46:00 [INFO] train episode 79: reward = -20.00, steps = 1050
    00:46:30 [INFO] train episode 80: reward = -21.00, steps = 912
    00:46:59 [INFO] train episode 81: reward = -21.00, steps = 943
    00:47:27 [INFO] train episode 82: reward = -21.00, steps = 874
    00:47:58 [INFO] train episode 83: reward = -21.00, steps = 941
    00:48:28 [INFO] train episode 84: reward = -19.00, steps = 950
    00:48:56 [INFO] train episode 85: reward = -21.00, steps = 880
    00:49:21 [INFO] train episode 86: reward = -21.00, steps = 791
    00:49:46 [INFO] train episode 87: reward = -21.00, steps = 776
    00:50:23 [INFO] train episode 88: reward = -18.00, steps = 1170
    00:50:56 [INFO] train episode 89: reward = -20.00, steps = 1024
    00:51:20 [INFO] train episode 90: reward = -21.00, steps = 760
    00:51:44 [INFO] train episode 91: reward = -21.00, steps = 776
    00:52:12 [INFO] train episode 92: reward = -21.00, steps = 848
    00:52:43 [INFO] train episode 93: reward = -21.00, steps = 896
    00:53:20 [INFO] train episode 94: reward = -19.00, steps = 1017
    00:53:53 [INFO] train episode 95: reward = -21.00, steps = 910
    00:54:25 [INFO] train episode 96: reward = -20.00, steps = 924
    00:55:03 [INFO] train episode 97: reward = -19.00, steps = 1146
    00:55:29 [INFO] train episode 98: reward = -21.00, steps = 789
    00:55:58 [INFO] train episode 99: reward = -21.00, steps = 883
    00:56:44 [INFO] train episode 100: reward = -19.00, steps = 1131
    00:57:33 [INFO] train episode 101: reward = -20.00, steps = 887
    00:58:03 [INFO] train episode 102: reward = -20.00, steps = 946
    00:58:31 [INFO] train episode 103: reward = -20.00, steps = 927
    00:59:04 [INFO] train episode 104: reward = -19.00, steps = 1102
    00:59:31 [INFO] train episode 105: reward = -21.00, steps = 883
    01:00:11 [INFO] train episode 106: reward = -19.00, steps = 1313
    01:01:11 [INFO] train episode 107: reward = -21.00, steps = 901
    01:03:30 [INFO] train episode 108: reward = -21.00, steps = 792
    01:06:36 [INFO] train episode 109: reward = -20.00, steps = 1057
    01:09:09 [INFO] train episode 110: reward = -20.00, steps = 879
    01:11:59 [INFO] train episode 111: reward = -21.00, steps = 968
    01:15:03 [INFO] train episode 112: reward = -20.00, steps = 1039
    01:17:47 [INFO] train episode 113: reward = -21.00, steps = 909
    01:20:25 [INFO] train episode 114: reward = -21.00, steps = 848
    01:23:18 [INFO] train episode 115: reward = -21.00, steps = 921
    01:26:25 [INFO] train episode 116: reward = -19.00, steps = 1008
    01:29:15 [INFO] train episode 117: reward = -21.00, steps = 910
    01:31:55 [INFO] train episode 118: reward = -20.00, steps = 866
    01:34:36 [INFO] train episode 119: reward = -20.00, steps = 872
    01:37:34 [INFO] train episode 120: reward = -20.00, steps = 964
    01:40:13 [INFO] train episode 121: reward = -20.00, steps = 865
    01:43:15 [INFO] train episode 122: reward = -20.00, steps = 986
    01:46:03 [INFO] train episode 123: reward = -21.00, steps = 911
    01:49:06 [INFO] train episode 124: reward = -20.00, steps = 991
    01:52:26 [INFO] train episode 125: reward = -18.00, steps = 1081
    01:55:21 [INFO] train episode 126: reward = -21.00, steps = 946
    01:58:25 [INFO] train episode 127: reward = -20.00, steps = 988
    02:01:20 [INFO] train episode 128: reward = -21.00, steps = 937
    02:04:43 [INFO] train episode 129: reward = -18.00, steps = 1100
    02:07:35 [INFO] train episode 130: reward = -21.00, steps = 930
    02:10:41 [INFO] train episode 131: reward = -19.00, steps = 1014
    02:14:10 [INFO] train episode 132: reward = -19.00, steps = 1135
    02:17:30 [INFO] train episode 133: reward = -20.00, steps = 1089
    02:21:36 [INFO] train episode 134: reward = -18.00, steps = 1327
    02:25:20 [INFO] train episode 135: reward = -19.00, steps = 1213
    02:28:57 [INFO] train episode 136: reward = -18.00, steps = 1178
    02:32:49 [INFO] train episode 137: reward = -19.00, steps = 1254
    02:36:55 [INFO] train episode 138: reward = -21.00, steps = 1333
    02:41:32 [INFO] train episode 139: reward = -18.00, steps = 1504
    02:45:41 [INFO] train episode 140: reward = -20.00, steps = 1350
    02:50:26 [INFO] train episode 141: reward = -15.00, steps = 1545
    02:55:57 [INFO] train episode 142: reward = -18.00, steps = 1799
    03:00:35 [INFO] train episode 143: reward = -17.00, steps = 1501
    03:05:41 [INFO] train episode 144: reward = -17.00, steps = 1670
    03:10:21 [INFO] train episode 145: reward = -19.00, steps = 1540
    03:17:03 [INFO] train episode 146: reward = -13.00, steps = 2200
    03:21:25 [INFO] train episode 147: reward = -20.00, steps = 1432
    03:26:09 [INFO] train episode 148: reward = -19.00, steps = 1560
    03:31:59 [INFO] train episode 149: reward = -16.00, steps = 1900
    03:38:04 [INFO] train episode 150: reward = -17.00, steps = 2011
    03:42:51 [INFO] train episode 151: reward = -20.00, steps = 1587
    03:49:19 [INFO] train episode 152: reward = -12.00, steps = 2145
    03:55:17 [INFO] train episode 153: reward = -14.00, steps = 1981
    04:01:02 [INFO] train episode 154: reward = -15.00, steps = 1898
    04:06:14 [INFO] train episode 155: reward = -18.00, steps = 1743
    04:11:30 [INFO] train episode 156: reward = -19.00, steps = 1774
    04:16:57 [INFO] train episode 157: reward = -18.00, steps = 1829
    04:22:51 [INFO] train episode 158: reward = -13.00, steps = 1980
    04:29:05 [INFO] train episode 159: reward = -18.00, steps = 2082
    04:33:53 [INFO] train episode 160: reward = -17.00, steps = 1597
    04:39:23 [INFO] train episode 161: reward = -16.00, steps = 1835
    04:46:41 [INFO] train episode 162: reward = -11.00, steps = 2420
    04:51:48 [INFO] train episode 163: reward = -16.00, steps = 1713
    04:55:37 [INFO] train episode 164: reward = -20.00, steps = 1278
    05:02:14 [INFO] train episode 165: reward = -10.00, steps = 2203
    05:07:22 [INFO] train episode 166: reward = -15.00, steps = 1713
    05:13:09 [INFO] train episode 167: reward = -16.00, steps = 1938
    05:18:04 [INFO] train episode 168: reward = -13.00, steps = 1643
    05:22:30 [INFO] train episode 169: reward = -16.00, steps = 1489
    05:29:01 [INFO] train episode 170: reward = -12.00, steps = 2193
    05:34:42 [INFO] train episode 171: reward = -11.00, steps = 1919
    05:38:47 [INFO] train episode 172: reward = -17.00, steps = 1377
    05:42:50 [INFO] train episode 173: reward = -15.00, steps = 1373
    05:48:17 [INFO] train episode 174: reward = -13.00, steps = 1846
    05:53:53 [INFO] train episode 175: reward = -15.00, steps = 1903
    06:00:27 [INFO] train episode 176: reward = -7.00, steps = 2216
    06:09:06 [INFO] train episode 177: reward = -5.00, steps = 2931
    06:16:54 [INFO] train episode 178: reward = -6.00, steps = 2658
    06:24:14 [INFO] train episode 179: reward = -5.00, steps = 2492
    06:33:02 [INFO] train episode 180: reward = -2.00, steps = 3016
    06:41:03 [INFO] train episode 181: reward = -1.00, steps = 2742
    06:46:34 [INFO] train episode 182: reward = -11.00, steps = 1888
    06:53:30 [INFO] train episode 183: reward = -9.00, steps = 2362
    07:00:59 [INFO] train episode 184: reward = -7.00, steps = 2553
    07:09:07 [INFO] train episode 185: reward = -3.00, steps = 2777
    07:15:35 [INFO] train episode 186: reward = -9.00, steps = 2204
    07:22:29 [INFO] train episode 187: reward = -6.00, steps = 2339
    07:26:50 [INFO] train episode 188: reward = -15.00, steps = 1477
    07:35:53 [INFO] train episode 189: reward = -4.00, steps = 3069
    07:44:09 [INFO] train episode 190: reward = -2.00, steps = 2819
    07:50:13 [INFO] train episode 191: reward = -8.00, steps = 2069
    07:56:51 [INFO] train episode 192: reward = -8.00, steps = 2237
    08:03:22 [INFO] train episode 193: reward = -10.00, steps = 2203
    08:08:04 [INFO] train episode 194: reward = -14.00, steps = 1604
    08:12:46 [INFO] train episode 195: reward = -13.00, steps = 1610
    08:15:36 [INFO] train episode 196: reward = -20.00, steps = 963
    08:21:59 [INFO] train episode 197: reward = -7.00, steps = 2175
    08:25:24 [INFO] train episode 198: reward = -17.00, steps = 1168
    08:32:23 [INFO] train episode 199: reward = -6.00, steps = 2397
    08:38:43 [INFO] train episode 200: reward = -6.00, steps = 2178
    08:43:19 [INFO] train episode 201: reward = -13.00, steps = 1563
    08:47:42 [INFO] train episode 202: reward = -17.00, steps = 1492
    08:54:06 [INFO] train episode 203: reward = -6.00, steps = 2193
    09:00:27 [INFO] train episode 204: reward = -6.00, steps = 2180
    09:05:35 [INFO] train episode 205: reward = -11.00, steps = 1756
    09:10:54 [INFO] train episode 206: reward = -11.00, steps = 1822
    09:15:36 [INFO] train episode 207: reward = -13.00, steps = 1608
    09:19:52 [INFO] train episode 208: reward = -15.00, steps = 1453
    09:26:13 [INFO] train episode 209: reward = -8.00, steps = 2151
    09:31:34 [INFO] train episode 210: reward = -11.00, steps = 1824
    09:37:11 [INFO] train episode 211: reward = -14.00, steps = 1654
    09:41:11 [INFO] train episode 212: reward = -17.00, steps = 1291
    09:45:14 [INFO] train episode 213: reward = -17.00, steps = 1351
    09:49:31 [INFO] train episode 214: reward = -15.00, steps = 1434
    09:55:01 [INFO] train episode 215: reward = -12.00, steps = 1711
    09:58:54 [INFO] train episode 216: reward = -17.00, steps = 1285
    10:05:21 [INFO] train episode 217: reward = -7.00, steps = 2134
    10:11:02 [INFO] train episode 218: reward = -10.00, steps = 1779
    10:14:51 [INFO] train episode 219: reward = -17.00, steps = 1171
    10:19:28 [INFO] train episode 220: reward = -14.00, steps = 1493
    10:24:33 [INFO] train episode 221: reward = -13.00, steps = 1599
    10:27:02 [INFO] train episode 222: reward = -21.00, steps = 809
    10:29:38 [INFO] train episode 223: reward = -20.00, steps = 835
    10:32:01 [INFO] train episode 224: reward = -21.00, steps = 787
    10:36:26 [INFO] train episode 225: reward = -14.00, steps = 1455
    10:41:36 [INFO] train episode 226: reward = -12.00, steps = 1714
    10:44:00 [INFO] train episode 227: reward = -21.00, steps = 792
    10:49:07 [INFO] train episode 228: reward = -11.00, steps = 1691
    10:54:47 [INFO] train episode 229: reward = -11.00, steps = 1886
    11:00:05 [INFO] train episode 230: reward = -13.00, steps = 1759
    11:05:19 [INFO] train episode 231: reward = -13.00, steps = 1733
    11:09:48 [INFO] train episode 232: reward = -16.00, steps = 1496
    11:15:01 [INFO] train episode 233: reward = -14.00, steps = 1743
    11:20:56 [INFO] train episode 234: reward = -9.00, steps = 1961
    11:26:54 [INFO] train episode 235: reward = -10.00, steps = 1983
    11:34:09 [INFO] train episode 236: reward = -7.00, steps = 2394
    11:41:16 [INFO] train episode 237: reward = -7.00, steps = 2354
    11:44:42 [INFO] train episode 238: reward = -17.00, steps = 1127
    11:48:52 [INFO] train episode 239: reward = -16.00, steps = 1370
    11:56:52 [INFO] train episode 240: reward = -3.00, steps = 2641
    12:00:18 [INFO] train episode 241: reward = -17.00, steps = 1126
    12:07:37 [INFO] train episode 242: reward = -6.00, steps = 2406
    12:10:55 [INFO] train episode 243: reward = -18.00, steps = 1090
    12:16:42 [INFO] train episode 244: reward = -10.00, steps = 1911
    12:20:42 [INFO] train episode 245: reward = -15.00, steps = 1316
    12:26:46 [INFO] train episode 246: reward = -11.00, steps = 2015
    12:32:22 [INFO] train episode 247: reward = -10.00, steps = 1855
    12:39:00 [INFO] train episode 248: reward = -5.00, steps = 2207
    12:45:50 [INFO] train episode 249: reward = -6.00, steps = 2274
    12:51:38 [INFO] train episode 250: reward = -11.00, steps = 1915
    12:57:47 [INFO] train episode 251: reward = -9.00, steps = 2030
    13:03:22 [INFO] train episode 252: reward = -11.00, steps = 1807
    13:09:16 [INFO] train episode 253: reward = -9.00, steps = 1955
    13:15:34 [INFO] train episode 254: reward = -8.00, steps = 2097
    13:20:16 [INFO] train episode 255: reward = -13.00, steps = 1566
    13:26:13 [INFO] train episode 256: reward = -9.00, steps = 1969
    13:30:53 [INFO] train episode 257: reward = -14.00, steps = 1551
    13:36:07 [INFO] train episode 258: reward = -11.00, steps = 1740
    13:42:27 [INFO] train episode 259: reward = -8.00, steps = 2108
    13:48:40 [INFO] train episode 260: reward = -11.00, steps = 1996
    13:55:42 [INFO] train episode 261: reward = -7.00, steps = 2143
    14:02:10 [INFO] train episode 262: reward = -7.00, steps = 2102
    14:09:00 [INFO] train episode 263: reward = -9.00, steps = 2103
    14:16:11 [INFO] train episode 264: reward = -6.00, steps = 2303
    14:20:51 [INFO] train episode 265: reward = -16.00, steps = 1492
    14:27:52 [INFO] train episode 266: reward = -7.00, steps = 2193
    14:34:51 [INFO] train episode 267: reward = -9.00, steps = 2133
    14:41:06 [INFO] train episode 268: reward = -11.00, steps = 1814
    14:47:18 [INFO] train episode 269: reward = -10.00, steps = 1980
    14:54:15 [INFO] train episode 270: reward = -8.00, steps = 2032
    15:00:26 [INFO] train episode 271: reward = -10.00, steps = 1871
    15:06:45 [INFO] train episode 272: reward = -10.00, steps = 1834
    15:12:23 [INFO] train episode 273: reward = -14.00, steps = 1697
    15:19:40 [INFO] train episode 274: reward = -6.00, steps = 2249
    15:25:27 [INFO] train episode 275: reward = -12.00, steps = 1796
    15:31:08 [INFO] train episode 276: reward = -12.00, steps = 1774
    15:36:40 [INFO] train episode 277: reward = -13.00, steps = 1668
    15:43:34 [INFO] train episode 278: reward = -9.00, steps = 2162
    15:49:43 [INFO] train episode 279: reward = -10.00, steps = 2027
    15:55:49 [INFO] train episode 280: reward = -9.00, steps = 2032
    16:01:52 [INFO] train episode 281: reward = -9.00, steps = 2004
    16:07:31 [INFO] train episode 282: reward = -11.00, steps = 1871
    16:13:22 [INFO] train episode 283: reward = -9.00, steps = 1908
    16:19:31 [INFO] train episode 284: reward = -9.00, steps = 1991
    16:24:58 [INFO] train episode 285: reward = -13.00, steps = 1652
    16:30:27 [INFO] train episode 286: reward = -13.00, steps = 1546
    16:37:57 [INFO] train episode 287: reward = -9.00, steps = 2125
    16:44:00 [INFO] train episode 288: reward = -13.00, steps = 1700
    16:49:20 [INFO] train episode 289: reward = -13.00, steps = 1613
    16:55:13 [INFO] train episode 290: reward = -10.00, steps = 1867
    17:01:05 [INFO] train episode 291: reward = -11.00, steps = 1853
    17:07:13 [INFO] train episode 292: reward = -10.00, steps = 1939
    17:12:58 [INFO] train episode 293: reward = -13.00, steps = 1792
    17:18:58 [INFO] train episode 294: reward = -11.00, steps = 1909
    17:23:56 [INFO] train episode 295: reward = -13.00, steps = 1595
    17:30:25 [INFO] train episode 296: reward = -9.00, steps = 1948
    17:37:32 [INFO] train episode 297: reward = -8.00, steps = 2081
    17:44:24 [INFO] train episode 298: reward = -8.00, steps = 2046
    17:51:39 [INFO] train episode 299: reward = -7.00, steps = 2133
    17:57:52 [INFO] train episode 300: reward = -10.00, steps = 1896
    18:04:26 [INFO] train episode 301: reward = -9.00, steps = 2034
    18:07:53 [INFO] train episode 302: reward = -17.00, steps = 1124
    18:13:35 [INFO] train episode 303: reward = -12.00, steps = 1834
    18:19:35 [INFO] train episode 304: reward = -10.00, steps = 1912
    18:28:13 [INFO] train episode 305: reward = 1.00, steps = 2804
    18:33:09 [INFO] train episode 306: reward = -13.00, steps = 1592
    18:38:42 [INFO] train episode 307: reward = -12.00, steps = 1793
    18:43:42 [INFO] train episode 308: reward = -13.00, steps = 1634
    18:48:17 [INFO] train episode 309: reward = -15.00, steps = 1486
    18:55:53 [INFO] train episode 310: reward = -3.00, steps = 2463
    19:00:48 [INFO] train episode 311: reward = -14.00, steps = 1597
    19:06:18 [INFO] train episode 312: reward = -13.00, steps = 1766
    19:10:58 [INFO] train episode 313: reward = -15.00, steps = 1436
    19:14:58 [INFO] train episode 314: reward = -16.00, steps = 1297
    19:20:28 [INFO] train episode 315: reward = -11.00, steps = 1756
    19:25:35 [INFO] train episode 316: reward = -13.00, steps = 1628
    19:29:16 [INFO] train episode 317: reward = -17.00, steps = 1188
    19:32:22 [INFO] train episode 318: reward = -19.00, steps = 982
    19:36:58 [INFO] train episode 319: reward = -15.00, steps = 1484
    19:41:47 [INFO] train episode 320: reward = -12.00, steps = 1565
    19:46:50 [INFO] train episode 321: reward = -13.00, steps = 1644
    19:52:26 [INFO] train episode 322: reward = -10.00, steps = 1826
    19:56:28 [INFO] train episode 323: reward = -17.00, steps = 1311
    20:00:29 [INFO] train episode 324: reward = -17.00, steps = 1315
    20:04:00 [INFO] train episode 325: reward = -18.00, steps = 1147
    20:07:43 [INFO] train episode 326: reward = -18.00, steps = 1212
    20:12:59 [INFO] train episode 327: reward = -13.00, steps = 1723
    20:16:37 [INFO] train episode 328: reward = -18.00, steps = 1186
    20:20:06 [INFO] train episode 329: reward = -17.00, steps = 1125
    20:23:26 [INFO] train episode 330: reward = -20.00, steps = 1075
    20:27:03 [INFO] train episode 331: reward = -17.00, steps = 1171
    20:30:14 [INFO] train episode 332: reward = -19.00, steps = 1025
    20:33:52 [INFO] train episode 333: reward = -17.00, steps = 1175
    20:38:54 [INFO] train episode 334: reward = -15.00, steps = 1631
    20:41:51 [INFO] train episode 335: reward = -20.00, steps = 956
    20:46:06 [INFO] train episode 336: reward = -15.00, steps = 1381
    20:49:57 [INFO] train episode 337: reward = -18.00, steps = 1250
    20:55:26 [INFO] train episode 338: reward = -12.00, steps = 1781
    20:59:49 [INFO] train episode 339: reward = -15.00, steps = 1421
    21:02:35 [INFO] train episode 340: reward = -20.00, steps = 892
    21:06:15 [INFO] train episode 341: reward = -18.00, steps = 1187
    21:10:29 [INFO] train episode 342: reward = -15.00, steps = 1379
    21:13:35 [INFO] train episode 343: reward = -19.00, steps = 1006
    21:17:03 [INFO] train episode 344: reward = -19.00, steps = 1021
    21:22:44 [INFO] train episode 345: reward = -12.00, steps = 1742
    21:27:05 [INFO] train episode 346: reward = -16.00, steps = 1266
    21:30:44 [INFO] train episode 347: reward = -18.00, steps = 1036
    21:36:15 [INFO] train episode 348: reward = -14.00, steps = 1557
    21:39:42 [INFO] train episode 349: reward = -20.00, steps = 1017
    21:44:38 [INFO] train episode 350: reward = -16.00, steps = 1599
    21:47:32 [INFO] train episode 351: reward = -19.00, steps = 933
    21:51:50 [INFO] train episode 352: reward = -18.00, steps = 1338
    21:56:14 [INFO] train episode 353: reward = -15.00, steps = 1389
    22:00:19 [INFO] train episode 354: reward = -18.00, steps = 1155
    22:04:17 [INFO] train episode 355: reward = -18.00, steps = 1150
    22:10:21 [INFO] train episode 356: reward = -12.00, steps = 1796
    22:14:03 [INFO] train episode 357: reward = -18.00, steps = 1105
    22:17:53 [INFO] train episode 358: reward = -17.00, steps = 1165
    22:21:08 [INFO] train episode 359: reward = -19.00, steps = 1002
    22:25:05 [INFO] train episode 360: reward = -19.00, steps = 1162
    22:28:04 [INFO] train episode 361: reward = -21.00, steps = 874
    22:33:35 [INFO] train episode 362: reward = -14.00, steps = 1636
    22:37:20 [INFO] train episode 363: reward = -18.00, steps = 1136
    22:40:32 [INFO] train episode 364: reward = -19.00, steps = 963
    22:46:32 [INFO] train episode 365: reward = -11.00, steps = 1792
    22:50:24 [INFO] train episode 366: reward = -18.00, steps = 1190
    22:54:17 [INFO] train episode 367: reward = -18.00, steps = 1195
    22:59:35 [INFO] train episode 368: reward = -16.00, steps = 1463
    23:03:35 [INFO] train episode 369: reward = -17.00, steps = 1281
    23:06:16 [INFO] train episode 370: reward = -21.00, steps = 857
    23:09:38 [INFO] train episode 371: reward = -18.00, steps = 1090
    23:12:55 [INFO] train episode 372: reward = -19.00, steps = 1061
    23:16:20 [INFO] train episode 373: reward = -19.00, steps = 1092
    23:20:07 [INFO] train episode 374: reward = -18.00, steps = 1223
    23:24:09 [INFO] train episode 375: reward = -16.00, steps = 1241
    23:29:20 [INFO] train episode 376: reward = -12.00, steps = 1661
    23:33:53 [INFO] train episode 377: reward = -17.00, steps = 1302
    23:37:17 [INFO] train episode 378: reward = -19.00, steps = 1008
    23:40:33 [INFO] train episode 379: reward = -20.00, steps = 1003
    23:44:40 [INFO] train episode 380: reward = -16.00, steps = 1268
    23:48:45 [INFO] train episode 381: reward = -18.00, steps = 1245
    23:54:02 [INFO] train episode 382: reward = -12.00, steps = 1631
    23:57:33 [INFO] train episode 383: reward = -19.00, steps = 1085
    00:01:39 [INFO] train episode 384: reward = -16.00, steps = 1303
    00:05:22 [INFO] train episode 385: reward = -19.00, steps = 1192
    00:12:35 [INFO] train episode 386: reward = -10.00, steps = 2303
    00:17:35 [INFO] train episode 387: reward = -13.00, steps = 1593
    00:20:52 [INFO] train episode 388: reward = -19.00, steps = 1018
    00:24:22 [INFO] train episode 389: reward = -19.00, steps = 1081
    00:27:45 [INFO] train episode 390: reward = -18.00, steps = 1073
    00:30:30 [INFO] train episode 391: reward = -20.00, steps = 888
    00:33:41 [INFO] train episode 392: reward = -19.00, steps = 1013
    00:37:14 [INFO] train episode 393: reward = -18.00, steps = 1146
    00:40:28 [INFO] train episode 394: reward = -21.00, steps = 948
    00:44:07 [INFO] train episode 395: reward = -19.00, steps = 1221
    00:47:26 [INFO] train episode 396: reward = -18.00, steps = 1116
    00:50:20 [INFO] train episode 397: reward = -20.00, steps = 979
    00:54:10 [INFO] train episode 398: reward = -16.00, steps = 1288
    00:57:59 [INFO] train episode 399: reward = -18.00, steps = 1290
    01:02:41 [INFO] train episode 400: reward = -14.00, steps = 1589
    01:06:22 [INFO] train episode 401: reward = -17.00, steps = 1245
    01:09:07 [INFO] train episode 402: reward = -19.00, steps = 933
    01:13:12 [INFO] train episode 403: reward = -18.00, steps = 1381
    01:16:24 [INFO] train episode 404: reward = -19.00, steps = 1085
    01:19:25 [INFO] train episode 405: reward = -19.00, steps = 1021
    01:23:28 [INFO] train episode 406: reward = -17.00, steps = 1359
    01:27:35 [INFO] train episode 407: reward = -17.00, steps = 1396
    01:31:16 [INFO] train episode 408: reward = -17.00, steps = 1243
    01:34:13 [INFO] train episode 409: reward = -18.00, steps = 997
    01:38:08 [INFO] train episode 410: reward = -17.00, steps = 1325
    01:41:38 [INFO] train episode 411: reward = -19.00, steps = 1180
    01:45:01 [INFO] train episode 412: reward = -18.00, steps = 1145
    01:48:15 [INFO] train episode 413: reward = -19.00, steps = 1092
    01:52:09 [INFO] train episode 414: reward = -17.00, steps = 1316
    01:57:53 [INFO] train episode 415: reward = -9.00, steps = 1941
    02:02:14 [INFO] train episode 416: reward = -16.00, steps = 1460
    02:07:49 [INFO] train episode 417: reward = -11.00, steps = 1880
    02:10:49 [INFO] train episode 418: reward = -19.00, steps = 1015
    02:13:57 [INFO] train episode 419: reward = -18.00, steps = 1056
    02:17:51 [INFO] train episode 420: reward = -15.00, steps = 1317
    02:21:18 [INFO] train episode 421: reward = -19.00, steps = 1153
    02:27:03 [INFO] train episode 422: reward = -11.00, steps = 1940
    02:31:25 [INFO] train episode 423: reward = -16.00, steps = 1474
    02:36:57 [INFO] train episode 424: reward = -11.00, steps = 1865
    02:41:37 [INFO] train episode 425: reward = -14.00, steps = 1572
    02:45:47 [INFO] train episode 426: reward = -16.00, steps = 1402
    02:50:15 [INFO] train episode 427: reward = -15.00, steps = 1506
    02:53:44 [INFO] train episode 428: reward = -19.00, steps = 1175
    02:59:28 [INFO] train episode 429: reward = -10.00, steps = 1937
    03:03:44 [INFO] train episode 430: reward = -16.00, steps = 1433
    03:08:20 [INFO] train episode 431: reward = -15.00, steps = 1552
    03:13:26 [INFO] train episode 432: reward = -12.00, steps = 1726
    03:20:19 [INFO] train episode 433: reward = -6.00, steps = 2326
    03:24:16 [INFO] train episode 434: reward = -17.00, steps = 1294
    03:28:11 [INFO] train episode 435: reward = -19.00, steps = 1249
    03:34:47 [INFO] train episode 436: reward = -8.00, steps = 2204
    03:38:38 [INFO] train episode 437: reward = -16.00, steps = 1294
    03:43:27 [INFO] train episode 438: reward = -13.00, steps = 1622
    03:48:01 [INFO] train episode 439: reward = -15.00, steps = 1532
    03:51:03 [INFO] train episode 440: reward = -20.00, steps = 1022
    03:55:55 [INFO] train episode 441: reward = -15.00, steps = 1635
    04:00:11 [INFO] train episode 442: reward = -14.00, steps = 1431
    04:05:28 [INFO] train episode 443: reward = -13.00, steps = 1763
    04:09:33 [INFO] train episode 444: reward = -16.00, steps = 1369
    04:17:35 [INFO] train episode 445: reward = -2.00, steps = 2687
    04:22:09 [INFO] train episode 446: reward = -14.00, steps = 1516
    04:30:30 [INFO] train episode 447: reward = -1.00, steps = 2780
    04:35:22 [INFO] train episode 448: reward = -12.00, steps = 1630
    04:41:33 [INFO] train episode 449: reward = 18.00, steps = 2079
    04:47:11 [INFO] train episode 450: reward = -12.00, steps = 1882
    04:51:37 [INFO] train episode 451: reward = -14.00, steps = 1485
    04:55:56 [INFO] train episode 452: reward = -15.00, steps = 1447
    04:59:48 [INFO] train episode 453: reward = -17.00, steps = 1291
    05:04:25 [INFO] train episode 454: reward = -13.00, steps = 1549
    05:08:52 [INFO] train episode 455: reward = -16.00, steps = 1485
    05:12:34 [INFO] train episode 456: reward = -17.00, steps = 1240
    05:19:18 [INFO] train episode 457: reward = -7.00, steps = 2269
    05:23:33 [INFO] train episode 458: reward = -16.00, steps = 1430
    05:30:28 [INFO] train episode 459: reward = -5.00, steps = 2336
    05:34:26 [INFO] train episode 460: reward = -15.00, steps = 1342
    05:37:39 [INFO] train episode 461: reward = -18.00, steps = 1088
    05:42:55 [INFO] train episode 462: reward = -11.00, steps = 1783
    05:48:11 [INFO] train episode 463: reward = -10.00, steps = 1789
    05:53:27 [INFO] train episode 464: reward = -13.00, steps = 1787
    05:57:38 [INFO] train episode 465: reward = -15.00, steps = 1413
    06:02:11 [INFO] train episode 466: reward = -16.00, steps = 1529
    06:06:59 [INFO] train episode 467: reward = -15.00, steps = 1622
    06:09:41 [INFO] train episode 468: reward = -21.00, steps = 911
    06:15:26 [INFO] train episode 469: reward = -11.00, steps = 1939
    06:20:06 [INFO] train episode 470: reward = -13.00, steps = 1578
    06:25:12 [INFO] train episode 471: reward = -14.00, steps = 1713
    06:29:09 [INFO] train episode 472: reward = -16.00, steps = 1328
    06:33:44 [INFO] train episode 473: reward = -15.00, steps = 1544
    06:38:10 [INFO] train episode 474: reward = -15.00, steps = 1492
    06:41:30 [INFO] train episode 475: reward = -19.00, steps = 1116
    06:48:04 [INFO] train episode 476: reward = -10.00, steps = 2215
    06:55:10 [INFO] train episode 477: reward = -4.00, steps = 2398
    06:59:46 [INFO] train episode 478: reward = -14.00, steps = 1553
    07:05:37 [INFO] train episode 479: reward = -11.00, steps = 1977
    07:09:43 [INFO] train episode 480: reward = -15.00, steps = 1382
    07:15:03 [INFO] train episode 481: reward = -10.00, steps = 1802
    07:22:26 [INFO] train episode 482: reward = -2.00, steps = 2487
    07:27:48 [INFO] train episode 483: reward = -10.00, steps = 1809
    07:31:48 [INFO] train episode 484: reward = -16.00, steps = 1344
    07:36:13 [INFO] train episode 485: reward = -14.00, steps = 1488
    07:42:30 [INFO] train episode 486: reward = -7.00, steps = 2126
    07:46:51 [INFO] train episode 487: reward = -15.00, steps = 1470
    07:53:11 [INFO] train episode 488: reward = -7.00, steps = 2141
    08:00:01 [INFO] train episode 489: reward = -6.00, steps = 2310
    08:07:52 [INFO] train episode 490: reward = -2.00, steps = 2654
    08:15:49 [INFO] train episode 491: reward = 1.00, steps = 2682
    08:23:37 [INFO] train episode 492: reward = 1.00, steps = 2627
    08:29:18 [INFO] train episode 493: reward = 18.00, steps = 1911
    08:33:49 [INFO] train episode 494: reward = -15.00, steps = 1535
    08:39:04 [INFO] train episode 495: reward = -12.00, steps = 1787
    08:45:49 [INFO] train episode 496: reward = -6.00, steps = 2281
    08:50:17 [INFO] train episode 497: reward = -15.00, steps = 1519
    08:55:19 [INFO] train episode 498: reward = -13.00, steps = 1708
    09:00:44 [INFO] train episode 499: reward = -11.00, steps = 1840
    09:07:37 [INFO] train episode 500: reward = 11.00, steps = 2329
    09:12:29 [INFO] train episode 501: reward = -14.00, steps = 1651
    09:18:32 [INFO] train episode 502: reward = -8.00, steps = 2043
    09:23:41 [INFO] train episode 503: reward = -11.00, steps = 1728
    09:28:22 [INFO] train episode 504: reward = -14.00, steps = 1582
    09:35:58 [INFO] train episode 505: reward = -3.00, steps = 2564
    09:40:39 [INFO] train episode 506: reward = -14.00, steps = 1584
    09:45:30 [INFO] train episode 507: reward = -13.00, steps = 1638
    09:52:55 [INFO] train episode 508: reward = -5.00, steps = 2506
    09:59:37 [INFO] train episode 509: reward = -8.00, steps = 2169
    10:06:24 [INFO] train episode 510: reward = -9.00, steps = 1944
    10:12:57 [INFO] train episode 511: reward = -10.00, steps = 1978
    10:22:17 [INFO] train episode 512: reward = 1.00, steps = 2760
    10:30:18 [INFO] train episode 513: reward = 10.00, steps = 2504
    10:38:05 [INFO] train episode 514: reward = -8.00, steps = 2250
    10:45:49 [INFO] train episode 515: reward = -6.00, steps = 2313
    10:55:09 [INFO] train episode 516: reward = -3.00, steps = 2792
    10:59:28 [INFO] train episode 517: reward = -16.00, steps = 1290
    11:06:54 [INFO] train episode 518: reward = 17.00, steps = 2152
    11:13:32 [INFO] train episode 519: reward = 17.00, steps = 2093
    11:21:27 [INFO] train episode 520: reward = 7.00, steps = 2527
    11:28:49 [INFO] train episode 521: reward = -6.00, steps = 2378
    11:36:44 [INFO] train episode 522: reward = -2.00, steps = 2488
    11:45:27 [INFO] train episode 523: reward = 1.00, steps = 2758
    11:52:48 [INFO] train episode 524: reward = -9.00, steps = 2275
    12:00:07 [INFO] train episode 525: reward = 14.00, steps = 2248
    12:09:28 [INFO] train episode 526: reward = -1.00, steps = 2757
    12:14:47 [INFO] train episode 527: reward = -13.00, steps = 1727
    12:23:03 [INFO] train episode 528: reward = -1.00, steps = 2677
    12:31:26 [INFO] train episode 529: reward = -2.00, steps = 2721
    12:39:57 [INFO] train episode 530: reward = -1.00, steps = 2757
    12:47:30 [INFO] train episode 531: reward = -4.00, steps = 2427
    12:55:43 [INFO] train episode 532: reward = -2.00, steps = 2671
    13:03:05 [INFO] train episode 533: reward = 8.00, steps = 2388
    13:10:04 [INFO] train episode 534: reward = -7.00, steps = 2260
    13:17:39 [INFO] train episode 535: reward = -3.00, steps = 2460
    13:24:37 [INFO] train episode 536: reward = -8.00, steps = 2254
    13:32:55 [INFO] train episode 537: reward = -4.00, steps = 2611
    13:39:12 [INFO] train episode 538: reward = -11.00, steps = 2009
    13:46:50 [INFO] train episode 539: reward = 8.00, steps = 2471
    13:51:46 [INFO] train episode 540: reward = -15.00, steps = 1592
    14:00:20 [INFO] train episode 541: reward = -1.00, steps = 2765
    14:08:48 [INFO] train episode 542: reward = -3.00, steps = 2578
    14:15:22 [INFO] train episode 543: reward = -11.00, steps = 1823
    14:23:40 [INFO] train episode 544: reward = -7.00, steps = 2294
    14:32:16 [INFO] train episode 545: reward = 8.00, steps = 2377
    14:38:45 [INFO] train episode 546: reward = -11.00, steps = 1792
    14:47:21 [INFO] train episode 547: reward = -5.00, steps = 2376
    14:56:11 [INFO] train episode 548: reward = -5.00, steps = 2568
    15:04:34 [INFO] train episode 549: reward = 1.00, steps = 2727
    15:12:56 [INFO] train episode 550: reward = -1.00, steps = 2730
    15:20:28 [INFO] train episode 551: reward = -6.00, steps = 2466
    15:24:39 [INFO] train episode 552: reward = -15.00, steps = 1360
    15:31:20 [INFO] train episode 553: reward = 14.00, steps = 2180
    15:36:59 [INFO] train episode 554: reward = 18.00, steps = 1843
    15:44:02 [INFO] train episode 555: reward = -5.00, steps = 2299
    15:52:52 [INFO] train episode 556: reward = -1.00, steps = 2882
    16:00:51 [INFO] train episode 557: reward = 8.00, steps = 2607
    16:08:27 [INFO] train episode 558: reward = -4.00, steps = 2483
    16:16:20 [INFO] train episode 559: reward = 8.00, steps = 2573
    16:25:10 [INFO] train episode 560: reward = -1.00, steps = 2884
    16:28:56 [INFO] train episode 561: reward = -16.00, steps = 1231
    16:36:40 [INFO] train episode 562: reward = -3.00, steps = 2527
    16:42:05 [INFO] train episode 563: reward = 17.00, steps = 1763
    16:49:49 [INFO] train episode 564: reward = 8.00, steps = 2527
    16:57:14 [INFO] train episode 565: reward = -1.00, steps = 2423
    17:03:05 [INFO] train episode 566: reward = -11.00, steps = 1838
    17:08:00 [INFO] train episode 567: reward = -12.00, steps = 1528
    17:15:21 [INFO] train episode 568: reward = 8.00, steps = 2234
    17:22:48 [INFO] train episode 569: reward = 12.00, steps = 2143
    17:31:15 [INFO] train episode 570: reward = -7.00, steps = 2439
    17:39:55 [INFO] train episode 571: reward = 11.00, steps = 2374
    17:48:56 [INFO] train episode 572: reward = -1.00, steps = 2686
    17:59:19 [INFO] train episode 573: reward = 1.00, steps = 3200
    18:05:21 [INFO] train episode 574: reward = 17.00, steps = 1876
    18:13:11 [INFO] train episode 575: reward = -4.00, steps = 2465
    18:23:13 [INFO] train episode 576: reward = -1.00, steps = 2838
    18:30:54 [INFO] train episode 577: reward = 10.00, steps = 2350
    18:39:08 [INFO] train episode 578: reward = 8.00, steps = 2473
    18:46:37 [INFO] train episode 579: reward = 7.00, steps = 2325
    18:51:39 [INFO] train episode 580: reward = -14.00, steps = 1534
    18:59:54 [INFO] train episode 581: reward = -6.00, steps = 2533
    19:07:54 [INFO] train episode 582: reward = 7.00, steps = 2477
    19:15:25 [INFO] train episode 583: reward = 6.00, steps = 2414
    19:23:12 [INFO] train episode 584: reward = -6.00, steps = 2495
    19:32:15 [INFO] train episode 585: reward = 5.00, steps = 2784
    19:39:50 [INFO] train episode 586: reward = 11.00, steps = 2363
    19:45:31 [INFO] train episode 587: reward = 17.00, steps = 1812
    19:53:45 [INFO] train episode 588: reward = 8.00, steps = 2652
    20:02:42 [INFO] train episode 589: reward = 1.00, steps = 2800
    20:10:19 [INFO] train episode 590: reward = 10.00, steps = 2427
    20:18:16 [INFO] train episode 591: reward = -3.00, steps = 2511
    20:26:21 [INFO] train episode 592: reward = 1.00, steps = 2586
    20:32:41 [INFO] train episode 593: reward = 13.00, steps = 2058
    20:41:02 [INFO] train episode 594: reward = -1.00, steps = 2724
    20:49:09 [INFO] train episode 595: reward = 7.00, steps = 2621
    20:56:51 [INFO] train episode 596: reward = 14.00, steps = 2301
    21:05:25 [INFO] train episode 597: reward = -4.00, steps = 2480
    21:10:57 [INFO] train episode 598: reward = -15.00, steps = 1628
    21:19:44 [INFO] train episode 599: reward = 8.00, steps = 2539
    21:27:37 [INFO] train episode 600: reward = 11.00, steps = 2313
    21:36:49 [INFO] train episode 601: reward = -3.00, steps = 2760
    21:45:08 [INFO] train episode 602: reward = 6.00, steps = 2474
    21:53:14 [INFO] train episode 603: reward = 6.00, steps = 2273
    22:00:53 [INFO] train episode 604: reward = 8.00, steps = 2226
    22:08:40 [INFO] train episode 605: reward = -8.00, steps = 2374
    22:16:45 [INFO] train episode 606: reward = -5.00, steps = 2421
    22:25:12 [INFO] train episode 607: reward = -6.00, steps = 2521
    22:31:09 [INFO] train episode 608: reward = 17.00, steps = 1773
    22:36:15 [INFO] train episode 609: reward = -14.00, steps = 1498
    22:44:52 [INFO] train episode 610: reward = -1.00, steps = 2528
    22:49:29 [INFO] train episode 611: reward = -15.00, steps = 1424
    22:56:06 [INFO] train episode 612: reward = -11.00, steps = 2089
    23:02:52 [INFO] train episode 613: reward = 16.00, steps = 2183
    23:06:29 [INFO] train episode 614: reward = -17.00, steps = 1171
    23:13:50 [INFO] train episode 615: reward = 8.00, steps = 2257
    23:19:53 [INFO] train episode 616: reward = 14.00, steps = 1903
    23:27:12 [INFO] train episode 617: reward = 6.00, steps = 2326
    23:32:37 [INFO] train episode 618: reward = 18.00, steps = 1747
    23:40:15 [INFO] train episode 619: reward = 6.00, steps = 2455
    23:48:24 [INFO] train episode 620: reward = 7.00, steps = 2547
    23:53:55 [INFO] train episode 621: reward = -13.00, steps = 1652
    23:58:24 [INFO] train episode 622: reward = -17.00, steps = 1238
    00:07:52 [INFO] train episode 623: reward = 1.00, steps = 2843
    00:16:37 [INFO] train episode 624: reward = -1.00, steps = 2699
    00:25:56 [INFO] train episode 625: reward = 6.00, steps = 2892
    00:32:47 [INFO] train episode 626: reward = 11.00, steps = 2216
    00:39:21 [INFO] train episode 627: reward = -7.00, steps = 1963
    00:44:56 [INFO] train episode 628: reward = 14.00, steps = 1867
    00:51:53 [INFO] train episode 629: reward = -8.00, steps = 2339
    00:58:17 [INFO] train episode 630: reward = 12.00, steps = 2167
    01:03:56 [INFO] train episode 631: reward = -11.00, steps = 1909
    01:08:34 [INFO] train episode 632: reward = 20.00, steps = 1563
    01:14:44 [INFO] train episode 633: reward = 12.00, steps = 2093
    01:22:35 [INFO] train episode 634: reward = 6.00, steps = 2639
    01:30:04 [INFO] train episode 635: reward = 8.00, steps = 2522
    01:35:40 [INFO] train episode 636: reward = -9.00, steps = 1879
    01:40:29 [INFO] train episode 637: reward = -13.00, steps = 1626
    01:47:03 [INFO] train episode 638: reward = -5.00, steps = 2207
    01:55:04 [INFO] train episode 639: reward = -2.00, steps = 2697
    02:03:20 [INFO] train episode 640: reward = 1.00, steps = 2767
    02:09:25 [INFO] train episode 641: reward = -7.00, steps = 2049
    02:17:25 [INFO] train episode 642: reward = -2.00, steps = 2686
    02:23:29 [INFO] train episode 643: reward = 11.00, steps = 2027
    02:27:02 [INFO] train episode 644: reward = -17.00, steps = 1193
    02:35:15 [INFO] train episode 645: reward = 1.00, steps = 2758
    02:40:44 [INFO] train episode 646: reward = 14.00, steps = 1839
    02:46:11 [INFO] train episode 647: reward = -10.00, steps = 1832
    02:52:45 [INFO] train episode 648: reward = -8.00, steps = 2211
    02:59:06 [INFO] train episode 649: reward = 8.00, steps = 2135
    03:05:54 [INFO] train episode 650: reward = 8.00, steps = 2284
    03:14:39 [INFO] train episode 651: reward = 1.00, steps = 2944
    03:18:54 [INFO] train episode 652: reward = -14.00, steps = 1429
    03:25:10 [INFO] train episode 653: reward = 14.00, steps = 2025
    03:32:18 [INFO] train episode 654: reward = -7.00, steps = 2377
    03:37:23 [INFO] train episode 655: reward = -10.00, steps = 1696
    03:44:24 [INFO] train episode 656: reward = 6.00, steps = 2338
    03:51:13 [INFO] train episode 657: reward = -7.00, steps = 2281
    03:58:31 [INFO] train episode 658: reward = 1.00, steps = 2456
    04:04:27 [INFO] train episode 659: reward = 11.00, steps = 1988
    04:12:02 [INFO] train episode 660: reward = 1.00, steps = 2549
    04:20:10 [INFO] train episode 661: reward = -1.00, steps = 2713
    04:28:05 [INFO] train episode 662: reward = -3.00, steps = 2620
    04:32:36 [INFO] train episode 663: reward = -14.00, steps = 1513
    04:37:55 [INFO] train episode 664: reward = -10.00, steps = 1789
    04:44:00 [INFO] train episode 665: reward = -9.00, steps = 2040
    04:48:57 [INFO] train episode 666: reward = -14.00, steps = 1667
    04:56:27 [INFO] train episode 667: reward = 1.00, steps = 2525
    05:01:22 [INFO] train episode 668: reward = -14.00, steps = 1658
    05:05:26 [INFO] train episode 669: reward = -16.00, steps = 1362
    05:13:02 [INFO] train episode 670: reward = 4.00, steps = 2554
    05:19:20 [INFO] train episode 671: reward = 8.00, steps = 2117
    05:26:19 [INFO] train episode 672: reward = -5.00, steps = 2329
    05:33:37 [INFO] train episode 673: reward = -6.00, steps = 2444
    05:40:36 [INFO] train episode 674: reward = 6.00, steps = 2352
    05:47:46 [INFO] train episode 675: reward = 1.00, steps = 2414
    05:52:50 [INFO] train episode 676: reward = -12.00, steps = 1707
    05:58:12 [INFO] train episode 677: reward = -12.00, steps = 1802
    06:05:46 [INFO] train episode 678: reward = -1.00, steps = 2538
    06:11:02 [INFO] train episode 679: reward = -10.00, steps = 1779
    06:16:27 [INFO] train episode 680: reward = 14.00, steps = 1821
    06:25:02 [INFO] train episode 681: reward = -1.00, steps = 2880
    06:32:42 [INFO] train episode 682: reward = 1.00, steps = 2572
    06:40:11 [INFO] train episode 683: reward = -1.00, steps = 2511
    06:46:49 [INFO] train episode 684: reward = 8.00, steps = 2227
    06:52:27 [INFO] train episode 685: reward = -11.00, steps = 1889
    06:59:15 [INFO] train episode 686: reward = -2.00, steps = 2283
    07:06:02 [INFO] train episode 687: reward = 8.00, steps = 2274
    07:14:14 [INFO] train episode 688: reward = 1.00, steps = 2740
    07:20:54 [INFO] train episode 689: reward = -7.00, steps = 2234
    07:27:11 [INFO] train episode 690: reward = 12.00, steps = 2093
    07:34:45 [INFO] train episode 691: reward = -6.00, steps = 2520
    07:40:05 [INFO] train episode 692: reward = 16.00, steps = 1748
    07:45:37 [INFO] train episode 693: reward = -10.00, steps = 1852
    07:51:14 [INFO] train episode 694: reward = -9.00, steps = 1884
    07:58:45 [INFO] train episode 695: reward = -1.00, steps = 2524
    08:06:30 [INFO] train episode 696: reward = -1.00, steps = 2598
    08:13:03 [INFO] train episode 697: reward = 7.00, steps = 2204
    08:18:24 [INFO] train episode 698: reward = 18.00, steps = 1797
    08:24:27 [INFO] train episode 699: reward = -5.00, steps = 2364
    08:31:22 [INFO] train episode 700: reward = -8.00, steps = 2326
    08:36:07 [INFO] train episode 701: reward = -13.00, steps = 1605
    08:42:48 [INFO] train episode 702: reward = 9.00, steps = 2253
    08:50:06 [INFO] train episode 703: reward = 8.00, steps = 2454
    08:56:21 [INFO] train episode 704: reward = 14.00, steps = 2107
    09:01:01 [INFO] train episode 705: reward = -13.00, steps = 1565
    09:09:00 [INFO] train episode 706: reward = -1.00, steps = 2685
    09:14:48 [INFO] train episode 707: reward = -12.00, steps = 1944
    09:21:45 [INFO] train episode 708: reward = 10.00, steps = 2215
    09:28:13 [INFO] train episode 709: reward = 13.00, steps = 2175
    09:37:25 [INFO] train episode 710: reward = 1.00, steps = 2709
    09:42:30 [INFO] train episode 711: reward = 19.00, steps = 1618
    09:48:42 [INFO] train episode 712: reward = -8.00, steps = 1967
    09:56:05 [INFO] train episode 713: reward = 14.00, steps = 2222
    10:04:01 [INFO] train episode 714: reward = 12.00, steps = 2351
    10:10:31 [INFO] train episode 715: reward = 17.00, steps = 1918
    10:17:43 [INFO] train episode 716: reward = -6.00, steps = 2192
    10:25:25 [INFO] train episode 717: reward = 13.00, steps = 2382
    10:33:31 [INFO] train episode 718: reward = 8.00, steps = 2420
    10:40:26 [INFO] train episode 719: reward = 14.00, steps = 2086
    10:49:49 [INFO] train episode 720: reward = 1.00, steps = 2874
    10:57:52 [INFO] train episode 721: reward = 1.00, steps = 2482
    11:05:15 [INFO] train episode 722: reward = 8.00, steps = 2283
    11:12:53 [INFO] train episode 723: reward = 6.00, steps = 2358
    11:20:00 [INFO] train episode 724: reward = 10.00, steps = 2213
    11:28:04 [INFO] train episode 725: reward = -4.00, steps = 2523
    11:34:34 [INFO] train episode 726: reward = -9.00, steps = 2071
    11:41:19 [INFO] train episode 727: reward = -9.00, steps = 2176
    11:45:43 [INFO] train episode 728: reward = -16.00, steps = 1415
    11:54:27 [INFO] train episode 729: reward = -1.00, steps = 2766
    12:01:44 [INFO] train episode 730: reward = -6.00, steps = 2331
    12:05:25 [INFO] train episode 731: reward = -16.00, steps = 1174
    12:09:52 [INFO] train episode 732: reward = -16.00, steps = 1432
    12:15:08 [INFO] train episode 733: reward = 18.00, steps = 1700
    12:22:12 [INFO] train episode 734: reward = 9.00, steps = 2273
    12:27:20 [INFO] train episode 735: reward = -14.00, steps = 1649
    12:31:52 [INFO] train episode 736: reward = -16.00, steps = 1464
    12:36:30 [INFO] train episode 737: reward = -14.00, steps = 1495
    12:40:34 [INFO] train episode 738: reward = -16.00, steps = 1311
    12:44:51 [INFO] train episode 739: reward = -15.00, steps = 1377
    12:49:25 [INFO] train episode 740: reward = -16.00, steps = 1471
    12:53:17 [INFO] train episode 741: reward = -16.00, steps = 1245
    13:01:14 [INFO] train episode 742: reward = 1.00, steps = 2543
    13:07:06 [INFO] train episode 743: reward = -9.00, steps = 1875
    13:15:47 [INFO] train episode 744: reward = -1.00, steps = 2775
    13:22:12 [INFO] train episode 745: reward = -10.00, steps = 2047
    13:30:26 [INFO] train episode 746: reward = -3.00, steps = 2637
    13:35:08 [INFO] train episode 747: reward = -15.00, steps = 1506
    13:40:45 [INFO] train episode 748: reward = -10.00, steps = 1808
    13:46:02 [INFO] train episode 749: reward = -12.00, steps = 1698
    13:51:51 [INFO] train episode 750: reward = 17.00, steps = 1862
    13:58:42 [INFO] train episode 751: reward = 10.00, steps = 2205
    14:05:46 [INFO] train episode 752: reward = -6.00, steps = 2276
    14:11:35 [INFO] train episode 753: reward = 16.00, steps = 1877
    14:18:57 [INFO] train episode 754: reward = 8.00, steps = 2379
    14:24:35 [INFO] train episode 755: reward = -13.00, steps = 1811
    14:31:48 [INFO] train episode 756: reward = -6.00, steps = 2327
    14:39:27 [INFO] train episode 757: reward = -6.00, steps = 2480
    14:45:08 [INFO] train episode 758: reward = -11.00, steps = 1835
    14:50:59 [INFO] train episode 759: reward = -12.00, steps = 1898
    14:57:11 [INFO] train episode 760: reward = -10.00, steps = 2002
    15:03:22 [INFO] train episode 761: reward = 14.00, steps = 1968
    15:06:48 [INFO] train episode 762: reward = -18.00, steps = 1063
    15:13:20 [INFO] train episode 763: reward = -10.00, steps = 1891
    15:18:26 [INFO] train episode 764: reward = -11.00, steps = 1554
    15:25:27 [INFO] train episode 765: reward = -8.00, steps = 2228
    15:29:53 [INFO] train episode 766: reward = -16.00, steps = 1372
    15:38:01 [INFO] train episode 767: reward = -1.00, steps = 2625
    15:45:29 [INFO] train episode 768: reward = 11.00, steps = 2342
    15:52:14 [INFO] train episode 769: reward = 10.00, steps = 2157
    16:00:52 [INFO] train episode 770: reward = 1.00, steps = 2758
    16:06:03 [INFO] train episode 771: reward = 19.00, steps = 1656
    16:12:29 [INFO] train episode 772: reward = -7.00, steps = 2065
    16:20:29 [INFO] train episode 773: reward = 8.00, steps = 2577
    16:27:56 [INFO] train episode 774: reward = -6.00, steps = 2375
    16:35:13 [INFO] train episode 775: reward = 9.00, steps = 2294
    16:40:19 [INFO] train episode 776: reward = -14.00, steps = 1579
    16:53:19 [INFO] train episode 777: reward = 1.00, steps = 2806
    17:01:34 [INFO] train episode 778: reward = 8.00, steps = 2221
    17:09:35 [INFO] train episode 779: reward = 12.00, steps = 2398
    17:18:17 [INFO] train episode 780: reward = -1.00, steps = 2722
    17:26:34 [INFO] train episode 781: reward = 9.00, steps = 2579
    17:32:33 [INFO] train episode 782: reward = 17.00, steps = 1865
    17:40:39 [INFO] train episode 783: reward = -3.00, steps = 2535
    17:49:16 [INFO] train episode 784: reward = 8.00, steps = 2716
    17:55:00 [INFO] train episode 785: reward = 17.00, steps = 1804
    18:03:42 [INFO] train episode 786: reward = 1.00, steps = 2694
    18:11:28 [INFO] train episode 787: reward = -1.00, steps = 2373
    18:17:19 [INFO] train episode 788: reward = 17.00, steps = 1821
    18:24:48 [INFO] train episode 789: reward = 8.00, steps = 2325
    18:33:13 [INFO] train episode 790: reward = 1.00, steps = 2642
    18:39:57 [INFO] train episode 791: reward = 14.00, steps = 2111
    18:45:58 [INFO] train episode 792: reward = 17.00, steps = 1896
    18:54:22 [INFO] train episode 793: reward = 3.00, steps = 2675
    19:03:12 [INFO] train episode 794: reward = -3.00, steps = 2596
    19:14:14 [INFO] train episode 795: reward = 8.00, steps = 2555
    19:21:42 [INFO] train episode 796: reward = 18.00, steps = 2091
    19:31:04 [INFO] train episode 797: reward = -1.00, steps = 2420
    19:37:11 [INFO] train episode 798: reward = -13.00, steps = 1732
    19:44:34 [INFO] train episode 799: reward = -2.00, steps = 2241
    19:51:08 [INFO] train episode 800: reward = 14.00, steps = 2016
    19:59:56 [INFO] train episode 801: reward = 1.00, steps = 2794
    20:06:07 [INFO] train episode 802: reward = 14.00, steps = 1916
    20:11:03 [INFO] train episode 803: reward = -16.00, steps = 1500
    20:16:44 [INFO] train episode 804: reward = -14.00, steps = 1749
    20:22:21 [INFO] train episode 805: reward = 17.00, steps = 1723
    20:27:55 [INFO] train episode 806: reward = 17.00, steps = 1775
    20:32:13 [INFO] train episode 807: reward = -16.00, steps = 1389
    20:39:41 [INFO] train episode 808: reward = 8.00, steps = 2397
    20:47:20 [INFO] train episode 809: reward = -10.00, steps = 2157
    20:50:15 [INFO] train episode 810: reward = -21.00, steps = 874
    20:59:33 [INFO] train episode 811: reward = -3.00, steps = 2690
    21:05:13 [INFO] train episode 812: reward = -13.00, steps = 1641
    21:12:55 [INFO] train episode 813: reward = -5.00, steps = 2089
    21:21:54 [INFO] train episode 814: reward = 11.00, steps = 2043
    21:28:16 [INFO] train episode 815: reward = -17.00, steps = 1482
    21:37:14 [INFO] train episode 816: reward = 10.00, steps = 2104
    21:44:37 [INFO] train episode 817: reward = -13.00, steps = 1653
    21:50:01 [INFO] train episode 818: reward = -17.00, steps = 1282
    22:00:53 [INFO] train episode 819: reward = -3.00, steps = 2601
    22:10:51 [INFO] train episode 820: reward = -1.00, steps = 2381
    22:17:03 [INFO] train episode 821: reward = -14.00, steps = 1484
    22:22:00 [INFO] train episode 822: reward = -16.00, steps = 1199
    22:30:20 [INFO] train episode 823: reward = -6.00, steps = 2287
    22:33:32 [INFO] train episode 824: reward = -20.00, steps = 1017
    22:38:51 [INFO] train episode 825: reward = -12.00, steps = 1692
    22:44:53 [INFO] train episode 826: reward = -10.00, steps = 1883
    22:51:31 [INFO] train episode 827: reward = 12.00, steps = 2054
    22:55:13 [INFO] train episode 828: reward = -19.00, steps = 1099
    23:01:46 [INFO] train episode 829: reward = -6.00, steps = 2119
    23:10:57 [INFO] train episode 830: reward = 1.00, steps = 2981
    23:16:55 [INFO] train episode 831: reward = 16.00, steps = 1916
    23:23:57 [INFO] train episode 832: reward = -4.00, steps = 2286
    23:31:18 [INFO] train episode 833: reward = -4.00, steps = 2387
    23:37:37 [INFO] train episode 834: reward = -11.00, steps = 2039
    23:40:24 [INFO] train episode 835: reward = -21.00, steps = 900
    23:44:02 [INFO] train episode 836: reward = -18.00, steps = 1179
    23:50:01 [INFO] train episode 837: reward = -11.00, steps = 1950
    23:55:23 [INFO] train episode 838: reward = -11.00, steps = 1748
    00:00:13 [INFO] train episode 839: reward = -13.00, steps = 1557
    00:07:13 [INFO] train episode 840: reward = -8.00, steps = 2264
    00:12:36 [INFO] train episode 841: reward = -13.00, steps = 1716
    00:19:19 [INFO] train episode 842: reward = 14.00, steps = 2040
    00:26:37 [INFO] train episode 843: reward = -5.00, steps = 2443
    00:31:05 [INFO] train episode 844: reward = -15.00, steps = 1491
    00:36:15 [INFO] train episode 845: reward = 18.00, steps = 1746
    00:43:05 [INFO] train episode 846: reward = 14.00, steps = 2320
    00:47:07 [INFO] train episode 847: reward = -15.00, steps = 1368
    00:52:19 [INFO] train episode 848: reward = -10.00, steps = 1764
    00:58:06 [INFO] train episode 849: reward = 14.00, steps = 1955
    01:05:45 [INFO] train episode 850: reward = 1.00, steps = 2582
    01:09:28 [INFO] train episode 851: reward = -17.00, steps = 1257
    01:16:41 [INFO] train episode 852: reward = -3.00, steps = 2439
    01:23:29 [INFO] train episode 853: reward = -6.00, steps = 2298
    01:29:14 [INFO] train episode 854: reward = -10.00, steps = 1938
    01:32:36 [INFO] train episode 855: reward = -17.00, steps = 1134
    01:38:58 [INFO] train episode 856: reward = 11.00, steps = 2160
    01:44:53 [INFO] train episode 857: reward = -10.00, steps = 2010
    01:50:55 [INFO] train episode 858: reward = 12.00, steps = 2043
    01:56:20 [INFO] train episode 859: reward = -12.00, steps = 1829
    02:03:47 [INFO] train episode 860: reward = 5.00, steps = 2509
    02:11:19 [INFO] train episode 861: reward = 1.00, steps = 2543
    02:15:20 [INFO] train episode 862: reward = -15.00, steps = 1352
    02:22:01 [INFO] train episode 863: reward = -6.00, steps = 2242
    02:29:07 [INFO] train episode 864: reward = 1.00, steps = 2378
    02:34:34 [INFO] train episode 865: reward = -11.00, steps = 1823
    02:41:52 [INFO] train episode 866: reward = 1.00, steps = 2449
    02:47:59 [INFO] train episode 867: reward = 14.00, steps = 2006
    02:56:35 [INFO] train episode 868: reward = -2.00, steps = 2890
    03:02:46 [INFO] train episode 869: reward = -11.00, steps = 2070
    03:10:27 [INFO] train episode 870: reward = 1.00, steps = 2575
    03:18:47 [INFO] train episode 871: reward = -1.00, steps = 2799
    03:24:02 [INFO] train episode 872: reward = -12.00, steps = 1753
    03:28:45 [INFO] train episode 873: reward = -14.00, steps = 1583
    03:36:52 [INFO] train episode 874: reward = 1.00, steps = 2723
    03:43:54 [INFO] train episode 875: reward = 8.00, steps = 2355
    03:47:50 [INFO] train episode 876: reward = -17.00, steps = 1314
    03:54:29 [INFO] train episode 877: reward = 8.00, steps = 2223
    04:01:39 [INFO] train episode 878: reward = 1.00, steps = 2391
    04:10:21 [INFO] train episode 879: reward = -1.00, steps = 2838
    04:14:41 [INFO] train episode 880: reward = -15.00, steps = 1450
    04:22:05 [INFO] train episode 881: reward = -6.00, steps = 2469
    04:26:48 [INFO] train episode 882: reward = -16.00, steps = 1581
    04:32:13 [INFO] train episode 883: reward = 17.00, steps = 1807
    04:38:15 [INFO] train episode 884: reward = 14.00, steps = 2020
    04:46:54 [INFO] train episode 885: reward = -1.00, steps = 2893
    04:52:55 [INFO] train episode 886: reward = 12.00, steps = 2016
    04:59:46 [INFO] train episode 887: reward = 8.00, steps = 2292
    05:05:58 [INFO] train episode 888: reward = 14.00, steps = 2082
    05:09:56 [INFO] train episode 889: reward = -18.00, steps = 1326
    05:14:02 [INFO] train episode 890: reward = -16.00, steps = 1372
    05:21:33 [INFO] train episode 891: reward = -4.00, steps = 2515
    05:25:28 [INFO] train episode 892: reward = -18.00, steps = 1314
    05:32:14 [INFO] train episode 893: reward = 11.00, steps = 2273
    05:40:42 [INFO] train episode 894: reward = 3.00, steps = 2848
    05:45:39 [INFO] train episode 895: reward = -12.00, steps = 1663
    05:51:52 [INFO] train episode 896: reward = 12.00, steps = 2095
    05:59:36 [INFO] train episode 897: reward = -2.00, steps = 2595
    06:04:02 [INFO] train episode 898: reward = -15.00, steps = 1490
    06:07:31 [INFO] train episode 899: reward = -19.00, steps = 1168
    06:16:08 [INFO] train episode 900: reward = 1.00, steps = 2884
    06:23:55 [INFO] train episode 901: reward = 1.00, steps = 2588
    06:30:06 [INFO] train episode 902: reward = 14.00, steps = 2075
    06:36:13 [INFO] train episode 903: reward = 16.00, steps = 2052
    06:43:22 [INFO] train episode 904: reward = -1.00, steps = 2407
    06:49:38 [INFO] train episode 905: reward = 12.00, steps = 2107
    06:56:09 [INFO] train episode 906: reward = 16.00, steps = 2197
    07:02:33 [INFO] train episode 907: reward = -5.00, steps = 2151
    07:09:01 [INFO] train episode 908: reward = -9.00, steps = 2177
    07:16:17 [INFO] train episode 909: reward = 8.00, steps = 2443
    07:23:06 [INFO] train episode 910: reward = 12.00, steps = 2278
    07:29:11 [INFO] train episode 911: reward = 11.00, steps = 2051
    07:31:58 [INFO] train episode 912: reward = -19.00, steps = 938
    07:38:49 [INFO] train episode 913: reward = 11.00, steps = 2310
    07:46:50 [INFO] train episode 914: reward = -3.00, steps = 2708
    07:53:13 [INFO] train episode 915: reward = -7.00, steps = 2168
    07:59:16 [INFO] train episode 916: reward = 16.00, steps = 2036
    08:06:08 [INFO] train episode 917: reward = 12.00, steps = 2304
    08:14:36 [INFO] train episode 918: reward = -2.00, steps = 2850
    08:20:48 [INFO] train episode 919: reward = -8.00, steps = 2083
    08:27:17 [INFO] train episode 920: reward = 14.00, steps = 1942
    08:36:11 [INFO] train episode 921: reward = -1.00, steps = 2817
    08:41:02 [INFO] train episode 922: reward = -12.00, steps = 1557
    08:49:36 [INFO] train episode 923: reward = 1.00, steps = 2624
    08:57:10 [INFO] train episode 924: reward = -4.00, steps = 2426
    09:02:07 [INFO] train episode 925: reward = -12.00, steps = 1589
    09:09:03 [INFO] train episode 926: reward = 8.00, steps = 2215
    09:17:10 [INFO] train episode 927: reward = -2.00, steps = 2636
    09:23:04 [INFO] train episode 928: reward = -11.00, steps = 1910
    09:29:31 [INFO] train episode 929: reward = 17.00, steps = 2100
    09:37:07 [INFO] train episode 930: reward = 8.00, steps = 2503
    09:44:42 [INFO] train episode 931: reward = 8.00, steps = 2505
    09:49:57 [INFO] train episode 932: reward = -10.00, steps = 1735
    09:57:23 [INFO] train episode 933: reward = -3.00, steps = 2440
    10:03:03 [INFO] train episode 934: reward = 16.00, steps = 1841
    10:11:04 [INFO] train episode 935: reward = -2.00, steps = 2611
    10:19:21 [INFO] train episode 936: reward = -1.00, steps = 2716
    10:26:12 [INFO] train episode 937: reward = 8.00, steps = 2233
    10:33:19 [INFO] train episode 938: reward = 11.00, steps = 2321
    10:39:35 [INFO] train episode 939: reward = 12.00, steps = 2056
    10:46:22 [INFO] train episode 940: reward = -8.00, steps = 2231
    10:51:06 [INFO] train episode 941: reward = -15.00, steps = 1557
    10:57:57 [INFO] train episode 942: reward = -7.00, steps = 2253
    11:03:50 [INFO] train episode 943: reward = 12.00, steps = 1934
    11:09:12 [INFO] train episode 944: reward = -10.00, steps = 1764
    11:15:24 [INFO] train episode 945: reward = 11.00, steps = 2039
    11:18:27 [INFO] train episode 946: reward = -19.00, steps = 1002
    11:25:40 [INFO] train episode 947: reward = -6.00, steps = 2369
    11:28:53 [INFO] train episode 948: reward = -18.00, steps = 1059
    11:36:12 [INFO] train episode 949: reward = -6.00, steps = 2403
    11:42:40 [INFO] train episode 950: reward = -9.00, steps = 2133
    11:45:14 [INFO] train episode 951: reward = -20.00, steps = 842
    11:53:19 [INFO] train episode 952: reward = 1.00, steps = 2670
    12:00:31 [INFO] train episode 953: reward = -1.00, steps = 2376
    12:07:16 [INFO] train episode 954: reward = 8.00, steps = 2219
    12:10:42 [INFO] train episode 955: reward = -16.00, steps = 1127
    12:17:05 [INFO] train episode 956: reward = -5.00, steps = 2093
    12:23:15 [INFO] train episode 957: reward = -9.00, steps = 2018
    12:29:39 [INFO] train episode 958: reward = -7.00, steps = 2104
    12:34:08 [INFO] train episode 959: reward = -16.00, steps = 1471
    12:37:41 [INFO] train episode 960: reward = -19.00, steps = 1172
    12:44:35 [INFO] train episode 961: reward = 11.00, steps = 2273
    12:49:03 [INFO] train episode 962: reward = -14.00, steps = 1475
    12:55:48 [INFO] train episode 963: reward = 8.00, steps = 2222
    13:02:23 [INFO] train episode 964: reward = -7.00, steps = 2169
    13:09:01 [INFO] train episode 965: reward = -7.00, steps = 2190
    13:13:16 [INFO] train episode 966: reward = -15.00, steps = 1394
    13:19:00 [INFO] train episode 967: reward = 18.00, steps = 1739
    13:25:37 [INFO] train episode 968: reward = 17.00, steps = 1804
    13:29:23 [INFO] train episode 969: reward = -19.00, steps = 1017
    13:34:15 [INFO] train episode 970: reward = -16.00, steps = 1308
    13:37:35 [INFO] train episode 971: reward = -19.00, steps = 919
    13:46:35 [INFO] train episode 972: reward = -2.00, steps = 2579
    13:54:29 [INFO] train episode 973: reward = 10.00, steps = 2354
    14:00:55 [INFO] train episode 974: reward = 11.00, steps = 2020
    14:07:30 [INFO] train episode 975: reward = 14.00, steps = 2093
    14:16:23 [INFO] train episode 976: reward = -2.00, steps = 2866
    14:21:44 [INFO] train episode 977: reward = 18.00, steps = 1726
    14:28:24 [INFO] train episode 978: reward = -7.00, steps = 2170
    14:31:26 [INFO] train episode 979: reward = -19.00, steps = 989
    14:36:23 [INFO] train episode 980: reward = -13.00, steps = 1619
    14:43:37 [INFO] train episode 981: reward = -4.00, steps = 2379
    14:46:56 [INFO] train episode 982: reward = -18.00, steps = 1090
    14:55:30 [INFO] train episode 983: reward = -1.00, steps = 2836
    15:00:58 [INFO] train episode 984: reward = -9.00, steps = 1801
    15:06:44 [INFO] train episode 985: reward = -10.00, steps = 1904
    15:12:51 [INFO] train episode 986: reward = 13.00, steps = 2008
    15:18:15 [INFO] train episode 987: reward = 19.00, steps = 1785
    15:27:00 [INFO] train episode 988: reward = -1.00, steps = 2879
    15:30:07 [INFO] train episode 989: reward = -18.00, steps = 1032
    15:36:02 [INFO] train episode 990: reward = -9.00, steps = 1956
    15:39:42 [INFO] train episode 991: reward = -16.00, steps = 1210
    15:47:51 [INFO] train episode 992: reward = 1.00, steps = 2694
    15:52:22 [INFO] train episode 993: reward = -14.00, steps = 1495
    15:57:50 [INFO] train episode 994: reward = -12.00, steps = 1821
    16:04:31 [INFO] train episode 995: reward = 8.00, steps = 2212
    16:08:59 [INFO] train episode 996: reward = -14.00, steps = 1479
    16:13:43 [INFO] train episode 997: reward = -14.00, steps = 1562
    16:21:03 [INFO] train episode 998: reward = 11.00, steps = 2416
    16:23:51 [INFO] train episode 999: reward = -19.00, steps = 915
    16:31:06 [INFO] train episode 1000: reward = -6.00, steps = 2394
    16:34:27 [INFO] train episode 1001: reward = -18.00, steps = 1100
    16:39:33 [INFO] train episode 1002: reward = -13.00, steps = 1680
    16:46:35 [INFO] train episode 1003: reward = 14.00, steps = 2323
    16:54:55 [INFO] train episode 1004: reward = 1.00, steps = 2756
    17:00:42 [INFO] train episode 1005: reward = 15.00, steps = 1902
    17:07:23 [INFO] train episode 1006: reward = -6.00, steps = 2181
    17:11:28 [INFO] train episode 1007: reward = -15.00, steps = 1326
    17:18:59 [INFO] train episode 1008: reward = 10.00, steps = 2482
    17:25:36 [INFO] train episode 1009: reward = -7.00, steps = 2173
    17:30:15 [INFO] train episode 1010: reward = -14.00, steps = 1524
    17:35:40 [INFO] train episode 1011: reward = 18.00, steps = 1779
    17:40:57 [INFO] train episode 1012: reward = -12.00, steps = 1732
    17:44:46 [INFO] train episode 1013: reward = -17.00, steps = 1257
    17:50:38 [INFO] train episode 1014: reward = 16.00, steps = 1942
    17:57:43 [INFO] train episode 1015: reward = 11.00, steps = 2345
    18:04:31 [INFO] train episode 1016: reward = 14.00, steps = 2235
    18:08:28 [INFO] train episode 1017: reward = -16.00, steps = 1299
    18:13:39 [INFO] train episode 1018: reward = 19.00, steps = 1692
    18:21:56 [INFO] train episode 1019: reward = -1.00, steps = 2746
    18:29:24 [INFO] train episode 1020: reward = -5.00, steps = 2466
    18:34:54 [INFO] train episode 1021: reward = -12.00, steps = 1818
    18:42:38 [INFO] train episode 1022: reward = -3.00, steps = 2561
    18:46:56 [INFO] train episode 1023: reward = -14.00, steps = 1430
    18:54:01 [INFO] train episode 1024: reward = 3.00, steps = 2343
    18:57:26 [INFO] train episode 1025: reward = -18.00, steps = 1125
    19:05:29 [INFO] train episode 1026: reward = -5.00, steps = 2664
    19:09:28 [INFO] train episode 1027: reward = -16.00, steps = 1328
    19:15:15 [INFO] train episode 1028: reward = 16.00, steps = 1925
    19:22:08 [INFO] train episode 1029: reward = -8.00, steps = 2283
    19:29:27 [INFO] train episode 1030: reward = 10.00, steps = 2436
    19:36:47 [INFO] train episode 1031: reward = -6.00, steps = 2411
    19:43:47 [INFO] train episode 1032: reward = -8.00, steps = 2331
    19:46:42 [INFO] train episode 1033: reward = -19.00, steps = 975
    19:54:41 [INFO] train episode 1034: reward = 8.00, steps = 2677
    20:01:34 [INFO] train episode 1035: reward = 11.00, steps = 2300
    20:08:07 [INFO] train episode 1036: reward = -9.00, steps = 2187
    20:15:46 [INFO] train episode 1037: reward = 9.00, steps = 2556
    20:22:01 [INFO] train episode 1038: reward = 18.00, steps = 2074
    20:28:53 [INFO] train episode 1039: reward = 11.00, steps = 2294
    20:34:02 [INFO] train episode 1040: reward = 19.00, steps = 1713
    20:40:28 [INFO] train episode 1041: reward = 15.00, steps = 2147
    20:46:34 [INFO] train episode 1042: reward = 11.00, steps = 2033
    20:52:13 [INFO] train episode 1043: reward = -12.00, steps = 1889
    20:58:02 [INFO] train episode 1044: reward = 16.00, steps = 1942
    21:03:31 [INFO] train episode 1045: reward = 16.00, steps = 1830
    21:09:02 [INFO] train episode 1046: reward = -12.00, steps = 1831
    21:14:56 [INFO] train episode 1047: reward = -11.00, steps = 1968
    21:22:30 [INFO] train episode 1048: reward = 11.00, steps = 2506
    21:26:38 [INFO] train episode 1049: reward = -17.00, steps = 1359
    21:33:51 [INFO] train episode 1050: reward = -5.00, steps = 2380
    21:40:06 [INFO] train episode 1051: reward = 11.00, steps = 2053
    21:44:11 [INFO] train episode 1052: reward = -15.00, steps = 1351
    21:51:30 [INFO] train episode 1053: reward = 1.00, steps = 2435
    21:59:54 [INFO] train episode 1054: reward = -1.00, steps = 2770
    22:08:24 [INFO] train episode 1055: reward = -1.00, steps = 2799
    22:15:51 [INFO] train episode 1056: reward = -2.00, steps = 2459
    22:20:35 [INFO] train episode 1057: reward = -14.00, steps = 1572
    22:26:18 [INFO] train episode 1058: reward = -10.00, steps = 1871
    22:29:34 [INFO] train episode 1059: reward = -20.00, steps = 1073
    22:34:33 [INFO] train episode 1060: reward = -12.00, steps = 1641
    22:41:21 [INFO] train episode 1061: reward = -8.00, steps = 2243
    22:49:14 [INFO] train episode 1062: reward = 3.00, steps = 2618
    22:52:30 [INFO] train episode 1063: reward = -18.00, steps = 1089
    22:59:56 [INFO] train episode 1064: reward = 1.00, steps = 2470
    23:08:40 [INFO] train episode 1065: reward = -1.00, steps = 2905
    23:16:23 [INFO] train episode 1066: reward = 4.00, steps = 2562
    23:21:22 [INFO] train episode 1067: reward = 19.00, steps = 1631
    23:27:28 [INFO] train episode 1068: reward = 20.00, steps = 2001
    23:34:32 [INFO] train episode 1069: reward = 14.00, steps = 2313
    23:34:33 [INFO] ==== test ====
    23:34:56 [INFO] test episode 0: reward = 19.00, steps = 1687
    23:35:19 [INFO] test episode 1: reward = 19.00, steps = 1685
    23:35:43 [INFO] test episode 2: reward = 19.00, steps = 1687
    23:36:10 [INFO] test episode 3: reward = 20.00, steps = 2004
    23:36:32 [INFO] test episode 4: reward = 20.00, steps = 1565
    23:36:53 [INFO] test episode 5: reward = 20.00, steps = 1566
    23:37:21 [INFO] test episode 6: reward = 20.00, steps = 1991
    23:37:42 [INFO] test episode 7: reward = 20.00, steps = 1564
    23:38:05 [INFO] test episode 8: reward = 19.00, steps = 1687
    23:38:33 [INFO] test episode 9: reward = 20.00, steps = 1992
    23:38:56 [INFO] test episode 10: reward = 19.00, steps = 1687
    23:39:18 [INFO] test episode 11: reward = 20.00, steps = 1566
    23:39:41 [INFO] test episode 12: reward = 19.00, steps = 1683
    23:40:08 [INFO] test episode 13: reward = 20.00, steps = 1990
    23:40:32 [INFO] test episode 14: reward = 19.00, steps = 1685
    23:40:59 [INFO] test episode 15: reward = 20.00, steps = 1994
    23:41:21 [INFO] test episode 16: reward = 20.00, steps = 1562
    23:41:48 [INFO] test episode 17: reward = 20.00, steps = 1993
    23:42:11 [INFO] test episode 18: reward = 19.00, steps = 1688
    23:42:34 [INFO] test episode 19: reward = 19.00, steps = 1688
    23:42:58 [INFO] test episode 20: reward = 19.00, steps = 1684
    23:43:26 [INFO] test episode 21: reward = 20.00, steps = 2003
    23:43:53 [INFO] test episode 22: reward = 20.00, steps = 1997
    23:44:21 [INFO] test episode 23: reward = 20.00, steps = 1993
    23:44:48 [INFO] test episode 24: reward = 20.00, steps = 2007
    23:45:12 [INFO] test episode 25: reward = 19.00, steps = 1686
    23:45:35 [INFO] test episode 26: reward = 19.00, steps = 1684
    23:45:58 [INFO] test episode 27: reward = 19.00, steps = 1685
    23:46:26 [INFO] test episode 28: reward = 20.00, steps = 1997
    23:46:49 [INFO] test episode 29: reward = 19.00, steps = 1683
    23:47:16 [INFO] test episode 30: reward = 20.00, steps = 1993
    23:47:39 [INFO] test episode 31: reward = 19.00, steps = 1683
    23:48:07 [INFO] test episode 32: reward = 20.00, steps = 2005
    23:48:35 [INFO] test episode 33: reward = 20.00, steps = 2005
    23:49:02 [INFO] test episode 34: reward = 20.00, steps = 2006
    23:49:24 [INFO] test episode 35: reward = 20.00, steps = 1566
    23:49:45 [INFO] test episode 36: reward = 20.00, steps = 1563
    23:50:13 [INFO] test episode 37: reward = 20.00, steps = 1992
    23:50:40 [INFO] test episode 38: reward = 20.00, steps = 2003
    23:51:03 [INFO] test episode 39: reward = 19.00, steps = 1688
    23:51:25 [INFO] test episode 40: reward = 20.00, steps = 1565
    23:51:48 [INFO] test episode 41: reward = 19.00, steps = 1683
    23:52:11 [INFO] test episode 42: reward = 19.00, steps = 1690
    23:52:39 [INFO] test episode 43: reward = 20.00, steps = 2005
    23:53:02 [INFO] test episode 44: reward = 19.00, steps = 1686
    23:53:25 [INFO] test episode 45: reward = 19.00, steps = 1688
    23:53:52 [INFO] test episode 46: reward = 20.00, steps = 1993
    23:54:15 [INFO] test episode 47: reward = 19.00, steps = 1684
    23:54:37 [INFO] test episode 48: reward = 20.00, steps = 1567
    23:55:04 [INFO] test episode 49: reward = 20.00, steps = 2005
    23:55:32 [INFO] test episode 50: reward = 20.00, steps = 2001
    23:55:59 [INFO] test episode 51: reward = 20.00, steps = 2002
    23:56:27 [INFO] test episode 52: reward = 20.00, steps = 2006
    23:56:55 [INFO] test episode 53: reward = 20.00, steps = 2006
    23:57:18 [INFO] test episode 54: reward = 19.00, steps = 1688
    23:57:39 [INFO] test episode 55: reward = 20.00, steps = 1561
    23:58:03 [INFO] test episode 56: reward = 19.00, steps = 1686
    23:58:30 [INFO] test episode 57: reward = 20.00, steps = 1996
    23:58:58 [INFO] test episode 58: reward = 20.00, steps = 2001
    23:59:25 [INFO] test episode 59: reward = 20.00, steps = 1994
    23:59:53 [INFO] test episode 60: reward = 20.00, steps = 1991
    00:00:18 [INFO] test episode 61: reward = 19.00, steps = 1683
    00:00:49 [INFO] test episode 62: reward = 20.00, steps = 2003
    00:01:12 [INFO] test episode 63: reward = 19.00, steps = 1687
    00:01:40 [INFO] test episode 64: reward = 20.00, steps = 1990
    00:02:08 [INFO] test episode 65: reward = 20.00, steps = 2005
    00:02:29 [INFO] test episode 66: reward = 20.00, steps = 1561
    00:02:53 [INFO] test episode 67: reward = 19.00, steps = 1683
    00:03:16 [INFO] test episode 68: reward = 19.00, steps = 1684
    00:03:39 [INFO] test episode 69: reward = 19.00, steps = 1686
    00:04:06 [INFO] test episode 70: reward = 20.00, steps = 2001
    00:04:30 [INFO] test episode 71: reward = 19.00, steps = 1683
    00:04:51 [INFO] test episode 72: reward = 20.00, steps = 1562
    00:05:19 [INFO] test episode 73: reward = 20.00, steps = 2004
    00:05:40 [INFO] test episode 74: reward = 20.00, steps = 1566
    00:06:09 [INFO] test episode 75: reward = 20.00, steps = 2001
    00:06:35 [INFO] test episode 76: reward = 20.00, steps = 1564
    00:07:00 [INFO] test episode 77: reward = 20.00, steps = 1564
    00:07:23 [INFO] test episode 78: reward = 20.00, steps = 1566
    00:07:44 [INFO] test episode 79: reward = 20.00, steps = 1565
    00:08:12 [INFO] test episode 80: reward = 20.00, steps = 1996
    00:08:42 [INFO] test episode 81: reward = 20.00, steps = 1992
    00:09:05 [INFO] test episode 82: reward = 19.00, steps = 1685
    00:09:35 [INFO] test episode 83: reward = 20.00, steps = 1993
    00:10:02 [INFO] test episode 84: reward = 20.00, steps = 1993
    00:10:26 [INFO] test episode 85: reward = 19.00, steps = 1685
    00:10:55 [INFO] test episode 86: reward = 20.00, steps = 2003
    00:11:25 [INFO] test episode 87: reward = 20.00, steps = 1996
    00:11:49 [INFO] test episode 88: reward = 19.00, steps = 1684
    00:12:16 [INFO] test episode 89: reward = 20.00, steps = 1990
    00:12:44 [INFO] test episode 90: reward = 20.00, steps = 1992
    00:13:12 [INFO] test episode 91: reward = 20.00, steps = 2003
    00:13:41 [INFO] test episode 92: reward = 20.00, steps = 1996
    00:14:06 [INFO] test episode 93: reward = 19.00, steps = 1690
    00:14:27 [INFO] test episode 94: reward = 20.00, steps = 1565
    00:14:56 [INFO] test episode 95: reward = 20.00, steps = 1992
    00:15:22 [INFO] test episode 96: reward = 19.00, steps = 1684
    00:15:50 [INFO] test episode 97: reward = 20.00, steps = 2005
    00:16:11 [INFO] test episode 98: reward = 20.00, steps = 1562
    00:16:34 [INFO] test episode 99: reward = 19.00, steps = 1686
    00:16:34 [INFO] average episode reward = 19.65 ± 0.48