# Use PPO to Play Acrobot-v1¶

PyTorch version

In [1]:

    %matplotlib inline

    import sys
    import logging
    import itertools

    import numpy as np
    np.random.seed(0)
    import pandas as pd
    import scipy.signal as signal
    import gym
    import matplotlib.pyplot as plt
    import torch
    import torch.nn as nn
    import torch.optim as optim
    import torch.distributions as distributions
    torch.manual_seed(0)

    logging.basicConfig(level=logging.INFO,
            format='%(asctime)s [%(levelname)s] %(message)s',
            stream=sys.stdout, datefmt='%H:%M:%S')

In [2]:

    env = gym.make('Acrobot-v1')
    for key in vars(env):
        logging.info('%s: %s', key, vars(env)[key])
    for key in vars(env.spec):
        logging.info('%s: %s', key, vars(env.spec)[key])

    22:56:35 [INFO] env: <AcrobotEnv<Acrobot-v1>>
    22:56:35 [INFO] action_space: Discrete(3)
    22:56:35 [INFO] observation_space: Box(-28.274333953857422, 28.274333953857422, (6,), float32)
    22:56:35 [INFO] reward_range: (-inf, inf)
    22:56:35 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 15}
    22:56:35 [INFO] _max_episode_steps: 500
    22:56:35 [INFO] _elapsed_steps: None
    22:56:35 [INFO] id: Acrobot-v1
    22:56:35 [INFO] entry_point: gym.envs.classic_control:AcrobotEnv
    22:56:35 [INFO] reward_threshold: -100.0
    22:56:35 [INFO] nondeterministic: False
    22:56:35 [INFO] max_episode_steps: 500
    22:56:35 [INFO] _kwargs: {}
    22:56:35 [INFO] _env_name: Acrobot

In [3]:

    class PPOReplayer:
        def __init__(self):
            self.fields = ['state', 'action', 'prob', 'advantage', 'return']
            self.memory = pd.DataFrame(columns=self.fields)

        def store(self, df):
            self.memory = pd.concat([self.memory, df[self.fields]], ignore_index=True)

        def sample(self, size):
            indices = np.random.choice(self.memory.shape[0], size=size)
            return (np.stack(self.memory.loc[indices, field]) for field in
                    self.fields)

In [4]:

    class PPOAgent:
        def __init__(self, env):
            self.gamma = 0.99

            self.replayer = PPOReplayer()

            self.actor_net = self.build_net(
                    input_size=env.observation_space.shape[0],
                    hidden_sizes=[100,],
                    output_size=env.action_space.n, output_activator=nn.Softmax(1))
            self.actor_optimizer = optim.Adam(self.actor_net.parameters(), 0.001)
            self.critic_net = self.build_net(
                    input_size=env.observation_space.shape[0],
                    hidden_sizes=[100,])
            self.critic_optimizer = optim.Adam(self.critic_net.parameters(), 0.002)
            self.critic_loss = nn.MSELoss()

        def build_net(self, input_size, hidden_sizes, output_size=1,
                output_activator=None):
            layers = []
            for input_size, output_size in zip(
                    [input_size,] + hidden_sizes, hidden_sizes + [output_size,]):
                layers.append(nn.Linear(input_size, output_size))
                layers.append(nn.ReLU())
            layers = layers[:-1]
            if output_activator:
                layers.append(output_activator)
            net = nn.Sequential(*layers)
            return net

        def reset(self, mode=None):
            self.mode = mode
            if self.mode == 'train':
                self.trajectory = []

        def step(self, observation, reward, terminated):
            state_tensor = torch.as_tensor(observation, dtype=torch.float).unsqueeze(0)
            prob_tensor = self.actor_net(state_tensor)
            action_tensor = distributions.Categorical(prob_tensor).sample()
            action = action_tensor.numpy()[0]
            if self.mode == 'train':
                self.trajectory += [observation, reward, terminated, action]
            return action

        def close(self):
            if self.mode == 'train':
                self.save_trajectory_to_replayer()
                if len(self.replayer.memory) >= 1000:
                    for batch in range(5):  # learn multiple times
                        self.learn()
                    self.replayer = PPOReplayer()
                            # reset replayer after the agent changes itself

        def save_trajectory_to_replayer(self):
            df = pd.DataFrame(
                    np.array(self.trajectory, dtype=object).reshape(-1, 4),
                    columns=['state', 'reward', 'terminated', 'action'])
            state_tensor = torch.as_tensor(np.stack(df['state']), dtype=torch.float)
            action_tensor = torch.as_tensor(df['action'], dtype=torch.long)
            v_tensor = self.critic_net(state_tensor)
            df['v'] = v_tensor.detach().numpy()
            prob_tensor = self.actor_net(state_tensor)
            pi_tensor = prob_tensor.gather(-1, action_tensor.unsqueeze(1)).squeeze(1)
            df['prob'] = pi_tensor.detach().numpy()
            df['next_v'] = df['v'].shift(-1).fillna(0.)
            df['u'] = df['reward'] + self.gamma * df['next_v']
            df['delta'] = df['u'] - df['v']
            df['advantage'] = signal.lfilter([1.,], [1., -self.gamma],
                    df['delta'][::-1])[::-1]
            df['return'] = signal.lfilter([1.,], [1., -self.gamma],
                    df['reward'][::-1])[::-1]
            self.replayer.store(df)

        def learn(self):
            states, actions, old_pis, advantages, returns = \
                    self.replayer.sample(size=64)
            state_tensor = torch.as_tensor(states, dtype=torch.float)
            action_tensor = torch.as_tensor(actions, dtype=torch.long)
            old_pi_tensor = torch.as_tensor(old_pis, dtype=torch.float)
            advantage_tensor = torch.as_tensor(advantages, dtype=torch.float)
            return_tensor = torch.as_tensor(returns, dtype=torch.float).unsqueeze(1)

            # update actor
            all_pi_tensor = self.actor_net(state_tensor)
            pi_tensor = all_pi_tensor.gather(1, action_tensor.unsqueeze(1)).squeeze(1)
            surrogate_advantage_tensor = (pi_tensor / old_pi_tensor) * \
                    advantage_tensor
            clip_times_advantage_tensor = 0.1 * surrogate_advantage_tensor
            max_surrogate_advantage_tensor = advantage_tensor + \
                    torch.where(advantage_tensor > 0.,
                    clip_times_advantage_tensor, -clip_times_advantage_tensor)
            clipped_surrogate_advantage_tensor = torch.min(
                    surrogate_advantage_tensor, max_surrogate_advantage_tensor)
            actor_loss_tensor = -clipped_surrogate_advantage_tensor.mean()
            self.actor_optimizer.zero_grad()
            actor_loss_tensor.backward()
            self.actor_optimizer.step()

            # update critic
            pred_tensor = self.critic_net(state_tensor)
            critic_loss_tensor = self.critic_loss(pred_tensor, return_tensor)
            self.critic_optimizer.zero_grad()
            critic_loss_tensor.backward()
            self.critic_optimizer.step()

    agent = PPOAgent(env)

In [5]:

    def play_episode(env, agent, seed=None, mode=None, render=False):
        observation, _ = env.reset(seed=seed)
        reward, terminated, truncated = 0., False, False
        agent.reset(mode=mode)
        episode_reward, elapsed_steps = 0., 0
        while True:
            action = agent.step(observation, reward, terminated)
            if render:
                env.render()
            if terminated or truncated:
                break
            observation, reward, terminated, truncated, _ = env.step(action)
            episode_reward += reward
            elapsed_steps += 1
        agent.close()
        return episode_reward, elapsed_steps

    logging.info('==== train ====')
    episode_rewards = []
    for episode in itertools.count():
        episode_reward, elapsed_steps = play_episode(env, agent, seed=episode,
                mode='train')
        episode_rewards.append(episode_reward)
        logging.info('train episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
        if np.mean(episode_rewards[-10:]) > -120:
            break
    plt.plot(episode_rewards)

    logging.info('==== test ====')
    episode_rewards = []
    for episode in range(100):
        episode_reward, elapsed_steps = play_episode(env, agent)
        episode_rewards.append(episode_reward)
        logging.info('test episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
    logging.info('average episode reward = %.2f ± %.2f',
            np.mean(episode_rewards), np.std(episode_rewards))

    22:56:35 [INFO] ==== train ====
    22:56:36 [INFO] NumExpr defaulting to 8 threads.
    22:56:36 [INFO] train episode 0: reward = -500.00, steps = 500
    22:56:37 [INFO] train episode 1: reward = -500.00, steps = 500
    22:56:38 [INFO] train episode 2: reward = -500.00, steps = 500
    22:56:40 [INFO] train episode 3: reward = -500.00, steps = 500
    22:56:41 [INFO] train episode 4: reward = -500.00, steps = 500
    22:56:42 [INFO] train episode 5: reward = -500.00, steps = 500
    22:56:43 [INFO] train episode 6: reward = -500.00, steps = 500
    22:56:45 [INFO] train episode 7: reward = -500.00, steps = 500
    22:56:46 [INFO] train episode 8: reward = -500.00, steps = 500
    22:56:47 [INFO] train episode 9: reward = -500.00, steps = 500
    22:56:48 [INFO] train episode 10: reward = -500.00, steps = 500
    22:56:50 [INFO] train episode 11: reward = -500.00, steps = 500
    22:56:51 [INFO] train episode 12: reward = -500.00, steps = 500
    22:56:52 [INFO] train episode 13: reward = -500.00, steps = 500
    22:56:53 [INFO] train episode 14: reward = -500.00, steps = 500
    22:56:55 [INFO] train episode 15: reward = -500.00, steps = 500
    22:56:56 [INFO] train episode 16: reward = -500.00, steps = 500
    22:56:57 [INFO] train episode 17: reward = -500.00, steps = 500
    22:56:58 [INFO] train episode 18: reward = -500.00, steps = 500
    22:57:00 [INFO] train episode 19: reward = -500.00, steps = 500
    22:57:01 [INFO] train episode 20: reward = -500.00, steps = 500
    22:57:02 [INFO] train episode 21: reward = -500.00, steps = 500
    22:57:03 [INFO] train episode 22: reward = -500.00, steps = 500
    22:57:05 [INFO] train episode 23: reward = -500.00, steps = 500
    22:57:06 [INFO] train episode 24: reward = -500.00, steps = 500
    22:57:07 [INFO] train episode 25: reward = -500.00, steps = 500
    22:57:08 [INFO] train episode 26: reward = -395.00, steps = 396
    22:57:09 [INFO] train episode 27: reward = -500.00, steps = 500
    22:57:10 [INFO] train episode 28: reward = -500.00, steps = 500
    22:57:11 [INFO] train episode 29: reward = -500.00, steps = 500
    22:57:13 [INFO] train episode 30: reward = -500.00, steps = 500
    22:57:14 [INFO] train episode 31: reward = -500.00, steps = 500
    22:57:15 [INFO] train episode 32: reward = -500.00, steps = 500
    22:57:16 [INFO] train episode 33: reward = -500.00, steps = 500
    22:57:17 [INFO] train episode 34: reward = -500.00, steps = 500
    22:57:18 [INFO] train episode 35: reward = -500.00, steps = 500
    22:57:20 [INFO] train episode 36: reward = -500.00, steps = 500
    22:57:21 [INFO] train episode 37: reward = -500.00, steps = 500
    22:57:22 [INFO] train episode 38: reward = -500.00, steps = 500
    22:57:23 [INFO] train episode 39: reward = -500.00, steps = 500
    22:57:24 [INFO] train episode 40: reward = -500.00, steps = 500
    22:57:25 [INFO] train episode 41: reward = -329.00, steps = 330
    22:57:26 [INFO] train episode 42: reward = -500.00, steps = 500
    22:57:28 [INFO] train episode 43: reward = -500.00, steps = 500
    22:57:29 [INFO] train episode 44: reward = -500.00, steps = 500
    22:57:30 [INFO] train episode 45: reward = -500.00, steps = 500
    22:57:31 [INFO] train episode 46: reward = -500.00, steps = 500
    22:57:32 [INFO] train episode 47: reward = -500.00, steps = 500
    22:57:34 [INFO] train episode 48: reward = -500.00, steps = 500
    22:57:35 [INFO] train episode 49: reward = -500.00, steps = 500
    22:57:36 [INFO] train episode 50: reward = -467.00, steps = 468
    22:57:37 [INFO] train episode 51: reward = -500.00, steps = 500
    22:57:38 [INFO] train episode 52: reward = -491.00, steps = 492
    22:57:39 [INFO] train episode 53: reward = -500.00, steps = 500
    22:57:40 [INFO] train episode 54: reward = -500.00, steps = 500
    22:57:41 [INFO] train episode 55: reward = -351.00, steps = 352
    22:57:42 [INFO] train episode 56: reward = -500.00, steps = 500
    22:57:43 [INFO] train episode 57: reward = -438.00, steps = 439
    22:57:43 [INFO] train episode 58: reward = -303.00, steps = 304
    22:57:44 [INFO] train episode 59: reward = -373.00, steps = 374
    22:57:45 [INFO] train episode 60: reward = -328.00, steps = 329
    22:57:45 [INFO] train episode 61: reward = -288.00, steps = 289
    22:57:46 [INFO] train episode 62: reward = -332.00, steps = 333
    22:57:47 [INFO] train episode 63: reward = -457.00, steps = 458
    22:57:48 [INFO] train episode 64: reward = -500.00, steps = 500
    22:57:49 [INFO] train episode 65: reward = -368.00, steps = 369
    22:57:50 [INFO] train episode 66: reward = -500.00, steps = 500
    22:57:50 [INFO] train episode 67: reward = -322.00, steps = 323
    22:57:51 [INFO] train episode 68: reward = -278.00, steps = 279
    22:57:51 [INFO] train episode 69: reward = -337.00, steps = 338
    22:57:52 [INFO] train episode 70: reward = -359.00, steps = 360
    22:57:53 [INFO] train episode 71: reward = -409.00, steps = 410
    22:57:53 [INFO] train episode 72: reward = -283.00, steps = 284
    22:57:54 [INFO] train episode 73: reward = -229.00, steps = 230
    22:57:55 [INFO] train episode 74: reward = -241.00, steps = 242
    22:57:56 [INFO] train episode 75: reward = -363.00, steps = 364
    22:57:56 [INFO] train episode 76: reward = -321.00, steps = 322
    22:57:57 [INFO] train episode 77: reward = -415.00, steps = 416
    22:57:58 [INFO] train episode 78: reward = -500.00, steps = 500
    22:57:59 [INFO] train episode 79: reward = -324.00, steps = 325
    22:58:00 [INFO] train episode 80: reward = -331.00, steps = 332
    22:58:00 [INFO] train episode 81: reward = -218.00, steps = 219
    22:58:01 [INFO] train episode 82: reward = -325.00, steps = 326
    22:58:02 [INFO] train episode 83: reward = -453.00, steps = 454
    22:58:02 [INFO] train episode 84: reward = -212.00, steps = 213
    22:58:02 [INFO] train episode 85: reward = -246.00, steps = 247
    22:58:03 [INFO] train episode 86: reward = -358.00, steps = 359
    22:58:03 [INFO] train episode 87: reward = -145.00, steps = 146
    22:58:04 [INFO] train episode 88: reward = -118.00, steps = 119
    22:58:04 [INFO] train episode 89: reward = -157.00, steps = 158
    22:58:05 [INFO] train episode 90: reward = -228.00, steps = 229
    22:58:05 [INFO] train episode 91: reward = -225.00, steps = 226
    22:58:05 [INFO] train episode 92: reward = -275.00, steps = 276
    22:58:06 [INFO] train episode 93: reward = -254.00, steps = 255
    22:58:06 [INFO] train episode 94: reward = -235.00, steps = 236
    22:58:07 [INFO] train episode 95: reward = -165.00, steps = 166
    22:58:07 [INFO] train episode 96: reward = -199.00, steps = 200
    22:58:07 [INFO] train episode 97: reward = -267.00, steps = 268
    22:58:08 [INFO] train episode 98: reward = -226.00, steps = 227
    22:58:08 [INFO] train episode 99: reward = -168.00, steps = 169
    22:58:09 [INFO] train episode 100: reward = -329.00, steps = 330
    22:58:09 [INFO] train episode 101: reward = -181.00, steps = 182
    22:58:09 [INFO] train episode 102: reward = -194.00, steps = 195
    22:58:10 [INFO] train episode 103: reward = -172.00, steps = 173
    22:58:10 [INFO] train episode 104: reward = -223.00, steps = 224
    22:58:11 [INFO] train episode 105: reward = -252.00, steps = 253
    22:58:11 [INFO] train episode 106: reward = -205.00, steps = 206
    22:58:11 [INFO] train episode 107: reward = -182.00, steps = 183
    22:58:12 [INFO] train episode 108: reward = -180.00, steps = 181
    22:58:12 [INFO] train episode 109: reward = -219.00, steps = 220
    22:58:13 [INFO] train episode 110: reward = -195.00, steps = 196
    22:58:13 [INFO] train episode 111: reward = -213.00, steps = 214
    22:58:13 [INFO] train episode 112: reward = -190.00, steps = 191
    22:58:14 [INFO] train episode 113: reward = -212.00, steps = 213
    22:58:14 [INFO] train episode 114: reward = -212.00, steps = 213
    22:58:14 [INFO] train episode 115: reward = -221.00, steps = 222
    22:58:14 [INFO] train episode 116: reward = -175.00, steps = 176
    22:58:15 [INFO] train episode 117: reward = -241.00, steps = 242
    22:58:15 [INFO] train episode 118: reward = -163.00, steps = 164
    22:58:15 [INFO] train episode 119: reward = -219.00, steps = 220
    22:58:16 [INFO] train episode 120: reward = -222.00, steps = 223
    22:58:16 [INFO] train episode 121: reward = -253.00, steps = 254
    22:58:17 [INFO] train episode 122: reward = -223.00, steps = 224
    22:58:17 [INFO] train episode 123: reward = -210.00, steps = 211
    22:58:17 [INFO] train episode 124: reward = -224.00, steps = 225
    22:58:18 [INFO] train episode 125: reward = -165.00, steps = 166
    22:58:18 [INFO] train episode 126: reward = -268.00, steps = 269
    22:58:18 [INFO] train episode 127: reward = -272.00, steps = 273
    22:58:19 [INFO] train episode 128: reward = -207.00, steps = 208
    22:58:19 [INFO] train episode 129: reward = -218.00, steps = 219
    22:58:19 [INFO] train episode 130: reward = -188.00, steps = 189
    22:58:20 [INFO] train episode 131: reward = -187.00, steps = 188
    22:58:20 [INFO] train episode 132: reward = -301.00, steps = 302
    22:58:21 [INFO] train episode 133: reward = -236.00, steps = 237
    22:58:21 [INFO] train episode 134: reward = -368.00, steps = 369
    22:58:21 [INFO] train episode 135: reward = -199.00, steps = 200
    22:58:22 [INFO] train episode 136: reward = -394.00, steps = 395
    22:58:22 [INFO] train episode 137: reward = -211.00, steps = 212
    22:58:23 [INFO] train episode 138: reward = -182.00, steps = 183
    22:58:23 [INFO] train episode 139: reward = -257.00, steps = 258
    22:58:23 [INFO] train episode 140: reward = -163.00, steps = 164
    22:58:23 [INFO] train episode 141: reward = -206.00, steps = 207
    22:58:24 [INFO] train episode 142: reward = -294.00, steps = 295
    22:58:24 [INFO] train episode 143: reward = -276.00, steps = 277
    22:58:24 [INFO] train episode 144: reward = -163.00, steps = 164
    22:58:25 [INFO] train episode 145: reward = -194.00, steps = 195
    22:58:25 [INFO] train episode 146: reward = -231.00, steps = 232
    22:58:25 [INFO] train episode 147: reward = -281.00, steps = 282
    22:58:26 [INFO] train episode 148: reward = -135.00, steps = 136
    22:58:26 [INFO] train episode 149: reward = -175.00, steps = 176
    22:58:26 [INFO] train episode 150: reward = -264.00, steps = 265
    22:58:27 [INFO] train episode 151: reward = -203.00, steps = 204
    22:58:27 [INFO] train episode 152: reward = -186.00, steps = 187
    22:58:27 [INFO] train episode 153: reward = -349.00, steps = 350
    22:58:28 [INFO] train episode 154: reward = -307.00, steps = 308
    22:58:28 [INFO] train episode 155: reward = -261.00, steps = 262
    22:58:29 [INFO] train episode 156: reward = -278.00, steps = 279
    22:58:29 [INFO] train episode 157: reward = -293.00, steps = 294
    22:58:30 [INFO] train episode 158: reward = -251.00, steps = 252
    22:58:30 [INFO] train episode 159: reward = -236.00, steps = 237
    22:58:30 [INFO] train episode 160: reward = -281.00, steps = 282
    22:58:31 [INFO] train episode 161: reward = -248.00, steps = 249
    22:58:31 [INFO] train episode 162: reward = -206.00, steps = 207
    22:58:31 [INFO] train episode 163: reward = -272.00, steps = 273
    22:58:32 [INFO] train episode 164: reward = -274.00, steps = 275
    22:58:32 [INFO] train episode 165: reward = -180.00, steps = 181
    22:58:33 [INFO] train episode 166: reward = -500.00, steps = 500
    22:58:33 [INFO] train episode 167: reward = -282.00, steps = 283
    22:58:34 [INFO] train episode 168: reward = -243.00, steps = 244
    22:58:34 [INFO] train episode 169: reward = -175.00, steps = 176
    22:58:34 [INFO] train episode 170: reward = -279.00, steps = 280
    22:58:35 [INFO] train episode 171: reward = -198.00, steps = 199
    22:58:35 [INFO] train episode 172: reward = -195.00, steps = 196
    22:58:35 [INFO] train episode 173: reward = -218.00, steps = 219
    22:58:36 [INFO] train episode 174: reward = -328.00, steps = 329
    22:58:36 [INFO] train episode 175: reward = -179.00, steps = 180
    22:58:36 [INFO] train episode 176: reward = -197.00, steps = 198
    22:58:37 [INFO] train episode 177: reward = -299.00, steps = 300
    22:58:37 [INFO] train episode 178: reward = -279.00, steps = 280
    22:58:38 [INFO] train episode 179: reward = -355.00, steps = 356
    22:58:39 [INFO] train episode 180: reward = -325.00, steps = 326
    22:58:39 [INFO] train episode 181: reward = -186.00, steps = 187
    22:58:39 [INFO] train episode 182: reward = -250.00, steps = 251
    22:58:40 [INFO] train episode 183: reward = -286.00, steps = 287
    22:58:40 [INFO] train episode 184: reward = -177.00, steps = 178
    22:58:41 [INFO] train episode 185: reward = -217.00, steps = 218
    22:58:41 [INFO] train episode 186: reward = -223.00, steps = 224
    22:58:41 [INFO] train episode 187: reward = -209.00, steps = 210
    22:58:42 [INFO] train episode 188: reward = -210.00, steps = 211
    22:58:42 [INFO] train episode 189: reward = -237.00, steps = 238
    22:58:42 [INFO] train episode 190: reward = -182.00, steps = 183
    22:58:43 [INFO] train episode 191: reward = -165.00, steps = 166
    22:58:43 [INFO] train episode 192: reward = -204.00, steps = 205
    22:58:43 [INFO] train episode 193: reward = -211.00, steps = 212
    22:58:44 [INFO] train episode 194: reward = -174.00, steps = 175
    22:58:44 [INFO] train episode 195: reward = -183.00, steps = 184
    22:58:44 [INFO] train episode 196: reward = -272.00, steps = 273
    22:58:45 [INFO] train episode 197: reward = -241.00, steps = 242
    22:58:45 [INFO] train episode 198: reward = -257.00, steps = 258
    22:58:45 [INFO] train episode 199: reward = -220.00, steps = 221
    22:58:46 [INFO] train episode 200: reward = -226.00, steps = 227
    22:58:46 [INFO] train episode 201: reward = -235.00, steps = 236
    22:58:47 [INFO] train episode 202: reward = -257.00, steps = 258
    22:58:47 [INFO] train episode 203: reward = -340.00, steps = 341
    22:58:47 [INFO] train episode 204: reward = -202.00, steps = 203
    22:58:48 [INFO] train episode 205: reward = -451.00, steps = 452
    22:58:48 [INFO] train episode 206: reward = -231.00, steps = 232
    22:58:49 [INFO] train episode 207: reward = -192.00, steps = 193
    22:58:49 [INFO] train episode 208: reward = -245.00, steps = 246
    22:58:49 [INFO] train episode 209: reward = -192.00, steps = 193
    22:58:50 [INFO] train episode 210: reward = -231.00, steps = 232
    22:58:50 [INFO] train episode 211: reward = -303.00, steps = 304
    22:58:50 [INFO] train episode 212: reward = -223.00, steps = 224
    22:58:51 [INFO] train episode 213: reward = -190.00, steps = 191
    22:58:51 [INFO] train episode 214: reward = -188.00, steps = 189
    22:58:51 [INFO] train episode 215: reward = -152.00, steps = 153
    22:58:51 [INFO] train episode 216: reward = -157.00, steps = 158
    22:58:52 [INFO] train episode 217: reward = -350.00, steps = 351
    22:58:52 [INFO] train episode 218: reward = -124.00, steps = 125
    22:58:53 [INFO] train episode 219: reward = -349.00, steps = 350
    22:58:53 [INFO] train episode 220: reward = -171.00, steps = 172
    22:58:53 [INFO] train episode 221: reward = -190.00, steps = 191
    22:58:54 [INFO] train episode 222: reward = -325.00, steps = 326
    22:58:54 [INFO] train episode 223: reward = -348.00, steps = 349
    22:58:54 [INFO] train episode 224: reward = -235.00, steps = 236
    22:58:55 [INFO] train episode 225: reward = -180.00, steps = 181
    22:58:55 [INFO] train episode 226: reward = -203.00, steps = 204
    22:58:55 [INFO] train episode 227: reward = -266.00, steps = 267
    22:58:56 [INFO] train episode 228: reward = -217.00, steps = 218
    22:58:56 [INFO] train episode 229: reward = -175.00, steps = 176
    22:58:56 [INFO] train episode 230: reward = -160.00, steps = 161
    22:58:56 [INFO] train episode 231: reward = -134.00, steps = 135
    22:58:57 [INFO] train episode 232: reward = -152.00, steps = 153
    22:58:57 [INFO] train episode 233: reward = -210.00, steps = 211
    22:58:57 [INFO] train episode 234: reward = -216.00, steps = 217
    22:58:57 [INFO] train episode 235: reward = -199.00, steps = 200
    22:58:58 [INFO] train episode 236: reward = -122.00, steps = 123
    22:58:58 [INFO] train episode 237: reward = -132.00, steps = 133
    22:58:58 [INFO] train episode 238: reward = -310.00, steps = 311
    22:58:58 [INFO] train episode 239: reward = -262.00, steps = 263
    22:58:59 [INFO] train episode 240: reward = -196.00, steps = 197
    22:58:59 [INFO] train episode 241: reward = -215.00, steps = 216
    22:58:59 [INFO] train episode 242: reward = -160.00, steps = 161
    22:59:00 [INFO] train episode 243: reward = -174.00, steps = 175
    22:59:00 [INFO] train episode 244: reward = -282.00, steps = 283
    22:59:00 [INFO] train episode 245: reward = -287.00, steps = 288
    22:59:01 [INFO] train episode 246: reward = -222.00, steps = 223
    22:59:01 [INFO] train episode 247: reward = -200.00, steps = 201
    22:59:01 [INFO] train episode 248: reward = -240.00, steps = 241
    22:59:01 [INFO] train episode 249: reward = -259.00, steps = 260
    22:59:02 [INFO] train episode 250: reward = -251.00, steps = 252
    22:59:02 [INFO] train episode 251: reward = -178.00, steps = 179
    22:59:02 [INFO] train episode 252: reward = -176.00, steps = 177
    22:59:02 [INFO] train episode 253: reward = -224.00, steps = 225
    22:59:03 [INFO] train episode 254: reward = -136.00, steps = 137
    22:59:03 [INFO] train episode 255: reward = -159.00, steps = 160
    22:59:03 [INFO] train episode 256: reward = -166.00, steps = 167
    22:59:03 [INFO] train episode 257: reward = -132.00, steps = 133
    22:59:04 [INFO] train episode 258: reward = -251.00, steps = 252
    22:59:04 [INFO] train episode 259: reward = -254.00, steps = 255
    22:59:04 [INFO] train episode 260: reward = -151.00, steps = 152
    22:59:04 [INFO] train episode 261: reward = -145.00, steps = 146
    22:59:05 [INFO] train episode 262: reward = -214.00, steps = 215
    22:59:05 [INFO] train episode 263: reward = -206.00, steps = 207
    22:59:05 [INFO] train episode 264: reward = -174.00, steps = 175
    22:59:05 [INFO] train episode 265: reward = -166.00, steps = 167
    22:59:06 [INFO] train episode 266: reward = -273.00, steps = 274
    22:59:06 [INFO] train episode 267: reward = -244.00, steps = 245
    22:59:06 [INFO] train episode 268: reward = -285.00, steps = 286
    22:59:06 [INFO] train episode 269: reward = -214.00, steps = 215
    22:59:07 [INFO] train episode 270: reward = -168.00, steps = 169
    22:59:07 [INFO] train episode 271: reward = -196.00, steps = 197
    22:59:07 [INFO] train episode 272: reward = -213.00, steps = 214
    22:59:07 [INFO] train episode 273: reward = -191.00, steps = 192
    22:59:08 [INFO] train episode 274: reward = -193.00, steps = 194
    22:59:08 [INFO] train episode 275: reward = -230.00, steps = 231
    22:59:08 [INFO] train episode 276: reward = -260.00, steps = 261
    22:59:09 [INFO] train episode 277: reward = -276.00, steps = 277
    22:59:09 [INFO] train episode 278: reward = -189.00, steps = 190
    22:59:09 [INFO] train episode 279: reward = -246.00, steps = 247
    22:59:10 [INFO] train episode 280: reward = -226.00, steps = 227
    22:59:10 [INFO] train episode 281: reward = -215.00, steps = 216
    22:59:10 [INFO] train episode 282: reward = -241.00, steps = 242
    22:59:10 [INFO] train episode 283: reward = -189.00, steps = 190
    22:59:11 [INFO] train episode 284: reward = -205.00, steps = 206
    22:59:11 [INFO] train episode 285: reward = -248.00, steps = 249
    22:59:11 [INFO] train episode 286: reward = -321.00, steps = 322
    22:59:12 [INFO] train episode 287: reward = -222.00, steps = 223
    22:59:12 [INFO] train episode 288: reward = -208.00, steps = 209
    22:59:12 [INFO] train episode 289: reward = -254.00, steps = 255
    22:59:12 [INFO] train episode 290: reward = -188.00, steps = 189
    22:59:12 [INFO] train episode 291: reward = -189.00, steps = 190
    22:59:13 [INFO] train episode 292: reward = -220.00, steps = 221
    22:59:13 [INFO] train episode 293: reward = -241.00, steps = 242
    22:59:13 [INFO] train episode 294: reward = -231.00, steps = 232
    22:59:14 [INFO] train episode 295: reward = -220.00, steps = 221
    22:59:14 [INFO] train episode 296: reward = -189.00, steps = 190
    22:59:14 [INFO] train episode 297: reward = -147.00, steps = 148
    22:59:14 [INFO] train episode 298: reward = -244.00, steps = 245
    22:59:15 [INFO] train episode 299: reward = -144.00, steps = 145
    22:59:15 [INFO] train episode 300: reward = -210.00, steps = 211
    22:59:15 [INFO] train episode 301: reward = -173.00, steps = 174
    22:59:15 [INFO] train episode 302: reward = -208.00, steps = 209
    22:59:16 [INFO] train episode 303: reward = -141.00, steps = 142
    22:59:16 [INFO] train episode 304: reward = -250.00, steps = 251
    22:59:16 [INFO] train episode 305: reward = -204.00, steps = 205
    22:59:16 [INFO] train episode 306: reward = -175.00, steps = 176
    22:59:17 [INFO] train episode 307: reward = -199.00, steps = 200
    22:59:17 [INFO] train episode 308: reward = -191.00, steps = 192
    22:59:17 [INFO] train episode 309: reward = -249.00, steps = 250
    22:59:17 [INFO] train episode 310: reward = -168.00, steps = 169
    22:59:18 [INFO] train episode 311: reward = -180.00, steps = 181
    22:59:18 [INFO] train episode 312: reward = -173.00, steps = 174
    22:59:18 [INFO] train episode 313: reward = -260.00, steps = 261
    22:59:19 [INFO] train episode 314: reward = -201.00, steps = 202
    22:59:19 [INFO] train episode 315: reward = -161.00, steps = 162
    22:59:19 [INFO] train episode 316: reward = -161.00, steps = 162
    22:59:19 [INFO] train episode 317: reward = -200.00, steps = 201
    22:59:20 [INFO] train episode 318: reward = -171.00, steps = 172
    22:59:20 [INFO] train episode 319: reward = -191.00, steps = 192
    22:59:20 [INFO] train episode 320: reward = -264.00, steps = 265
    22:59:20 [INFO] train episode 321: reward = -175.00, steps = 176
    22:59:21 [INFO] train episode 322: reward = -152.00, steps = 153
    22:59:21 [INFO] train episode 323: reward = -168.00, steps = 169
    22:59:21 [INFO] train episode 324: reward = -235.00, steps = 236
    22:59:21 [INFO] train episode 325: reward = -222.00, steps = 223
    22:59:22 [INFO] train episode 326: reward = -191.00, steps = 192
    22:59:22 [INFO] train episode 327: reward = -185.00, steps = 186
    22:59:22 [INFO] train episode 328: reward = -205.00, steps = 206
    22:59:22 [INFO] train episode 329: reward = -175.00, steps = 176
    22:59:23 [INFO] train episode 330: reward = -190.00, steps = 191
    22:59:23 [INFO] train episode 331: reward = -199.00, steps = 200
    22:59:23 [INFO] train episode 332: reward = -155.00, steps = 156
    22:59:23 [INFO] train episode 333: reward = -218.00, steps = 219
    22:59:24 [INFO] train episode 334: reward = -170.00, steps = 171
    22:59:24 [INFO] train episode 335: reward = -223.00, steps = 224
    22:59:24 [INFO] train episode 336: reward = -184.00, steps = 185
    22:59:25 [INFO] train episode 337: reward = -363.00, steps = 364
    22:59:25 [INFO] train episode 338: reward = -218.00, steps = 219
    22:59:25 [INFO] train episode 339: reward = -162.00, steps = 163
    22:59:25 [INFO] train episode 340: reward = -167.00, steps = 168
    22:59:26 [INFO] train episode 341: reward = -200.00, steps = 201
    22:59:26 [INFO] train episode 342: reward = -295.00, steps = 296
    22:59:26 [INFO] train episode 343: reward = -128.00, steps = 129
    22:59:26 [INFO] train episode 344: reward = -193.00, steps = 194
    22:59:27 [INFO] train episode 345: reward = -221.00, steps = 222
    22:59:27 [INFO] train episode 346: reward = -162.00, steps = 163
    22:59:27 [INFO] train episode 347: reward = -207.00, steps = 208
    22:59:27 [INFO] train episode 348: reward = -164.00, steps = 165
    22:59:28 [INFO] train episode 349: reward = -163.00, steps = 164
    22:59:28 [INFO] train episode 350: reward = -197.00, steps = 198
    22:59:28 [INFO] train episode 351: reward = -162.00, steps = 163
    22:59:28 [INFO] train episode 352: reward = -208.00, steps = 209
    22:59:29 [INFO] train episode 353: reward = -183.00, steps = 184
    22:59:29 [INFO] train episode 354: reward = -179.00, steps = 180
    22:59:29 [INFO] train episode 355: reward = -285.00, steps = 286
    22:59:29 [INFO] train episode 356: reward = -158.00, steps = 159
    22:59:30 [INFO] train episode 357: reward = -194.00, steps = 195
    22:59:30 [INFO] train episode 358: reward = -171.00, steps = 172
    22:59:30 [INFO] train episode 359: reward = -177.00, steps = 178
    22:59:30 [INFO] train episode 360: reward = -194.00, steps = 195
    22:59:31 [INFO] train episode 361: reward = -226.00, steps = 227
    22:59:31 [INFO] train episode 362: reward = -185.00, steps = 186
    22:59:31 [INFO] train episode 363: reward = -191.00, steps = 192
    22:59:31 [INFO] train episode 364: reward = -165.00, steps = 166
    22:59:32 [INFO] train episode 365: reward = -162.00, steps = 163
    22:59:32 [INFO] train episode 366: reward = -190.00, steps = 191
    22:59:32 [INFO] train episode 367: reward = -161.00, steps = 162
    22:59:32 [INFO] train episode 368: reward = -209.00, steps = 210
    22:59:33 [INFO] train episode 369: reward = -422.00, steps = 423
    22:59:33 [INFO] train episode 370: reward = -149.00, steps = 150
    22:59:33 [INFO] train episode 371: reward = -195.00, steps = 196
    22:59:33 [INFO] train episode 372: reward = -192.00, steps = 193
    22:59:34 [INFO] train episode 373: reward = -153.00, steps = 154
    22:59:34 [INFO] train episode 374: reward = -181.00, steps = 182
    22:59:34 [INFO] train episode 375: reward = -155.00, steps = 156
    22:59:34 [INFO] train episode 376: reward = -229.00, steps = 230
    22:59:35 [INFO] train episode 377: reward = -178.00, steps = 179
    22:59:35 [INFO] train episode 378: reward = -188.00, steps = 189
    22:59:35 [INFO] train episode 379: reward = -150.00, steps = 151
    22:59:35 [INFO] train episode 380: reward = -282.00, steps = 283
    22:59:36 [INFO] train episode 381: reward = -160.00, steps = 161
    22:59:36 [INFO] train episode 382: reward = -229.00, steps = 230
    22:59:36 [INFO] train episode 383: reward = -177.00, steps = 178
    22:59:36 [INFO] train episode 384: reward = -246.00, steps = 247
    22:59:37 [INFO] train episode 385: reward = -215.00, steps = 216
    22:59:37 [INFO] train episode 386: reward = -170.00, steps = 171
    22:59:37 [INFO] train episode 387: reward = -195.00, steps = 196
    22:59:37 [INFO] train episode 388: reward = -192.00, steps = 193
    22:59:38 [INFO] train episode 389: reward = -168.00, steps = 169
    22:59:38 [INFO] train episode 390: reward = -166.00, steps = 167
    22:59:38 [INFO] train episode 391: reward = -187.00, steps = 188
    22:59:38 [INFO] train episode 392: reward = -178.00, steps = 179
    22:59:39 [INFO] train episode 393: reward = -183.00, steps = 184
    22:59:39 [INFO] train episode 394: reward = -186.00, steps = 187
    22:59:39 [INFO] train episode 395: reward = -221.00, steps = 222
    22:59:39 [INFO] train episode 396: reward = -159.00, steps = 160
    22:59:40 [INFO] train episode 397: reward = -133.00, steps = 134
    22:59:40 [INFO] train episode 398: reward = -257.00, steps = 258
    22:59:40 [INFO] train episode 399: reward = -266.00, steps = 267
    22:59:40 [INFO] train episode 400: reward = -195.00, steps = 196
    22:59:41 [INFO] train episode 401: reward = -173.00, steps = 174
    22:59:41 [INFO] train episode 402: reward = -144.00, steps = 145
    22:59:41 [INFO] train episode 403: reward = -174.00, steps = 175
    22:59:41 [INFO] train episode 404: reward = -192.00, steps = 193
    22:59:42 [INFO] train episode 405: reward = -178.00, steps = 179
    22:59:42 [INFO] train episode 406: reward = -228.00, steps = 229
    22:59:42 [INFO] train episode 407: reward = -207.00, steps = 208
    22:59:42 [INFO] train episode 408: reward = -156.00, steps = 157
    22:59:43 [INFO] train episode 409: reward = -216.00, steps = 217
    22:59:43 [INFO] train episode 410: reward = -135.00, steps = 136
    22:59:43 [INFO] train episode 411: reward = -194.00, steps = 195
    22:59:43 [INFO] train episode 412: reward = -186.00, steps = 187
    22:59:44 [INFO] train episode 413: reward = -253.00, steps = 254
    22:59:44 [INFO] train episode 414: reward = -268.00, steps = 269
    22:59:44 [INFO] train episode 415: reward = -173.00, steps = 174
    22:59:45 [INFO] train episode 416: reward = -351.00, steps = 352
    22:59:45 [INFO] train episode 417: reward = -204.00, steps = 205
    22:59:45 [INFO] train episode 418: reward = -180.00, steps = 181
    22:59:46 [INFO] train episode 419: reward = -198.00, steps = 199
    22:59:46 [INFO] train episode 420: reward = -206.00, steps = 207
    22:59:46 [INFO] train episode 421: reward = -146.00, steps = 147
    22:59:46 [INFO] train episode 422: reward = -191.00, steps = 192
    22:59:47 [INFO] train episode 423: reward = -222.00, steps = 223
    22:59:47 [INFO] train episode 424: reward = -197.00, steps = 198
    22:59:47 [INFO] train episode 425: reward = -138.00, steps = 139
    22:59:47 [INFO] train episode 426: reward = -209.00, steps = 210
    22:59:48 [INFO] train episode 427: reward = -189.00, steps = 190
    22:59:48 [INFO] train episode 428: reward = -181.00, steps = 182
    22:59:48 [INFO] train episode 429: reward = -165.00, steps = 166
    22:59:48 [INFO] train episode 430: reward = -186.00, steps = 187
    22:59:49 [INFO] train episode 431: reward = -171.00, steps = 172
    22:59:49 [INFO] train episode 432: reward = -175.00, steps = 176
    22:59:49 [INFO] train episode 433: reward = -216.00, steps = 217
    22:59:49 [INFO] train episode 434: reward = -143.00, steps = 144
    22:59:50 [INFO] train episode 435: reward = -151.00, steps = 152
    22:59:50 [INFO] train episode 436: reward = -132.00, steps = 133
    22:59:50 [INFO] train episode 437: reward = -151.00, steps = 152
    22:59:50 [INFO] train episode 438: reward = -232.00, steps = 233
    22:59:50 [INFO] train episode 439: reward = -166.00, steps = 167
    22:59:51 [INFO] train episode 440: reward = -179.00, steps = 180
    22:59:51 [INFO] train episode 441: reward = -166.00, steps = 167
    22:59:51 [INFO] train episode 442: reward = -193.00, steps = 194
    22:59:51 [INFO] train episode 443: reward = -152.00, steps = 153
    22:59:52 [INFO] train episode 444: reward = -291.00, steps = 292
    22:59:52 [INFO] train episode 445: reward = -144.00, steps = 145
    22:59:52 [INFO] train episode 446: reward = -261.00, steps = 262
    22:59:53 [INFO] train episode 447: reward = -178.00, steps = 179
    22:59:53 [INFO] train episode 448: reward = -242.00, steps = 243
    22:59:53 [INFO] train episode 449: reward = -155.00, steps = 156
    22:59:53 [INFO] train episode 450: reward = -188.00, steps = 189
    22:59:54 [INFO] train episode 451: reward = -159.00, steps = 160
    22:59:54 [INFO] train episode 452: reward = -311.00, steps = 312
    22:59:54 [INFO] train episode 453: reward = -188.00, steps = 189
    22:59:54 [INFO] train episode 454: reward = -270.00, steps = 271
    22:59:55 [INFO] train episode 455: reward = -224.00, steps = 225
    22:59:55 [INFO] train episode 456: reward = -122.00, steps = 123
    22:59:55 [INFO] train episode 457: reward = -210.00, steps = 211
    22:59:55 [INFO] train episode 458: reward = -156.00, steps = 157
    22:59:55 [INFO] train episode 459: reward = -147.00, steps = 148
    22:59:55 [INFO] train episode 460: reward = -143.00, steps = 144
    22:59:56 [INFO] train episode 461: reward = -256.00, steps = 257
    22:59:56 [INFO] train episode 462: reward = -158.00, steps = 159
    22:59:56 [INFO] train episode 463: reward = -196.00, steps = 197
    22:59:56 [INFO] train episode 464: reward = -175.00, steps = 176
    22:59:57 [INFO] train episode 465: reward = -196.00, steps = 197
    22:59:57 [INFO] train episode 466: reward = -159.00, steps = 160
    22:59:57 [INFO] train episode 467: reward = -143.00, steps = 144
    22:59:57 [INFO] train episode 468: reward = -192.00, steps = 193
    22:59:57 [INFO] train episode 469: reward = -204.00, steps = 205
    22:59:58 [INFO] train episode 470: reward = -229.00, steps = 230
    22:59:58 [INFO] train episode 471: reward = -153.00, steps = 154
    22:59:58 [INFO] train episode 472: reward = -304.00, steps = 305
    22:59:58 [INFO] train episode 473: reward = -149.00, steps = 150
    22:59:59 [INFO] train episode 474: reward = -162.00, steps = 163
    22:59:59 [INFO] train episode 475: reward = -170.00, steps = 171
    22:59:59 [INFO] train episode 476: reward = -117.00, steps = 118
    22:59:59 [INFO] train episode 477: reward = -153.00, steps = 154
    22:59:59 [INFO] train episode 478: reward = -190.00, steps = 191
    23:00:00 [INFO] train episode 479: reward = -223.00, steps = 224
    23:00:00 [INFO] train episode 480: reward = -183.00, steps = 184
    23:00:00 [INFO] train episode 481: reward = -149.00, steps = 150
    23:00:00 [INFO] train episode 482: reward = -208.00, steps = 209
    23:00:00 [INFO] train episode 483: reward = -176.00, steps = 177
    23:00:01 [INFO] train episode 484: reward = -179.00, steps = 180
    23:00:01 [INFO] train episode 485: reward = -153.00, steps = 154
    23:00:01 [INFO] train episode 486: reward = -200.00, steps = 201
    23:00:01 [INFO] train episode 487: reward = -179.00, steps = 180
    23:00:01 [INFO] train episode 488: reward = -191.00, steps = 192
    23:00:02 [INFO] train episode 489: reward = -167.00, steps = 168
    23:00:02 [INFO] train episode 490: reward = -184.00, steps = 185
    23:00:02 [INFO] train episode 491: reward = -173.00, steps = 174
    23:00:02 [INFO] train episode 492: reward = -195.00, steps = 196
    23:00:03 [INFO] train episode 493: reward = -268.00, steps = 269
    23:00:03 [INFO] train episode 494: reward = -160.00, steps = 161
    23:00:03 [INFO] train episode 495: reward = -181.00, steps = 182
    23:00:03 [INFO] train episode 496: reward = -155.00, steps = 156
    23:00:03 [INFO] train episode 497: reward = -143.00, steps = 144
    23:00:03 [INFO] train episode 498: reward = -166.00, steps = 167
    23:00:04 [INFO] train episode 499: reward = -207.00, steps = 208
    23:00:04 [INFO] train episode 500: reward = -193.00, steps = 194
    23:00:04 [INFO] train episode 501: reward = -194.00, steps = 195
    23:00:04 [INFO] train episode 502: reward = -138.00, steps = 139
    23:00:05 [INFO] train episode 503: reward = -249.00, steps = 250
    23:00:05 [INFO] train episode 504: reward = -289.00, steps = 290
    23:00:05 [INFO] train episode 505: reward = -241.00, steps = 242
    23:00:05 [INFO] train episode 506: reward = -132.00, steps = 133
    23:00:05 [INFO] train episode 507: reward = -150.00, steps = 151
    23:00:06 [INFO] train episode 508: reward = -176.00, steps = 177
    23:00:06 [INFO] train episode 509: reward = -225.00, steps = 226
    23:00:06 [INFO] train episode 510: reward = -261.00, steps = 262
    23:00:06 [INFO] train episode 511: reward = -155.00, steps = 156
    23:00:07 [INFO] train episode 512: reward = -159.00, steps = 160
    23:00:07 [INFO] train episode 513: reward = -172.00, steps = 173
    23:00:07 [INFO] train episode 514: reward = -165.00, steps = 166
    23:00:07 [INFO] train episode 515: reward = -144.00, steps = 145
    23:00:07 [INFO] train episode 516: reward = -156.00, steps = 157
    23:00:07 [INFO] train episode 517: reward = -184.00, steps = 185
    23:00:08 [INFO] train episode 518: reward = -172.00, steps = 173
    23:00:08 [INFO] train episode 519: reward = -296.00, steps = 297
    23:00:08 [INFO] train episode 520: reward = -110.00, steps = 111
    23:00:08 [INFO] train episode 521: reward = -159.00, steps = 160
    23:00:08 [INFO] train episode 522: reward = -190.00, steps = 191
    23:00:09 [INFO] train episode 523: reward = -262.00, steps = 263
    23:00:09 [INFO] train episode 524: reward = -158.00, steps = 159
    23:00:09 [INFO] train episode 525: reward = -151.00, steps = 152
    23:00:09 [INFO] train episode 526: reward = -162.00, steps = 163
    23:00:09 [INFO] train episode 527: reward = -176.00, steps = 177
    23:00:10 [INFO] train episode 528: reward = -190.00, steps = 191
    23:00:10 [INFO] train episode 529: reward = -175.00, steps = 176
    23:00:10 [INFO] train episode 530: reward = -206.00, steps = 207
    23:00:10 [INFO] train episode 531: reward = -244.00, steps = 245
    23:00:10 [INFO] train episode 532: reward = -136.00, steps = 137
    23:00:11 [INFO] train episode 533: reward = -141.00, steps = 142
    23:00:11 [INFO] train episode 534: reward = -133.00, steps = 134
    23:00:11 [INFO] train episode 535: reward = -206.00, steps = 207
    23:00:11 [INFO] train episode 536: reward = -169.00, steps = 170
    23:00:11 [INFO] train episode 537: reward = -178.00, steps = 179
    23:00:11 [INFO] train episode 538: reward = -161.00, steps = 162
    23:00:12 [INFO] train episode 539: reward = -139.00, steps = 140
    23:00:12 [INFO] train episode 540: reward = -192.00, steps = 193
    23:00:12 [INFO] train episode 541: reward = -200.00, steps = 201
    23:00:12 [INFO] train episode 542: reward = -115.00, steps = 116
    23:00:12 [INFO] train episode 543: reward = -163.00, steps = 164
    23:00:13 [INFO] train episode 544: reward = -277.00, steps = 278
    23:00:13 [INFO] train episode 545: reward = -190.00, steps = 191
    23:00:13 [INFO] train episode 546: reward = -179.00, steps = 180
    23:00:13 [INFO] train episode 547: reward = -187.00, steps = 188
    23:00:14 [INFO] train episode 548: reward = -314.00, steps = 315
    23:00:14 [INFO] train episode 549: reward = -187.00, steps = 188
    23:00:14 [INFO] train episode 550: reward = -243.00, steps = 244
    23:00:15 [INFO] train episode 551: reward = -174.00, steps = 175
    23:00:15 [INFO] train episode 552: reward = -197.00, steps = 198
    23:00:15 [INFO] train episode 553: reward = -191.00, steps = 192
    23:00:15 [INFO] train episode 554: reward = -175.00, steps = 176
    23:00:15 [INFO] train episode 555: reward = -184.00, steps = 185
    23:00:15 [INFO] train episode 556: reward = -144.00, steps = 145
    23:00:16 [INFO] train episode 557: reward = -205.00, steps = 206
    23:00:16 [INFO] train episode 558: reward = -152.00, steps = 153
    23:00:16 [INFO] train episode 559: reward = -165.00, steps = 166
    23:00:16 [INFO] train episode 560: reward = -173.00, steps = 174
    23:00:16 [INFO] train episode 561: reward = -184.00, steps = 185
    23:00:17 [INFO] train episode 562: reward = -167.00, steps = 168
    23:00:17 [INFO] train episode 563: reward = -221.00, steps = 222
    23:00:17 [INFO] train episode 564: reward = -184.00, steps = 185
    23:00:17 [INFO] train episode 565: reward = -188.00, steps = 189
    23:00:18 [INFO] train episode 566: reward = -181.00, steps = 182
    23:00:18 [INFO] train episode 567: reward = -170.00, steps = 171
    23:00:18 [INFO] train episode 568: reward = -189.00, steps = 190
    23:00:18 [INFO] train episode 569: reward = -159.00, steps = 160
    23:00:18 [INFO] train episode 570: reward = -186.00, steps = 187
    23:00:19 [INFO] train episode 571: reward = -168.00, steps = 169
    23:00:19 [INFO] train episode 572: reward = -140.00, steps = 141
    23:00:19 [INFO] train episode 573: reward = -194.00, steps = 195
    23:00:19 [INFO] train episode 574: reward = -220.00, steps = 221
    23:00:19 [INFO] train episode 575: reward = -168.00, steps = 169
    23:00:20 [INFO] train episode 576: reward = -156.00, steps = 157
    23:00:20 [INFO] train episode 577: reward = -196.00, steps = 197
    23:00:20 [INFO] train episode 578: reward = -170.00, steps = 171
    23:00:20 [INFO] train episode 579: reward = -181.00, steps = 182
    23:00:21 [INFO] train episode 580: reward = -364.00, steps = 365
    23:00:21 [INFO] train episode 581: reward = -215.00, steps = 216
    23:00:21 [INFO] train episode 582: reward = -166.00, steps = 167
    23:00:21 [INFO] train episode 583: reward = -183.00, steps = 184
    23:00:21 [INFO] train episode 584: reward = -164.00, steps = 165
    23:00:22 [INFO] train episode 585: reward = -131.00, steps = 132
    23:00:22 [INFO] train episode 586: reward = -235.00, steps = 236
    23:00:22 [INFO] train episode 587: reward = -202.00, steps = 203
    23:00:22 [INFO] train episode 588: reward = -170.00, steps = 171
    23:00:22 [INFO] train episode 589: reward = -151.00, steps = 152
    23:00:23 [INFO] train episode 590: reward = -205.00, steps = 206
    23:00:23 [INFO] train episode 591: reward = -168.00, steps = 169
    23:00:23 [INFO] train episode 592: reward = -179.00, steps = 180
    23:00:23 [INFO] train episode 593: reward = -216.00, steps = 217
    23:00:23 [INFO] train episode 594: reward = -181.00, steps = 182
    23:00:24 [INFO] train episode 595: reward = -149.00, steps = 150
    23:00:24 [INFO] train episode 596: reward = -138.00, steps = 139
    23:00:24 [INFO] train episode 597: reward = -160.00, steps = 161
    23:00:24 [INFO] train episode 598: reward = -163.00, steps = 164
    23:00:24 [INFO] train episode 599: reward = -179.00, steps = 180
    23:00:25 [INFO] train episode 600: reward = -191.00, steps = 192
    23:00:25 [INFO] train episode 601: reward = -202.00, steps = 203
    23:00:25 [INFO] train episode 602: reward = -247.00, steps = 248
    23:00:25 [INFO] train episode 603: reward = -252.00, steps = 253
    23:00:25 [INFO] train episode 604: reward = -209.00, steps = 210
    23:00:26 [INFO] train episode 605: reward = -168.00, steps = 169
    23:00:26 [INFO] train episode 606: reward = -226.00, steps = 227
    23:00:26 [INFO] train episode 607: reward = -153.00, steps = 154
    23:00:26 [INFO] train episode 608: reward = -130.00, steps = 131
    23:00:27 [INFO] train episode 609: reward = -184.00, steps = 185
    23:00:27 [INFO] train episode 610: reward = -179.00, steps = 180
    23:00:27 [INFO] train episode 611: reward = -165.00, steps = 166
    23:00:27 [INFO] train episode 612: reward = -326.00, steps = 327
    23:00:27 [INFO] train episode 613: reward = -200.00, steps = 201
    23:00:28 [INFO] train episode 614: reward = -139.00, steps = 140
    23:00:28 [INFO] train episode 615: reward = -230.00, steps = 231
    23:00:28 [INFO] train episode 616: reward = -181.00, steps = 182
    23:00:28 [INFO] train episode 617: reward = -190.00, steps = 191
    23:00:28 [INFO] train episode 618: reward = -137.00, steps = 138
    23:00:29 [INFO] train episode 619: reward = -183.00, steps = 184
    23:00:29 [INFO] train episode 620: reward = -210.00, steps = 211
    23:00:29 [INFO] train episode 621: reward = -151.00, steps = 152
    23:00:29 [INFO] train episode 622: reward = -295.00, steps = 296
    23:00:30 [INFO] train episode 623: reward = -227.00, steps = 228
    23:00:30 [INFO] train episode 624: reward = -120.00, steps = 121
    23:00:30 [INFO] train episode 625: reward = -201.00, steps = 202
    23:00:30 [INFO] train episode 626: reward = -210.00, steps = 211
    23:00:30 [INFO] train episode 627: reward = -140.00, steps = 141
    23:00:30 [INFO] train episode 628: reward = -158.00, steps = 159
    23:00:31 [INFO] train episode 629: reward = -137.00, steps = 138
    23:00:31 [INFO] train episode 630: reward = -236.00, steps = 237
    23:00:31 [INFO] train episode 631: reward = -172.00, steps = 173
    23:00:31 [INFO] train episode 632: reward = -297.00, steps = 298
    23:00:32 [INFO] train episode 633: reward = -126.00, steps = 127
    23:00:32 [INFO] train episode 634: reward = -172.00, steps = 173
    23:00:32 [INFO] train episode 635: reward = -253.00, steps = 254
    23:00:32 [INFO] train episode 636: reward = -202.00, steps = 203
    23:00:32 [INFO] train episode 637: reward = -203.00, steps = 204
    23:00:33 [INFO] train episode 638: reward = -179.00, steps = 180
    23:00:33 [INFO] train episode 639: reward = -169.00, steps = 170
    23:00:33 [INFO] train episode 640: reward = -166.00, steps = 167
    23:00:33 [INFO] train episode 641: reward = -167.00, steps = 168
    23:00:34 [INFO] train episode 642: reward = -272.00, steps = 273
    23:00:34 [INFO] train episode 643: reward = -165.00, steps = 166
    23:00:34 [INFO] train episode 644: reward = -153.00, steps = 154
    23:00:34 [INFO] train episode 645: reward = -165.00, steps = 166
    23:00:34 [INFO] train episode 646: reward = -225.00, steps = 226
    23:00:35 [INFO] train episode 647: reward = -172.00, steps = 173
    23:00:35 [INFO] train episode 648: reward = -204.00, steps = 205
    23:00:35 [INFO] train episode 649: reward = -142.00, steps = 143
    23:00:35 [INFO] train episode 650: reward = -180.00, steps = 181
    23:00:35 [INFO] train episode 651: reward = -194.00, steps = 195
    23:00:36 [INFO] train episode 652: reward = -195.00, steps = 196
    23:00:36 [INFO] train episode 653: reward = -184.00, steps = 185
    23:00:36 [INFO] train episode 654: reward = -114.00, steps = 115
    23:00:36 [INFO] train episode 655: reward = -167.00, steps = 168
    23:00:36 [INFO] train episode 656: reward = -201.00, steps = 202
    23:00:37 [INFO] train episode 657: reward = -154.00, steps = 155
    23:00:37 [INFO] train episode 658: reward = -124.00, steps = 125
    23:00:37 [INFO] train episode 659: reward = -167.00, steps = 168
    23:00:37 [INFO] train episode 660: reward = -175.00, steps = 176
    23:00:37 [INFO] train episode 661: reward = -229.00, steps = 230
    23:00:37 [INFO] train episode 662: reward = -158.00, steps = 159
    23:00:38 [INFO] train episode 663: reward = -196.00, steps = 197
    23:00:38 [INFO] train episode 664: reward = -201.00, steps = 202
    23:00:38 [INFO] train episode 665: reward = -211.00, steps = 212
    23:00:38 [INFO] train episode 666: reward = -195.00, steps = 196
    23:00:39 [INFO] train episode 667: reward = -196.00, steps = 197
    23:00:39 [INFO] train episode 668: reward = -257.00, steps = 258
    23:00:39 [INFO] train episode 669: reward = -135.00, steps = 136
    23:00:39 [INFO] train episode 670: reward = -214.00, steps = 215
    23:00:39 [INFO] train episode 671: reward = -176.00, steps = 177
    23:00:40 [INFO] train episode 672: reward = -162.00, steps = 163
    23:00:40 [INFO] train episode 673: reward = -175.00, steps = 176
    23:00:40 [INFO] train episode 674: reward = -200.00, steps = 201
    23:00:40 [INFO] train episode 675: reward = -191.00, steps = 192
    23:00:40 [INFO] train episode 676: reward = -128.00, steps = 129
    23:00:41 [INFO] train episode 677: reward = -143.00, steps = 144
    23:00:41 [INFO] train episode 678: reward = -187.00, steps = 188
    23:00:41 [INFO] train episode 679: reward = -129.00, steps = 130
    23:00:41 [INFO] train episode 680: reward = -175.00, steps = 176
    23:00:41 [INFO] train episode 681: reward = -171.00, steps = 172
    23:00:42 [INFO] train episode 682: reward = -324.00, steps = 325
    23:00:42 [INFO] train episode 683: reward = -131.00, steps = 132
    23:00:42 [INFO] train episode 684: reward = -152.00, steps = 153
    23:00:42 [INFO] train episode 685: reward = -181.00, steps = 182
    23:00:42 [INFO] train episode 686: reward = -144.00, steps = 145
    23:00:43 [INFO] train episode 687: reward = -200.00, steps = 201
    23:00:43 [INFO] train episode 688: reward = -209.00, steps = 210
    23:00:43 [INFO] train episode 689: reward = -152.00, steps = 153
    23:00:43 [INFO] train episode 690: reward = -207.00, steps = 208
    23:00:43 [INFO] train episode 691: reward = -130.00, steps = 131
    23:00:44 [INFO] train episode 692: reward = -169.00, steps = 170
    23:00:44 [INFO] train episode 693: reward = -180.00, steps = 181
    23:00:44 [INFO] train episode 694: reward = -324.00, steps = 325
    23:00:44 [INFO] train episode 695: reward = -175.00, steps = 176
    23:00:44 [INFO] train episode 696: reward = -191.00, steps = 192
    23:00:45 [INFO] train episode 697: reward = -201.00, steps = 202
    23:00:45 [INFO] train episode 698: reward = -180.00, steps = 181
    23:00:45 [INFO] train episode 699: reward = -172.00, steps = 173
    23:00:45 [INFO] train episode 700: reward = -181.00, steps = 182
    23:00:45 [INFO] train episode 701: reward = -140.00, steps = 141
    23:00:46 [INFO] train episode 702: reward = -149.00, steps = 150
    23:00:46 [INFO] train episode 703: reward = -291.00, steps = 292
    23:00:46 [INFO] train episode 704: reward = -188.00, steps = 189
    23:00:46 [INFO] train episode 705: reward = -210.00, steps = 211
    23:00:46 [INFO] train episode 706: reward = -169.00, steps = 170
    23:00:47 [INFO] train episode 707: reward = -167.00, steps = 168
    23:00:47 [INFO] train episode 708: reward = -175.00, steps = 176
    23:00:47 [INFO] train episode 709: reward = -145.00, steps = 146
    23:00:47 [INFO] train episode 710: reward = -168.00, steps = 169
    23:00:47 [INFO] train episode 711: reward = -142.00, steps = 143
    23:00:48 [INFO] train episode 712: reward = -151.00, steps = 152
    23:00:48 [INFO] train episode 713: reward = -118.00, steps = 119
    23:00:48 [INFO] train episode 714: reward = -255.00, steps = 256
    23:00:48 [INFO] train episode 715: reward = -249.00, steps = 250
    23:00:48 [INFO] train episode 716: reward = -155.00, steps = 156
    23:00:49 [INFO] train episode 717: reward = -163.00, steps = 164
    23:00:49 [INFO] train episode 718: reward = -167.00, steps = 168
    23:00:49 [INFO] train episode 719: reward = -163.00, steps = 164
    23:00:49 [INFO] train episode 720: reward = -141.00, steps = 142
    23:00:49 [INFO] train episode 721: reward = -148.00, steps = 149
    23:00:50 [INFO] train episode 722: reward = -187.00, steps = 188
    23:00:50 [INFO] train episode 723: reward = -177.00, steps = 178
    23:00:50 [INFO] train episode 724: reward = -183.00, steps = 184
    23:00:50 [INFO] train episode 725: reward = -221.00, steps = 222
    23:00:51 [INFO] train episode 726: reward = -189.00, steps = 190
    23:00:51 [INFO] train episode 727: reward = -167.00, steps = 168
    23:00:51 [INFO] train episode 728: reward = -140.00, steps = 141
    23:00:51 [INFO] train episode 729: reward = -181.00, steps = 182
    23:00:51 [INFO] train episode 730: reward = -214.00, steps = 215
    23:00:52 [INFO] train episode 731: reward = -192.00, steps = 193
    23:00:52 [INFO] train episode 732: reward = -210.00, steps = 211
    23:00:52 [INFO] train episode 733: reward = -155.00, steps = 156
    23:00:52 [INFO] train episode 734: reward = -174.00, steps = 175
    23:00:52 [INFO] train episode 735: reward = -163.00, steps = 164
    23:00:53 [INFO] train episode 736: reward = -161.00, steps = 162
    23:00:53 [INFO] train episode 737: reward = -156.00, steps = 157
    23:00:53 [INFO] train episode 738: reward = -136.00, steps = 137
    23:00:53 [INFO] train episode 739: reward = -197.00, steps = 198
    23:00:53 [INFO] train episode 740: reward = -197.00, steps = 198
    23:00:54 [INFO] train episode 741: reward = -131.00, steps = 132
    23:00:54 [INFO] train episode 742: reward = -171.00, steps = 172
    23:00:54 [INFO] train episode 743: reward = -117.00, steps = 118
    23:00:54 [INFO] train episode 744: reward = -171.00, steps = 172
    23:00:54 [INFO] train episode 745: reward = -133.00, steps = 134
    23:00:54 [INFO] train episode 746: reward = -186.00, steps = 187
    23:00:55 [INFO] train episode 747: reward = -139.00, steps = 140
    23:00:55 [INFO] train episode 748: reward = -174.00, steps = 175
    23:00:55 [INFO] train episode 749: reward = -206.00, steps = 207
    23:00:55 [INFO] train episode 750: reward = -158.00, steps = 159
    23:00:55 [INFO] train episode 751: reward = -235.00, steps = 236
    23:00:55 [INFO] train episode 752: reward = -160.00, steps = 161
    23:00:56 [INFO] train episode 753: reward = -183.00, steps = 184
    23:00:56 [INFO] train episode 754: reward = -181.00, steps = 182
    23:00:56 [INFO] train episode 755: reward = -167.00, steps = 168
    23:00:56 [INFO] train episode 756: reward = -181.00, steps = 182
    23:00:57 [INFO] train episode 757: reward = -500.00, steps = 500
    23:00:57 [INFO] train episode 758: reward = -178.00, steps = 179
    23:00:57 [INFO] train episode 759: reward = -154.00, steps = 155
    23:00:57 [INFO] train episode 760: reward = -176.00, steps = 177
    23:00:58 [INFO] train episode 761: reward = -221.00, steps = 222
    23:00:58 [INFO] train episode 762: reward = -145.00, steps = 146
    23:00:58 [INFO] train episode 763: reward = -175.00, steps = 176
    23:00:58 [INFO] train episode 764: reward = -135.00, steps = 136
    23:00:58 [INFO] train episode 765: reward = -131.00, steps = 132
    23:00:59 [INFO] train episode 766: reward = -177.00, steps = 178
    23:00:59 [INFO] train episode 767: reward = -183.00, steps = 184
    23:00:59 [INFO] train episode 768: reward = -190.00, steps = 191
    23:00:59 [INFO] train episode 769: reward = -154.00, steps = 155
    23:00:59 [INFO] train episode 770: reward = -126.00, steps = 127
    23:01:00 [INFO] train episode 771: reward = -174.00, steps = 175
    23:01:00 [INFO] train episode 772: reward = -203.00, steps = 204
    23:01:00 [INFO] train episode 773: reward = -164.00, steps = 165
    23:01:00 [INFO] train episode 774: reward = -155.00, steps = 156
    23:01:00 [INFO] train episode 775: reward = -123.00, steps = 124
    23:01:01 [INFO] train episode 776: reward = -165.00, steps = 166
    23:01:01 [INFO] train episode 777: reward = -128.00, steps = 129
    23:01:01 [INFO] train episode 778: reward = -178.00, steps = 179
    23:01:01 [INFO] train episode 779: reward = -175.00, steps = 176
    23:01:01 [INFO] train episode 780: reward = -200.00, steps = 201
    23:01:01 [INFO] train episode 781: reward = -155.00, steps = 156
    23:01:02 [INFO] train episode 782: reward = -148.00, steps = 149
    23:01:02 [INFO] train episode 783: reward = -204.00, steps = 205
    23:01:02 [INFO] train episode 784: reward = -149.00, steps = 150
    23:01:02 [INFO] train episode 785: reward = -212.00, steps = 213
    23:01:02 [INFO] train episode 786: reward = -149.00, steps = 150
    23:01:03 [INFO] train episode 787: reward = -150.00, steps = 151
    23:01:03 [INFO] train episode 788: reward = -170.00, steps = 171
    23:01:03 [INFO] train episode 789: reward = -202.00, steps = 203
    23:01:03 [INFO] train episode 790: reward = -233.00, steps = 234
    23:01:03 [INFO] train episode 791: reward = -162.00, steps = 163
    23:01:04 [INFO] train episode 792: reward = -193.00, steps = 194
    23:01:04 [INFO] train episode 793: reward = -187.00, steps = 188
    23:01:04 [INFO] train episode 794: reward = -167.00, steps = 168
    23:01:04 [INFO] train episode 795: reward = -159.00, steps = 160
    23:01:04 [INFO] train episode 796: reward = -166.00, steps = 167
    23:01:04 [INFO] train episode 797: reward = -152.00, steps = 153
    23:01:05 [INFO] train episode 798: reward = -235.00, steps = 236
    23:01:05 [INFO] train episode 799: reward = -175.00, steps = 176
    23:01:05 [INFO] train episode 800: reward = -184.00, steps = 185
    23:01:05 [INFO] train episode 801: reward = -191.00, steps = 192
    23:01:05 [INFO] train episode 802: reward = -140.00, steps = 141
    23:01:06 [INFO] train episode 803: reward = -115.00, steps = 116
    23:01:06 [INFO] train episode 804: reward = -170.00, steps = 171
    23:01:06 [INFO] train episode 805: reward = -161.00, steps = 162
    23:01:06 [INFO] train episode 806: reward = -183.00, steps = 184
    23:01:06 [INFO] train episode 807: reward = -158.00, steps = 159
    23:01:07 [INFO] train episode 808: reward = -193.00, steps = 194
    23:01:07 [INFO] train episode 809: reward = -147.00, steps = 148
    23:01:07 [INFO] train episode 810: reward = -197.00, steps = 198
    23:01:07 [INFO] train episode 811: reward = -166.00, steps = 167
    23:01:07 [INFO] train episode 812: reward = -144.00, steps = 145
    23:01:07 [INFO] train episode 813: reward = -141.00, steps = 142
    23:01:08 [INFO] train episode 814: reward = -121.00, steps = 122
    23:01:08 [INFO] train episode 815: reward = -165.00, steps = 166
    23:01:08 [INFO] train episode 816: reward = -192.00, steps = 193
    23:01:08 [INFO] train episode 817: reward = -169.00, steps = 170
    23:01:08 [INFO] train episode 818: reward = -222.00, steps = 223
    23:01:09 [INFO] train episode 819: reward = -181.00, steps = 182
    23:01:09 [INFO] train episode 820: reward = -161.00, steps = 162
    23:01:09 [INFO] train episode 821: reward = -133.00, steps = 134
    23:01:09 [INFO] train episode 822: reward = -174.00, steps = 175
    23:01:09 [INFO] train episode 823: reward = -154.00, steps = 155
    23:01:09 [INFO] train episode 824: reward = -165.00, steps = 166
    23:01:10 [INFO] train episode 825: reward = -214.00, steps = 215
    23:01:10 [INFO] train episode 826: reward = -177.00, steps = 178
    23:01:10 [INFO] train episode 827: reward = -201.00, steps = 202
    23:01:10 [INFO] train episode 828: reward = -276.00, steps = 277
    23:01:10 [INFO] train episode 829: reward = -212.00, steps = 213
    23:01:11 [INFO] train episode 830: reward = -132.00, steps = 133
    23:01:11 [INFO] train episode 831: reward = -137.00, steps = 138
    23:01:11 [INFO] train episode 832: reward = -164.00, steps = 165
    23:01:11 [INFO] train episode 833: reward = -122.00, steps = 123
    23:01:11 [INFO] train episode 834: reward = -176.00, steps = 177
    23:01:11 [INFO] train episode 835: reward = -161.00, steps = 162
    23:01:12 [INFO] train episode 836: reward = -132.00, steps = 133
    23:01:12 [INFO] train episode 837: reward = -194.00, steps = 195
    23:01:12 [INFO] train episode 838: reward = -162.00, steps = 163
    23:01:12 [INFO] train episode 839: reward = -127.00, steps = 128
    23:01:12 [INFO] train episode 840: reward = -249.00, steps = 250
    23:01:13 [INFO] train episode 841: reward = -132.00, steps = 133
    23:01:13 [INFO] train episode 842: reward = -195.00, steps = 196
    23:01:13 [INFO] train episode 843: reward = -167.00, steps = 168
    23:01:13 [INFO] train episode 844: reward = -189.00, steps = 190
    23:01:13 [INFO] train episode 845: reward = -165.00, steps = 166
    23:01:14 [INFO] train episode 846: reward = -228.00, steps = 229
    23:01:14 [INFO] train episode 847: reward = -199.00, steps = 200
    23:01:14 [INFO] train episode 848: reward = -162.00, steps = 163
    23:01:14 [INFO] train episode 849: reward = -153.00, steps = 154
    23:01:14 [INFO] train episode 850: reward = -164.00, steps = 165
    23:01:14 [INFO] train episode 851: reward = -232.00, steps = 233
    23:01:15 [INFO] train episode 852: reward = -149.00, steps = 150
    23:01:15 [INFO] train episode 853: reward = -251.00, steps = 252
    23:01:15 [INFO] train episode 854: reward = -141.00, steps = 142
    23:01:15 [INFO] train episode 855: reward = -158.00, steps = 159
    23:01:15 [INFO] train episode 856: reward = -165.00, steps = 166
    23:01:15 [INFO] train episode 857: reward = -162.00, steps = 163
    23:01:16 [INFO] train episode 858: reward = -117.00, steps = 118
    23:01:16 [INFO] train episode 859: reward = -173.00, steps = 174
    23:01:16 [INFO] train episode 860: reward = -177.00, steps = 178
    23:01:16 [INFO] train episode 861: reward = -133.00, steps = 134
    23:01:16 [INFO] train episode 862: reward = -129.00, steps = 130
    23:01:16 [INFO] train episode 863: reward = -194.00, steps = 195
    23:01:17 [INFO] train episode 864: reward = -143.00, steps = 144
    23:01:17 [INFO] train episode 865: reward = -196.00, steps = 197
    23:01:17 [INFO] train episode 866: reward = -155.00, steps = 156
    23:01:17 [INFO] train episode 867: reward = -176.00, steps = 177
    23:01:17 [INFO] train episode 868: reward = -147.00, steps = 148
    23:01:17 [INFO] train episode 869: reward = -164.00, steps = 165
    23:01:18 [INFO] train episode 870: reward = -359.00, steps = 360
    23:01:18 [INFO] train episode 871: reward = -145.00, steps = 146
    23:01:18 [INFO] train episode 872: reward = -165.00, steps = 166
    23:01:18 [INFO] train episode 873: reward = -148.00, steps = 149
    23:01:19 [INFO] train episode 874: reward = -212.00, steps = 213
    23:01:19 [INFO] train episode 875: reward = -199.00, steps = 200
    23:01:19 [INFO] train episode 876: reward = -219.00, steps = 220
    23:01:19 [INFO] train episode 877: reward = -171.00, steps = 172
    23:01:19 [INFO] train episode 878: reward = -147.00, steps = 148
    23:01:19 [INFO] train episode 879: reward = -232.00, steps = 233
    23:01:20 [INFO] train episode 880: reward = -148.00, steps = 149
    23:01:20 [INFO] train episode 881: reward = -139.00, steps = 140
    23:01:20 [INFO] train episode 882: reward = -226.00, steps = 227
    23:01:20 [INFO] train episode 883: reward = -151.00, steps = 152
    23:01:20 [INFO] train episode 884: reward = -205.00, steps = 206
    23:01:20 [INFO] train episode 885: reward = -124.00, steps = 125
    23:01:21 [INFO] train episode 886: reward = -197.00, steps = 198
    23:01:21 [INFO] train episode 887: reward = -211.00, steps = 212
    23:01:21 [INFO] train episode 888: reward = -165.00, steps = 166
    23:01:21 [INFO] train episode 889: reward = -172.00, steps = 173
    23:01:21 [INFO] train episode 890: reward = -174.00, steps = 175
    23:01:22 [INFO] train episode 891: reward = -179.00, steps = 180
    23:01:22 [INFO] train episode 892: reward = -160.00, steps = 161
    23:01:22 [INFO] train episode 893: reward = -192.00, steps = 193
    23:01:22 [INFO] train episode 894: reward = -142.00, steps = 143
    23:01:22 [INFO] train episode 895: reward = -208.00, steps = 209
    23:01:23 [INFO] train episode 896: reward = -500.00, steps = 500
    23:01:23 [INFO] train episode 897: reward = -163.00, steps = 164
    23:01:23 [INFO] train episode 898: reward = -155.00, steps = 156
    23:01:23 [INFO] train episode 899: reward = -103.00, steps = 104
    23:01:23 [INFO] train episode 900: reward = -245.00, steps = 246
    23:01:24 [INFO] train episode 901: reward = -152.00, steps = 153
    23:01:24 [INFO] train episode 902: reward = -168.00, steps = 169
    23:01:24 [INFO] train episode 903: reward = -142.00, steps = 143
    23:01:24 [INFO] train episode 904: reward = -213.00, steps = 214
    23:01:24 [INFO] train episode 905: reward = -154.00, steps = 155
    23:01:24 [INFO] train episode 906: reward = -161.00, steps = 162
    23:01:25 [INFO] train episode 907: reward = -136.00, steps = 137
    23:01:25 [INFO] train episode 908: reward = -168.00, steps = 169
    23:01:25 [INFO] train episode 909: reward = -164.00, steps = 165
    23:01:25 [INFO] train episode 910: reward = -148.00, steps = 149
    23:01:25 [INFO] train episode 911: reward = -168.00, steps = 169
    23:01:25 [INFO] train episode 912: reward = -142.00, steps = 143
    23:01:26 [INFO] train episode 913: reward = -155.00, steps = 156
    23:01:26 [INFO] train episode 914: reward = -172.00, steps = 173
    23:01:26 [INFO] train episode 915: reward = -184.00, steps = 185
    23:01:26 [INFO] train episode 916: reward = -204.00, steps = 205
    23:01:26 [INFO] train episode 917: reward = -155.00, steps = 156
    23:01:27 [INFO] train episode 918: reward = -150.00, steps = 151
    23:01:27 [INFO] train episode 919: reward = -175.00, steps = 176
    23:01:27 [INFO] train episode 920: reward = -174.00, steps = 175
    23:01:27 [INFO] train episode 921: reward = -155.00, steps = 156
    23:01:27 [INFO] train episode 922: reward = -152.00, steps = 153
    23:01:27 [INFO] train episode 923: reward = -136.00, steps = 137
    23:01:27 [INFO] train episode 924: reward = -138.00, steps = 139
    23:01:28 [INFO] train episode 925: reward = -150.00, steps = 151
    23:01:28 [INFO] train episode 926: reward = -146.00, steps = 147
    23:01:28 [INFO] train episode 927: reward = -171.00, steps = 172
    23:01:28 [INFO] train episode 928: reward = -208.00, steps = 209
    23:01:28 [INFO] train episode 929: reward = -216.00, steps = 217
    23:01:29 [INFO] train episode 930: reward = -159.00, steps = 160
    23:01:29 [INFO] train episode 931: reward = -180.00, steps = 181
    23:01:29 [INFO] train episode 932: reward = -189.00, steps = 190
    23:01:29 [INFO] train episode 933: reward = -143.00, steps = 144
    23:01:29 [INFO] train episode 934: reward = -142.00, steps = 143
    23:01:29 [INFO] train episode 935: reward = -164.00, steps = 165
    23:01:30 [INFO] train episode 936: reward = -145.00, steps = 146
    23:01:30 [INFO] train episode 937: reward = -154.00, steps = 155
    23:01:30 [INFO] train episode 938: reward = -146.00, steps = 147
    23:01:30 [INFO] train episode 939: reward = -178.00, steps = 179
    23:01:30 [INFO] train episode 940: reward = -183.00, steps = 184
    23:01:30 [INFO] train episode 941: reward = -226.00, steps = 227
    23:01:31 [INFO] train episode 942: reward = -182.00, steps = 183
    23:01:31 [INFO] train episode 943: reward = -167.00, steps = 168
    23:01:31 [INFO] train episode 944: reward = -228.00, steps = 229
    23:01:31 [INFO] train episode 945: reward = -139.00, steps = 140
    23:01:31 [INFO] train episode 946: reward = -155.00, steps = 156
    23:01:31 [INFO] train episode 947: reward = -184.00, steps = 185
    23:01:32 [INFO] train episode 948: reward = -173.00, steps = 174
    23:01:32 [INFO] train episode 949: reward = -191.00, steps = 192
    23:01:32 [INFO] train episode 950: reward = -170.00, steps = 171
    23:01:32 [INFO] train episode 951: reward = -194.00, steps = 195
    23:01:32 [INFO] train episode 952: reward = -160.00, steps = 161
    23:01:33 [INFO] train episode 953: reward = -150.00, steps = 151
    23:01:33 [INFO] train episode 954: reward = -168.00, steps = 169
    23:01:33 [INFO] train episode 955: reward = -190.00, steps = 191
    23:01:33 [INFO] train episode 956: reward = -152.00, steps = 153
    23:01:33 [INFO] train episode 957: reward = -142.00, steps = 143
    23:01:33 [INFO] train episode 958: reward = -142.00, steps = 143
    23:01:34 [INFO] train episode 959: reward = -230.00, steps = 231
    23:01:34 [INFO] train episode 960: reward = -188.00, steps = 189
    23:01:34 [INFO] train episode 961: reward = -179.00, steps = 180
    23:01:34 [INFO] train episode 962: reward = -319.00, steps = 320
    23:01:35 [INFO] train episode 963: reward = -170.00, steps = 171
    23:01:35 [INFO] train episode 964: reward = -165.00, steps = 166
    23:01:35 [INFO] train episode 965: reward = -128.00, steps = 129
    23:01:35 [INFO] train episode 966: reward = -144.00, steps = 145
    23:01:35 [INFO] train episode 967: reward = -202.00, steps = 203
    23:01:35 [INFO] train episode 968: reward = -154.00, steps = 155
    23:01:36 [INFO] train episode 969: reward = -169.00, steps = 170
    23:01:36 [INFO] train episode 970: reward = -179.00, steps = 180
    23:01:36 [INFO] train episode 971: reward = -158.00, steps = 159
    23:01:36 [INFO] train episode 972: reward = -159.00, steps = 160
    23:01:36 [INFO] train episode 973: reward = -149.00, steps = 150
    23:01:36 [INFO] train episode 974: reward = -171.00, steps = 172
    23:01:37 [INFO] train episode 975: reward = -206.00, steps = 207
    23:01:37 [INFO] train episode 976: reward = -141.00, steps = 142
    23:01:37 [INFO] train episode 977: reward = -215.00, steps = 216
    23:01:37 [INFO] train episode 978: reward = -169.00, steps = 170
    23:01:37 [INFO] train episode 979: reward = -134.00, steps = 135
    23:01:38 [INFO] train episode 980: reward = -172.00, steps = 173
    23:01:38 [INFO] train episode 981: reward = -202.00, steps = 203
    23:01:38 [INFO] train episode 982: reward = -140.00, steps = 141
    23:01:38 [INFO] train episode 983: reward = -144.00, steps = 145
    23:01:38 [INFO] train episode 984: reward = -214.00, steps = 215
    23:01:39 [INFO] train episode 985: reward = -170.00, steps = 171
    23:01:39 [INFO] train episode 986: reward = -238.00, steps = 239
    23:01:39 [INFO] train episode 987: reward = -166.00, steps = 167
    23:01:39 [INFO] train episode 988: reward = -171.00, steps = 172
    23:01:39 [INFO] train episode 989: reward = -117.00, steps = 118
    23:01:39 [INFO] train episode 990: reward = -170.00, steps = 171
    23:01:40 [INFO] train episode 991: reward = -159.00, steps = 160
    23:01:40 [INFO] train episode 992: reward = -185.00, steps = 186
    23:01:40 [INFO] train episode 993: reward = -152.00, steps = 153
    23:01:40 [INFO] train episode 994: reward = -166.00, steps = 167
    23:01:40 [INFO] train episode 995: reward = -189.00, steps = 190
    23:01:41 [INFO] train episode 996: reward = -212.00, steps = 213
    23:01:41 [INFO] train episode 997: reward = -181.00, steps = 182
    23:01:41 [INFO] train episode 998: reward = -151.00, steps = 152
    23:01:41 [INFO] train episode 999: reward = -150.00, steps = 151
    23:01:41 [INFO] train episode 1000: reward = -158.00, steps = 159
    23:01:41 [INFO] train episode 1001: reward = -212.00, steps = 213
    23:01:42 [INFO] train episode 1002: reward = -155.00, steps = 156
    23:01:42 [INFO] train episode 1003: reward = -240.00, steps = 241
    23:01:42 [INFO] train episode 1004: reward = -143.00, steps = 144
    23:01:42 [INFO] train episode 1005: reward = -262.00, steps = 263
    23:01:42 [INFO] train episode 1006: reward = -181.00, steps = 182
    23:01:43 [INFO] train episode 1007: reward = -172.00, steps = 173
    23:01:43 [INFO] train episode 1008: reward = -319.00, steps = 320
    23:01:43 [INFO] train episode 1009: reward = -143.00, steps = 144
    23:01:43 [INFO] train episode 1010: reward = -167.00, steps = 168
    23:01:43 [INFO] train episode 1011: reward = -151.00, steps = 152
    23:01:44 [INFO] train episode 1012: reward = -220.00, steps = 221
    23:01:44 [INFO] train episode 1013: reward = -402.00, steps = 403
    23:01:44 [INFO] train episode 1014: reward = -278.00, steps = 279
    23:01:44 [INFO] train episode 1015: reward = -145.00, steps = 146
    23:01:45 [INFO] train episode 1016: reward = -166.00, steps = 167
    23:01:45 [INFO] train episode 1017: reward = -232.00, steps = 233
    23:01:45 [INFO] train episode 1018: reward = -158.00, steps = 159
    23:01:45 [INFO] train episode 1019: reward = -157.00, steps = 158
    23:01:45 [INFO] train episode 1020: reward = -169.00, steps = 170
    23:01:46 [INFO] train episode 1021: reward = -158.00, steps = 159
    23:01:46 [INFO] train episode 1022: reward = -196.00, steps = 197
    23:01:46 [INFO] train episode 1023: reward = -197.00, steps = 198
    23:01:46 [INFO] train episode 1024: reward = -172.00, steps = 173
    23:01:46 [INFO] train episode 1025: reward = -192.00, steps = 193
    23:01:47 [INFO] train episode 1026: reward = -188.00, steps = 189
    23:01:47 [INFO] train episode 1027: reward = -130.00, steps = 131
    23:01:47 [INFO] train episode 1028: reward = -212.00, steps = 213
    23:01:47 [INFO] train episode 1029: reward = -188.00, steps = 189
    23:01:47 [INFO] train episode 1030: reward = -134.00, steps = 135
    23:01:47 [INFO] train episode 1031: reward = -214.00, steps = 215
    23:01:48 [INFO] train episode 1032: reward = -164.00, steps = 165
    23:01:48 [INFO] train episode 1033: reward = -144.00, steps = 145
    23:01:48 [INFO] train episode 1034: reward = -250.00, steps = 251
    23:01:48 [INFO] train episode 1035: reward = -229.00, steps = 230
    23:01:48 [INFO] train episode 1036: reward = -163.00, steps = 164
    23:01:49 [INFO] train episode 1037: reward = -144.00, steps = 145
    23:01:49 [INFO] train episode 1038: reward = -150.00, steps = 151
    23:01:49 [INFO] train episode 1039: reward = -195.00, steps = 196
    23:01:49 [INFO] train episode 1040: reward = -182.00, steps = 183
    23:01:49 [INFO] train episode 1041: reward = -135.00, steps = 136
    23:01:50 [INFO] train episode 1042: reward = -190.00, steps = 191
    23:01:50 [INFO] train episode 1043: reward = -243.00, steps = 244
    23:01:50 [INFO] train episode 1044: reward = -199.00, steps = 200
    23:01:50 [INFO] train episode 1045: reward = -187.00, steps = 188
    23:01:50 [INFO] train episode 1046: reward = -203.00, steps = 204
    23:01:51 [INFO] train episode 1047: reward = -164.00, steps = 165
    23:01:51 [INFO] train episode 1048: reward = -162.00, steps = 163
    23:01:51 [INFO] train episode 1049: reward = -176.00, steps = 177
    23:01:51 [INFO] train episode 1050: reward = -167.00, steps = 168
    23:01:51 [INFO] train episode 1051: reward = -176.00, steps = 177
    23:01:51 [INFO] train episode 1052: reward = -197.00, steps = 198
    23:01:52 [INFO] train episode 1053: reward = -181.00, steps = 182
    23:01:52 [INFO] train episode 1054: reward = -149.00, steps = 150
    23:01:52 [INFO] train episode 1055: reward = -140.00, steps = 141
    23:01:52 [INFO] train episode 1056: reward = -151.00, steps = 152
    23:01:52 [INFO] train episode 1057: reward = -179.00, steps = 180
    23:01:52 [INFO] train episode 1058: reward = -170.00, steps = 171
    23:01:53 [INFO] train episode 1059: reward = -155.00, steps = 156
    23:01:53 [INFO] train episode 1060: reward = -156.00, steps = 157
    23:01:53 [INFO] train episode 1061: reward = -171.00, steps = 172
    23:01:53 [INFO] train episode 1062: reward = -158.00, steps = 159
    23:01:53 [INFO] train episode 1063: reward = -172.00, steps = 173
    23:01:54 [INFO] train episode 1064: reward = -218.00, steps = 219
    23:01:54 [INFO] train episode 1065: reward = -199.00, steps = 200
    23:01:54 [INFO] train episode 1066: reward = -149.00, steps = 150
    23:01:54 [INFO] train episode 1067: reward = -140.00, steps = 141
    23:01:54 [INFO] train episode 1068: reward = -146.00, steps = 147
    23:01:54 [INFO] train episode 1069: reward = -177.00, steps = 178
    23:01:55 [INFO] train episode 1070: reward = -198.00, steps = 199
    23:01:55 [INFO] train episode 1071: reward = -167.00, steps = 168
    23:01:55 [INFO] train episode 1072: reward = -149.00, steps = 150
    23:01:55 [INFO] train episode 1073: reward = -170.00, steps = 171
    23:01:55 [INFO] train episode 1074: reward = -139.00, steps = 140
    23:01:56 [INFO] train episode 1075: reward = -182.00, steps = 183
    23:01:56 [INFO] train episode 1076: reward = -193.00, steps = 194
    23:01:56 [INFO] train episode 1077: reward = -215.00, steps = 216
    23:01:56 [INFO] train episode 1078: reward = -203.00, steps = 204
    23:01:56 [INFO] train episode 1079: reward = -182.00, steps = 183
    23:01:57 [INFO] train episode 1080: reward = -209.00, steps = 210
    23:01:57 [INFO] train episode 1081: reward = -183.00, steps = 184
    23:01:57 [INFO] train episode 1082: reward = -177.00, steps = 178
    23:01:57 [INFO] train episode 1083: reward = -163.00, steps = 164
    23:01:57 [INFO] train episode 1084: reward = -183.00, steps = 184
    23:01:57 [INFO] train episode 1085: reward = -183.00, steps = 184
    23:01:58 [INFO] train episode 1086: reward = -207.00, steps = 208
    23:01:58 [INFO] train episode 1087: reward = -181.00, steps = 182
    23:01:58 [INFO] train episode 1088: reward = -186.00, steps = 187
    23:01:58 [INFO] train episode 1089: reward = -219.00, steps = 220
    23:01:58 [INFO] train episode 1090: reward = -148.00, steps = 149
    23:01:59 [INFO] train episode 1091: reward = -156.00, steps = 157
    23:01:59 [INFO] train episode 1092: reward = -174.00, steps = 175
    23:01:59 [INFO] train episode 1093: reward = -191.00, steps = 192
    23:01:59 [INFO] train episode 1094: reward = -208.00, steps = 209
    23:01:59 [INFO] train episode 1095: reward = -317.00, steps = 318
    23:02:00 [INFO] train episode 1096: reward = -224.00, steps = 225
    23:02:00 [INFO] train episode 1097: reward = -151.00, steps = 152
    23:02:00 [INFO] train episode 1098: reward = -208.00, steps = 209
    23:02:00 [INFO] train episode 1099: reward = -163.00, steps = 164
    23:02:00 [INFO] train episode 1100: reward = -161.00, steps = 162
    23:02:01 [INFO] train episode 1101: reward = -164.00, steps = 165
    23:02:01 [INFO] train episode 1102: reward = -188.00, steps = 189
    23:02:01 [INFO] train episode 1103: reward = -235.00, steps = 236
    23:02:01 [INFO] train episode 1104: reward = -211.00, steps = 212
    23:02:01 [INFO] train episode 1105: reward = -267.00, steps = 268
    23:02:02 [INFO] train episode 1106: reward = -166.00, steps = 167
    23:02:02 [INFO] train episode 1107: reward = -163.00, steps = 164
    23:02:02 [INFO] train episode 1108: reward = -150.00, steps = 151
    23:02:02 [INFO] train episode 1109: reward = -228.00, steps = 229
    23:02:02 [INFO] train episode 1110: reward = -222.00, steps = 223
    23:02:03 [INFO] train episode 1111: reward = -186.00, steps = 187
    23:02:03 [INFO] train episode 1112: reward = -237.00, steps = 238
    23:02:03 [INFO] train episode 1113: reward = -207.00, steps = 208
    23:02:03 [INFO] train episode 1114: reward = -183.00, steps = 184
    23:02:03 [INFO] train episode 1115: reward = -152.00, steps = 153
    23:02:04 [INFO] train episode 1116: reward = -128.00, steps = 129
    23:02:04 [INFO] train episode 1117: reward = -277.00, steps = 278
    23:02:04 [INFO] train episode 1118: reward = -211.00, steps = 212
    23:02:04 [INFO] train episode 1119: reward = -144.00, steps = 145
    23:02:04 [INFO] train episode 1120: reward = -156.00, steps = 157
    23:02:04 [INFO] train episode 1121: reward = -186.00, steps = 187
    23:02:05 [INFO] train episode 1122: reward = -162.00, steps = 163
    23:02:05 [INFO] train episode 1123: reward = -141.00, steps = 142
    23:02:05 [INFO] train episode 1124: reward = -185.00, steps = 186
    23:02:05 [INFO] train episode 1125: reward = -177.00, steps = 178
    23:02:05 [INFO] train episode 1126: reward = -186.00, steps = 187
    23:02:06 [INFO] train episode 1127: reward = -168.00, steps = 169
    23:02:06 [INFO] train episode 1128: reward = -164.00, steps = 165
    23:02:06 [INFO] train episode 1129: reward = -172.00, steps = 173
    23:02:06 [INFO] train episode 1130: reward = -155.00, steps = 156
    23:02:06 [INFO] train episode 1131: reward = -176.00, steps = 177
    23:02:06 [INFO] train episode 1132: reward = -152.00, steps = 153
    23:02:07 [INFO] train episode 1133: reward = -175.00, steps = 176
    23:02:07 [INFO] train episode 1134: reward = -173.00, steps = 174
    23:02:07 [INFO] train episode 1135: reward = -217.00, steps = 218
    23:02:07 [INFO] train episode 1136: reward = -146.00, steps = 147
    23:02:07 [INFO] train episode 1137: reward = -170.00, steps = 171
    23:02:08 [INFO] train episode 1138: reward = -179.00, steps = 180
    23:02:08 [INFO] train episode 1139: reward = -140.00, steps = 141
    23:02:08 [INFO] train episode 1140: reward = -151.00, steps = 152
    23:02:08 [INFO] train episode 1141: reward = -179.00, steps = 180
    23:02:08 [INFO] train episode 1142: reward = -142.00, steps = 143
    23:02:08 [INFO] train episode 1143: reward = -192.00, steps = 193
    23:02:09 [INFO] train episode 1144: reward = -247.00, steps = 248
    23:02:09 [INFO] train episode 1145: reward = -147.00, steps = 148
    23:02:09 [INFO] train episode 1146: reward = -125.00, steps = 126
    23:02:09 [INFO] train episode 1147: reward = -141.00, steps = 142
    23:02:09 [INFO] train episode 1148: reward = -129.00, steps = 130
    23:02:09 [INFO] train episode 1149: reward = -192.00, steps = 193
    23:02:10 [INFO] train episode 1150: reward = -251.00, steps = 252
    23:02:10 [INFO] train episode 1151: reward = -183.00, steps = 184
    23:02:10 [INFO] train episode 1152: reward = -257.00, steps = 258
    23:02:10 [INFO] train episode 1153: reward = -202.00, steps = 203
    23:02:10 [INFO] train episode 1154: reward = -142.00, steps = 143
    23:02:11 [INFO] train episode 1155: reward = -127.00, steps = 128
    23:02:11 [INFO] train episode 1156: reward = -150.00, steps = 151
    23:02:11 [INFO] train episode 1157: reward = -170.00, steps = 171
    23:02:11 [INFO] train episode 1158: reward = -179.00, steps = 180
    23:02:11 [INFO] train episode 1159: reward = -146.00, steps = 147
    23:02:11 [INFO] train episode 1160: reward = -117.00, steps = 118
    23:02:12 [INFO] train episode 1161: reward = -189.00, steps = 190
    23:02:12 [INFO] train episode 1162: reward = -358.00, steps = 359
    23:02:12 [INFO] train episode 1163: reward = -131.00, steps = 132
    23:02:12 [INFO] train episode 1164: reward = -171.00, steps = 172
    23:02:12 [INFO] train episode 1165: reward = -134.00, steps = 135
    23:02:13 [INFO] train episode 1166: reward = -189.00, steps = 190
    23:02:13 [INFO] train episode 1167: reward = -148.00, steps = 149
    23:02:13 [INFO] train episode 1168: reward = -178.00, steps = 179
    23:02:13 [INFO] train episode 1169: reward = -240.00, steps = 241
    23:02:13 [INFO] train episode 1170: reward = -118.00, steps = 119
    23:02:14 [INFO] train episode 1171: reward = -326.00, steps = 327
    23:02:14 [INFO] train episode 1172: reward = -153.00, steps = 154
    23:02:14 [INFO] train episode 1173: reward = -173.00, steps = 174
    23:02:14 [INFO] train episode 1174: reward = -137.00, steps = 138
    23:02:15 [INFO] train episode 1175: reward = -393.00, steps = 394
    23:02:15 [INFO] train episode 1176: reward = -168.00, steps = 169
    23:02:15 [INFO] train episode 1177: reward = -258.00, steps = 259
    23:02:15 [INFO] train episode 1178: reward = -195.00, steps = 196
    23:02:16 [INFO] train episode 1179: reward = -152.00, steps = 153
    23:02:16 [INFO] train episode 1180: reward = -232.00, steps = 233
    23:02:16 [INFO] train episode 1181: reward = -156.00, steps = 157
    23:02:16 [INFO] train episode 1182: reward = -161.00, steps = 162
    23:02:16 [INFO] train episode 1183: reward = -137.00, steps = 138
    23:02:16 [INFO] train episode 1184: reward = -168.00, steps = 169
    23:02:17 [INFO] train episode 1185: reward = -147.00, steps = 148
    23:02:17 [INFO] train episode 1186: reward = -232.00, steps = 233
    23:02:17 [INFO] train episode 1187: reward = -173.00, steps = 174
    23:02:17 [INFO] train episode 1188: reward = -150.00, steps = 151
    23:02:17 [INFO] train episode 1189: reward = -194.00, steps = 195
    23:02:17 [INFO] train episode 1190: reward = -147.00, steps = 148
    23:02:18 [INFO] train episode 1191: reward = -200.00, steps = 201
    23:02:18 [INFO] train episode 1192: reward = -202.00, steps = 203
    23:02:18 [INFO] train episode 1193: reward = -304.00, steps = 305
    23:02:18 [INFO] train episode 1194: reward = -116.00, steps = 117
    23:02:19 [INFO] train episode 1195: reward = -198.00, steps = 199
    23:02:19 [INFO] train episode 1196: reward = -152.00, steps = 153
    23:02:19 [INFO] train episode 1197: reward = -211.00, steps = 212
    23:02:19 [INFO] train episode 1198: reward = -118.00, steps = 119
    23:02:19 [INFO] train episode 1199: reward = -174.00, steps = 175
    23:02:19 [INFO] train episode 1200: reward = -157.00, steps = 158
    23:02:20 [INFO] train episode 1201: reward = -123.00, steps = 124
    23:02:20 [INFO] train episode 1202: reward = -145.00, steps = 146
    23:02:20 [INFO] train episode 1203: reward = -225.00, steps = 226
    23:02:20 [INFO] train episode 1204: reward = -152.00, steps = 153
    23:02:20 [INFO] train episode 1205: reward = -167.00, steps = 168
    23:02:20 [INFO] train episode 1206: reward = -149.00, steps = 150
    23:02:21 [INFO] train episode 1207: reward = -151.00, steps = 152
    23:02:21 [INFO] train episode 1208: reward = -150.00, steps = 151
    23:02:21 [INFO] train episode 1209: reward = -196.00, steps = 197
    23:02:21 [INFO] train episode 1210: reward = -248.00, steps = 249
    23:02:21 [INFO] train episode 1211: reward = -147.00, steps = 148
    23:02:21 [INFO] train episode 1212: reward = -277.00, steps = 278
    23:02:22 [INFO] train episode 1213: reward = -204.00, steps = 205
    23:02:22 [INFO] train episode 1214: reward = -143.00, steps = 144
    23:02:22 [INFO] train episode 1215: reward = -159.00, steps = 160
    23:02:22 [INFO] train episode 1216: reward = -216.00, steps = 217
    23:02:22 [INFO] train episode 1217: reward = -165.00, steps = 166
    23:02:23 [INFO] train episode 1218: reward = -147.00, steps = 148
    23:02:23 [INFO] train episode 1219: reward = -149.00, steps = 150
    23:02:23 [INFO] train episode 1220: reward = -126.00, steps = 127
    23:02:23 [INFO] train episode 1221: reward = -164.00, steps = 165
    23:02:23 [INFO] train episode 1222: reward = -127.00, steps = 128
    23:02:23 [INFO] train episode 1223: reward = -146.00, steps = 147
    23:02:23 [INFO] train episode 1224: reward = -169.00, steps = 170
    23:02:24 [INFO] train episode 1225: reward = -137.00, steps = 138
    23:02:24 [INFO] train episode 1226: reward = -145.00, steps = 146
    23:02:24 [INFO] train episode 1227: reward = -213.00, steps = 214
    23:02:24 [INFO] train episode 1228: reward = -215.00, steps = 216
    23:02:24 [INFO] train episode 1229: reward = -158.00, steps = 159
    23:02:25 [INFO] train episode 1230: reward = -158.00, steps = 159
    23:02:25 [INFO] train episode 1231: reward = -176.00, steps = 177
    23:02:25 [INFO] train episode 1232: reward = -146.00, steps = 147
    23:02:25 [INFO] train episode 1233: reward = -153.00, steps = 154
    23:02:25 [INFO] train episode 1234: reward = -266.00, steps = 267
    23:02:25 [INFO] train episode 1235: reward = -177.00, steps = 178
    23:02:26 [INFO] train episode 1236: reward = -152.00, steps = 153
    23:02:26 [INFO] train episode 1237: reward = -171.00, steps = 172
    23:02:26 [INFO] train episode 1238: reward = -159.00, steps = 160
    23:02:26 [INFO] train episode 1239: reward = -153.00, steps = 154
    23:02:26 [INFO] train episode 1240: reward = -168.00, steps = 169
    23:02:26 [INFO] train episode 1241: reward = -161.00, steps = 162
    23:02:27 [INFO] train episode 1242: reward = -161.00, steps = 162
    23:02:27 [INFO] train episode 1243: reward = -125.00, steps = 126
    23:02:27 [INFO] train episode 1244: reward = -157.00, steps = 158
    23:02:27 [INFO] train episode 1245: reward = -169.00, steps = 170
    23:02:27 [INFO] train episode 1246: reward = -126.00, steps = 127
    23:02:27 [INFO] train episode 1247: reward = -183.00, steps = 184
    23:02:28 [INFO] train episode 1248: reward = -171.00, steps = 172
    23:02:28 [INFO] train episode 1249: reward = -140.00, steps = 141
    23:02:28 [INFO] train episode 1250: reward = -166.00, steps = 167
    23:02:28 [INFO] train episode 1251: reward = -192.00, steps = 193
    23:02:28 [INFO] train episode 1252: reward = -277.00, steps = 278
    23:02:29 [INFO] train episode 1253: reward = -179.00, steps = 180
    23:02:29 [INFO] train episode 1254: reward = -123.00, steps = 124
    23:02:29 [INFO] train episode 1255: reward = -177.00, steps = 178
    23:02:29 [INFO] train episode 1256: reward = -189.00, steps = 190
    23:02:29 [INFO] train episode 1257: reward = -171.00, steps = 172
    23:02:29 [INFO] train episode 1258: reward = -164.00, steps = 165
    23:02:30 [INFO] train episode 1259: reward = -227.00, steps = 228
    23:02:30 [INFO] train episode 1260: reward = -134.00, steps = 135
    23:02:30 [INFO] train episode 1261: reward = -184.00, steps = 185
    23:02:30 [INFO] train episode 1262: reward = -150.00, steps = 151
    23:02:30 [INFO] train episode 1263: reward = -167.00, steps = 168
    23:02:31 [INFO] train episode 1264: reward = -164.00, steps = 165
    23:02:31 [INFO] train episode 1265: reward = -167.00, steps = 168
    23:02:31 [INFO] train episode 1266: reward = -195.00, steps = 196
    23:02:31 [INFO] train episode 1267: reward = -154.00, steps = 155
    23:02:31 [INFO] train episode 1268: reward = -166.00, steps = 167
    23:02:31 [INFO] train episode 1269: reward = -147.00, steps = 148
    23:02:32 [INFO] train episode 1270: reward = -214.00, steps = 215
    23:02:32 [INFO] train episode 1271: reward = -126.00, steps = 127
    23:02:32 [INFO] train episode 1272: reward = -153.00, steps = 154
    23:02:32 [INFO] train episode 1273: reward = -149.00, steps = 150
    23:02:32 [INFO] train episode 1274: reward = -142.00, steps = 143
    23:02:32 [INFO] train episode 1275: reward = -208.00, steps = 209
    23:02:33 [INFO] train episode 1276: reward = -231.00, steps = 232
    23:02:33 [INFO] train episode 1277: reward = -151.00, steps = 152
    23:02:33 [INFO] train episode 1278: reward = -155.00, steps = 156
    23:02:33 [INFO] train episode 1279: reward = -198.00, steps = 199
    23:02:33 [INFO] train episode 1280: reward = -125.00, steps = 126
    23:02:33 [INFO] train episode 1281: reward = -155.00, steps = 156
    23:02:34 [INFO] train episode 1282: reward = -152.00, steps = 153
    23:02:34 [INFO] train episode 1283: reward = -154.00, steps = 155
    23:02:34 [INFO] train episode 1284: reward = -137.00, steps = 138
    23:02:34 [INFO] train episode 1285: reward = -243.00, steps = 244
    23:02:34 [INFO] train episode 1286: reward = -112.00, steps = 113
    23:02:34 [INFO] train episode 1287: reward = -162.00, steps = 163
    23:02:35 [INFO] train episode 1288: reward = -149.00, steps = 150
    23:02:35 [INFO] train episode 1289: reward = -124.00, steps = 125
    23:02:35 [INFO] train episode 1290: reward = -172.00, steps = 173
    23:02:35 [INFO] train episode 1291: reward = -183.00, steps = 184
    23:02:35 [INFO] train episode 1292: reward = -185.00, steps = 186
    23:02:35 [INFO] train episode 1293: reward = -147.00, steps = 148
    23:02:35 [INFO] train episode 1294: reward = -162.00, steps = 163
    23:02:36 [INFO] train episode 1295: reward = -183.00, steps = 184
    23:02:36 [INFO] train episode 1296: reward = -177.00, steps = 178
    23:02:36 [INFO] train episode 1297: reward = -148.00, steps = 149
    23:02:36 [INFO] train episode 1298: reward = -127.00, steps = 128
    23:02:36 [INFO] train episode 1299: reward = -191.00, steps = 192
    23:02:37 [INFO] train episode 1300: reward = -224.00, steps = 225
    23:02:37 [INFO] train episode 1301: reward = -172.00, steps = 173
    23:02:37 [INFO] train episode 1302: reward = -154.00, steps = 155
    23:02:37 [INFO] train episode 1303: reward = -158.00, steps = 159
    23:02:37 [INFO] train episode 1304: reward = -157.00, steps = 158
    23:02:37 [INFO] train episode 1305: reward = -153.00, steps = 154
    23:02:38 [INFO] train episode 1306: reward = -143.00, steps = 144
    23:02:38 [INFO] train episode 1307: reward = -179.00, steps = 180
    23:02:38 [INFO] train episode 1308: reward = -143.00, steps = 144
    23:02:38 [INFO] train episode 1309: reward = -143.00, steps = 144
    23:02:38 [INFO] train episode 1310: reward = -149.00, steps = 150
    23:02:38 [INFO] train episode 1311: reward = -142.00, steps = 143
    23:02:38 [INFO] train episode 1312: reward = -165.00, steps = 166
    23:02:39 [INFO] train episode 1313: reward = -158.00, steps = 159
    23:02:39 [INFO] train episode 1314: reward = -194.00, steps = 195
    23:02:39 [INFO] train episode 1315: reward = -137.00, steps = 138
    23:02:39 [INFO] train episode 1316: reward = -146.00, steps = 147
    23:02:39 [INFO] train episode 1317: reward = -161.00, steps = 162
    23:02:39 [INFO] train episode 1318: reward = -149.00, steps = 150
    23:02:40 [INFO] train episode 1319: reward = -168.00, steps = 169
    23:02:40 [INFO] train episode 1320: reward = -268.00, steps = 269
    23:02:40 [INFO] train episode 1321: reward = -153.00, steps = 154
    23:02:40 [INFO] train episode 1322: reward = -147.00, steps = 148
    23:02:40 [INFO] train episode 1323: reward = -160.00, steps = 161
    23:02:40 [INFO] train episode 1324: reward = -143.00, steps = 144
    23:02:41 [INFO] train episode 1325: reward = -170.00, steps = 171
    23:02:41 [INFO] train episode 1326: reward = -161.00, steps = 162
    23:02:41 [INFO] train episode 1327: reward = -182.00, steps = 183
    23:02:41 [INFO] train episode 1328: reward = -212.00, steps = 213
    23:02:41 [INFO] train episode 1329: reward = -164.00, steps = 165
    23:02:41 [INFO] train episode 1330: reward = -142.00, steps = 143
    23:02:42 [INFO] train episode 1331: reward = -187.00, steps = 188
    23:02:42 [INFO] train episode 1332: reward = -139.00, steps = 140
    23:02:42 [INFO] train episode 1333: reward = -136.00, steps = 137
    23:02:42 [INFO] train episode 1334: reward = -116.00, steps = 117
    23:02:42 [INFO] train episode 1335: reward = -153.00, steps = 154
    23:02:42 [INFO] train episode 1336: reward = -149.00, steps = 150
    23:02:43 [INFO] train episode 1337: reward = -189.00, steps = 190
    23:02:43 [INFO] train episode 1338: reward = -185.00, steps = 186
    23:02:43 [INFO] train episode 1339: reward = -141.00, steps = 142
    23:02:43 [INFO] train episode 1340: reward = -124.00, steps = 125
    23:02:43 [INFO] train episode 1341: reward = -268.00, steps = 269
    23:02:44 [INFO] train episode 1342: reward = -136.00, steps = 137
    23:02:44 [INFO] train episode 1343: reward = -196.00, steps = 197
    23:02:44 [INFO] train episode 1344: reward = -217.00, steps = 218
    23:02:44 [INFO] train episode 1345: reward = -163.00, steps = 164
    23:02:44 [INFO] train episode 1346: reward = -136.00, steps = 137
    23:02:45 [INFO] train episode 1347: reward = -146.00, steps = 147
    23:02:45 [INFO] train episode 1348: reward = -136.00, steps = 137
    23:02:45 [INFO] train episode 1349: reward = -158.00, steps = 159
    23:02:45 [INFO] train episode 1350: reward = -216.00, steps = 217
    23:02:45 [INFO] train episode 1351: reward = -187.00, steps = 188
    23:02:45 [INFO] train episode 1352: reward = -126.00, steps = 127
    23:02:45 [INFO] train episode 1353: reward = -142.00, steps = 143
    23:02:46 [INFO] train episode 1354: reward = -117.00, steps = 118
    23:02:46 [INFO] train episode 1355: reward = -113.00, steps = 114
    23:02:46 [INFO] train episode 1356: reward = -162.00, steps = 163
    23:02:46 [INFO] train episode 1357: reward = -128.00, steps = 129
    23:02:46 [INFO] train episode 1358: reward = -174.00, steps = 175
    23:02:46 [INFO] train episode 1359: reward = -138.00, steps = 139
    23:02:47 [INFO] train episode 1360: reward = -160.00, steps = 161
    23:02:47 [INFO] train episode 1361: reward = -134.00, steps = 135
    23:02:47 [INFO] train episode 1362: reward = -207.00, steps = 208
    23:02:47 [INFO] train episode 1363: reward = -143.00, steps = 144
    23:02:47 [INFO] train episode 1364: reward = -166.00, steps = 167
    23:02:47 [INFO] train episode 1365: reward = -221.00, steps = 222
    23:02:48 [INFO] train episode 1366: reward = -180.00, steps = 181
    23:02:48 [INFO] train episode 1367: reward = -151.00, steps = 152
    23:02:48 [INFO] train episode 1368: reward = -168.00, steps = 169
    23:02:48 [INFO] train episode 1369: reward = -142.00, steps = 143
    23:02:48 [INFO] train episode 1370: reward = -142.00, steps = 143
    23:02:48 [INFO] train episode 1371: reward = -168.00, steps = 169
    23:02:49 [INFO] train episode 1372: reward = -180.00, steps = 181
    23:02:49 [INFO] train episode 1373: reward = -133.00, steps = 134
    23:02:49 [INFO] train episode 1374: reward = -207.00, steps = 208
    23:02:49 [INFO] train episode 1375: reward = -178.00, steps = 179
    23:02:49 [INFO] train episode 1376: reward = -151.00, steps = 152
    23:02:49 [INFO] train episode 1377: reward = -115.00, steps = 116
    23:02:50 [INFO] train episode 1378: reward = -159.00, steps = 160
    23:02:50 [INFO] train episode 1379: reward = -150.00, steps = 151
    23:02:50 [INFO] train episode 1380: reward = -110.00, steps = 111
    23:02:50 [INFO] train episode 1381: reward = -117.00, steps = 118
    23:02:50 [INFO] train episode 1382: reward = -135.00, steps = 136
    23:02:50 [INFO] train episode 1383: reward = -180.00, steps = 181
    23:02:50 [INFO] train episode 1384: reward = -136.00, steps = 137
    23:02:51 [INFO] train episode 1385: reward = -147.00, steps = 148
    23:02:51 [INFO] train episode 1386: reward = -108.00, steps = 109
    23:02:51 [INFO] train episode 1387: reward = -123.00, steps = 124
    23:02:51 [INFO] train episode 1388: reward = -153.00, steps = 154
    23:02:51 [INFO] train episode 1389: reward = -153.00, steps = 154
    23:02:51 [INFO] train episode 1390: reward = -132.00, steps = 133
    23:02:52 [INFO] train episode 1391: reward = -173.00, steps = 174
    23:02:52 [INFO] train episode 1392: reward = -141.00, steps = 142
    23:02:52 [INFO] train episode 1393: reward = -175.00, steps = 176
    23:02:52 [INFO] train episode 1394: reward = -228.00, steps = 229
    23:02:52 [INFO] train episode 1395: reward = -158.00, steps = 159
    23:02:52 [INFO] train episode 1396: reward = -150.00, steps = 151
    23:02:53 [INFO] train episode 1397: reward = -225.00, steps = 226
    23:02:53 [INFO] train episode 1398: reward = -180.00, steps = 181
    23:02:53 [INFO] train episode 1399: reward = -172.00, steps = 173
    23:02:53 [INFO] train episode 1400: reward = -176.00, steps = 177
    23:02:53 [INFO] train episode 1401: reward = -152.00, steps = 153
    23:02:54 [INFO] train episode 1402: reward = -127.00, steps = 128
    23:02:54 [INFO] train episode 1403: reward = -224.00, steps = 225
    23:02:54 [INFO] train episode 1404: reward = -131.00, steps = 132
    23:02:54 [INFO] train episode 1405: reward = -160.00, steps = 161
    23:02:54 [INFO] train episode 1406: reward = -178.00, steps = 179
    23:02:54 [INFO] train episode 1407: reward = -152.00, steps = 153
    23:02:55 [INFO] train episode 1408: reward = -121.00, steps = 122
    23:02:55 [INFO] train episode 1409: reward = -177.00, steps = 178
    23:02:55 [INFO] train episode 1410: reward = -155.00, steps = 156
    23:02:55 [INFO] train episode 1411: reward = -140.00, steps = 141
    23:02:55 [INFO] train episode 1412: reward = -184.00, steps = 185
    23:02:56 [INFO] train episode 1413: reward = -247.00, steps = 248
    23:02:56 [INFO] train episode 1414: reward = -170.00, steps = 171
    23:02:56 [INFO] train episode 1415: reward = -158.00, steps = 159
    23:02:56 [INFO] train episode 1416: reward = -145.00, steps = 146
    23:02:56 [INFO] train episode 1417: reward = -275.00, steps = 276
    23:02:56 [INFO] train episode 1418: reward = -216.00, steps = 217
    23:02:57 [INFO] train episode 1419: reward = -119.00, steps = 120
    23:02:57 [INFO] train episode 1420: reward = -166.00, steps = 167
    23:02:57 [INFO] train episode 1421: reward = -133.00, steps = 134
    23:02:57 [INFO] train episode 1422: reward = -164.00, steps = 165
    23:02:57 [INFO] train episode 1423: reward = -158.00, steps = 159
    23:02:57 [INFO] train episode 1424: reward = -162.00, steps = 163
    23:02:58 [INFO] train episode 1425: reward = -128.00, steps = 129
    23:02:58 [INFO] train episode 1426: reward = -161.00, steps = 162
    23:02:58 [INFO] train episode 1427: reward = -167.00, steps = 168
    23:02:58 [INFO] train episode 1428: reward = -166.00, steps = 167
    23:02:58 [INFO] train episode 1429: reward = -139.00, steps = 140
    23:02:59 [INFO] train episode 1430: reward = -130.00, steps = 131
    23:02:59 [INFO] train episode 1431: reward = -149.00, steps = 150
    23:02:59 [INFO] train episode 1432: reward = -144.00, steps = 145
    23:02:59 [INFO] train episode 1433: reward = -163.00, steps = 164
    23:02:59 [INFO] train episode 1434: reward = -130.00, steps = 131
    23:02:59 [INFO] train episode 1435: reward = -157.00, steps = 158
    23:03:00 [INFO] train episode 1436: reward = -166.00, steps = 167
    23:03:00 [INFO] train episode 1437: reward = -219.00, steps = 220
    23:03:00 [INFO] train episode 1438: reward = -157.00, steps = 158
    23:03:00 [INFO] train episode 1439: reward = -147.00, steps = 148
    23:03:00 [INFO] train episode 1440: reward = -145.00, steps = 146
    23:03:00 [INFO] train episode 1441: reward = -125.00, steps = 126
    23:03:01 [INFO] train episode 1442: reward = -145.00, steps = 146
    23:03:01 [INFO] train episode 1443: reward = -168.00, steps = 169
    23:03:01 [INFO] train episode 1444: reward = -134.00, steps = 135
    23:03:01 [INFO] train episode 1445: reward = -170.00, steps = 171
    23:03:01 [INFO] train episode 1446: reward = -133.00, steps = 134
    23:03:01 [INFO] train episode 1447: reward = -144.00, steps = 145
    23:03:02 [INFO] train episode 1448: reward = -164.00, steps = 165
    23:03:02 [INFO] train episode 1449: reward = -226.00, steps = 227
    23:03:02 [INFO] train episode 1450: reward = -144.00, steps = 145
    23:03:02 [INFO] train episode 1451: reward = -209.00, steps = 210
    23:03:02 [INFO] train episode 1452: reward = -139.00, steps = 140
    23:03:03 [INFO] train episode 1453: reward = -146.00, steps = 147
    23:03:03 [INFO] train episode 1454: reward = -207.00, steps = 208
    23:03:03 [INFO] train episode 1455: reward = -176.00, steps = 177
    23:03:03 [INFO] train episode 1456: reward = -129.00, steps = 130
    23:03:03 [INFO] train episode 1457: reward = -120.00, steps = 121
    23:03:03 [INFO] train episode 1458: reward = -159.00, steps = 160
    23:03:03 [INFO] train episode 1459: reward = -137.00, steps = 138
    23:03:04 [INFO] train episode 1460: reward = -166.00, steps = 167
    23:03:04 [INFO] train episode 1461: reward = -148.00, steps = 149
    23:03:04 [INFO] train episode 1462: reward = -138.00, steps = 139
    23:03:04 [INFO] train episode 1463: reward = -231.00, steps = 232
    23:03:04 [INFO] train episode 1464: reward = -154.00, steps = 155
    23:03:05 [INFO] train episode 1465: reward = -155.00, steps = 156
    23:03:05 [INFO] train episode 1466: reward = -158.00, steps = 159
    23:03:05 [INFO] train episode 1467: reward = -153.00, steps = 154
    23:03:05 [INFO] train episode 1468: reward = -128.00, steps = 129
    23:03:05 [INFO] train episode 1469: reward = -150.00, steps = 151
    23:03:05 [INFO] train episode 1470: reward = -140.00, steps = 141
    23:03:05 [INFO] train episode 1471: reward = -152.00, steps = 153
    23:03:06 [INFO] train episode 1472: reward = -158.00, steps = 159
    23:03:06 [INFO] train episode 1473: reward = -233.00, steps = 234
    23:03:06 [INFO] train episode 1474: reward = -246.00, steps = 247
    23:03:06 [INFO] train episode 1475: reward = -149.00, steps = 150
    23:03:06 [INFO] train episode 1476: reward = -136.00, steps = 137
    23:03:06 [INFO] train episode 1477: reward = -142.00, steps = 143
    23:03:07 [INFO] train episode 1478: reward = -176.00, steps = 177
    23:03:07 [INFO] train episode 1479: reward = -134.00, steps = 135
    23:03:07 [INFO] train episode 1480: reward = -223.00, steps = 224
    23:03:07 [INFO] train episode 1481: reward = -189.00, steps = 190
    23:03:07 [INFO] train episode 1482: reward = -157.00, steps = 158
    23:03:08 [INFO] train episode 1483: reward = -140.00, steps = 141
    23:03:08 [INFO] train episode 1484: reward = -237.00, steps = 238
    23:03:08 [INFO] train episode 1485: reward = -149.00, steps = 150
    23:03:08 [INFO] train episode 1486: reward = -162.00, steps = 163
    23:03:08 [INFO] train episode 1487: reward = -179.00, steps = 180
    23:03:08 [INFO] train episode 1488: reward = -142.00, steps = 143
    23:03:09 [INFO] train episode 1489: reward = -154.00, steps = 155
    23:03:09 [INFO] train episode 1490: reward = -135.00, steps = 136
    23:03:09 [INFO] train episode 1491: reward = -165.00, steps = 166
    23:03:09 [INFO] train episode 1492: reward = -135.00, steps = 136
    23:03:09 [INFO] train episode 1493: reward = -152.00, steps = 153
    23:03:09 [INFO] train episode 1494: reward = -149.00, steps = 150
    23:03:10 [INFO] train episode 1495: reward = -162.00, steps = 163
    23:03:10 [INFO] train episode 1496: reward = -152.00, steps = 153
    23:03:10 [INFO] train episode 1497: reward = -160.00, steps = 161
    23:03:10 [INFO] train episode 1498: reward = -143.00, steps = 144
    23:03:10 [INFO] train episode 1499: reward = -171.00, steps = 172
    23:03:10 [INFO] train episode 1500: reward = -141.00, steps = 142
    23:03:11 [INFO] train episode 1501: reward = -174.00, steps = 175
    23:03:11 [INFO] train episode 1502: reward = -168.00, steps = 169
    23:03:11 [INFO] train episode 1503: reward = -149.00, steps = 150
    23:03:11 [INFO] train episode 1504: reward = -141.00, steps = 142
    23:03:11 [INFO] train episode 1505: reward = -108.00, steps = 109
    23:03:11 [INFO] train episode 1506: reward = -119.00, steps = 120
    23:03:11 [INFO] train episode 1507: reward = -141.00, steps = 142
    23:03:12 [INFO] train episode 1508: reward = -158.00, steps = 159
    23:03:12 [INFO] train episode 1509: reward = -138.00, steps = 139
    23:03:12 [INFO] train episode 1510: reward = -148.00, steps = 149
    23:03:12 [INFO] train episode 1511: reward = -138.00, steps = 139
    23:03:12 [INFO] train episode 1512: reward = -142.00, steps = 143
    23:03:12 [INFO] train episode 1513: reward = -154.00, steps = 155
    23:03:12 [INFO] train episode 1514: reward = -143.00, steps = 144
    23:03:13 [INFO] train episode 1515: reward = -145.00, steps = 146
    23:03:13 [INFO] train episode 1516: reward = -111.00, steps = 112
    23:03:13 [INFO] train episode 1517: reward = -144.00, steps = 145
    23:03:13 [INFO] train episode 1518: reward = -148.00, steps = 149
    23:03:13 [INFO] train episode 1519: reward = -119.00, steps = 120
    23:03:13 [INFO] train episode 1520: reward = -133.00, steps = 134
    23:03:14 [INFO] train episode 1521: reward = -179.00, steps = 180
    23:03:14 [INFO] train episode 1522: reward = -147.00, steps = 148
    23:03:14 [INFO] train episode 1523: reward = -197.00, steps = 198
    23:03:14 [INFO] train episode 1524: reward = -146.00, steps = 147
    23:03:14 [INFO] train episode 1525: reward = -143.00, steps = 144
    23:03:14 [INFO] train episode 1526: reward = -169.00, steps = 170
    23:03:15 [INFO] train episode 1527: reward = -155.00, steps = 156
    23:03:15 [INFO] train episode 1528: reward = -157.00, steps = 158
    23:03:15 [INFO] train episode 1529: reward = -153.00, steps = 154
    23:03:15 [INFO] train episode 1530: reward = -184.00, steps = 185
    23:03:15 [INFO] train episode 1531: reward = -121.00, steps = 122
    23:03:15 [INFO] train episode 1532: reward = -174.00, steps = 175
    23:03:16 [INFO] train episode 1533: reward = -148.00, steps = 149
    23:03:16 [INFO] train episode 1534: reward = -185.00, steps = 186
    23:03:16 [INFO] train episode 1535: reward = -150.00, steps = 151
    23:03:16 [INFO] train episode 1536: reward = -142.00, steps = 143
    23:03:16 [INFO] train episode 1537: reward = -153.00, steps = 154
    23:03:16 [INFO] train episode 1538: reward = -119.00, steps = 120
    23:03:17 [INFO] train episode 1539: reward = -145.00, steps = 146
    23:03:17 [INFO] train episode 1540: reward = -163.00, steps = 164
    23:03:17 [INFO] train episode 1541: reward = -166.00, steps = 167
    23:03:17 [INFO] train episode 1542: reward = -146.00, steps = 147
    23:03:17 [INFO] train episode 1543: reward = -140.00, steps = 141
    23:03:17 [INFO] train episode 1544: reward = -170.00, steps = 171
    23:03:18 [INFO] train episode 1545: reward = -115.00, steps = 116
    23:03:18 [INFO] train episode 1546: reward = -148.00, steps = 149
    23:03:18 [INFO] train episode 1547: reward = -198.00, steps = 199
    23:03:18 [INFO] train episode 1548: reward = -162.00, steps = 163
    23:03:18 [INFO] train episode 1549: reward = -185.00, steps = 186
    23:03:18 [INFO] train episode 1550: reward = -172.00, steps = 173
    23:03:19 [INFO] train episode 1551: reward = -159.00, steps = 160
    23:03:19 [INFO] train episode 1552: reward = -197.00, steps = 198
    23:03:19 [INFO] train episode 1553: reward = -126.00, steps = 127
    23:03:19 [INFO] train episode 1554: reward = -126.00, steps = 127
    23:03:19 [INFO] train episode 1555: reward = -312.00, steps = 313
    23:03:19 [INFO] train episode 1556: reward = -148.00, steps = 149
    23:03:20 [INFO] train episode 1557: reward = -133.00, steps = 134
    23:03:20 [INFO] train episode 1558: reward = -169.00, steps = 170
    23:03:20 [INFO] train episode 1559: reward = -134.00, steps = 135
    23:03:20 [INFO] train episode 1560: reward = -171.00, steps = 172
    23:03:20 [INFO] train episode 1561: reward = -162.00, steps = 163
    23:03:21 [INFO] train episode 1562: reward = -161.00, steps = 162
    23:03:21 [INFO] train episode 1563: reward = -222.00, steps = 223
    23:03:21 [INFO] train episode 1564: reward = -153.00, steps = 154
    23:03:21 [INFO] train episode 1565: reward = -147.00, steps = 148
    23:03:21 [INFO] train episode 1566: reward = -137.00, steps = 138
    23:03:21 [INFO] train episode 1567: reward = -128.00, steps = 129
    23:03:22 [INFO] train episode 1568: reward = -127.00, steps = 128
    23:03:22 [INFO] train episode 1569: reward = -134.00, steps = 135
    23:03:22 [INFO] train episode 1570: reward = -197.00, steps = 198
    23:03:22 [INFO] train episode 1571: reward = -174.00, steps = 175
    23:03:22 [INFO] train episode 1572: reward = -169.00, steps = 170
    23:03:22 [INFO] train episode 1573: reward = -166.00, steps = 167
    23:03:23 [INFO] train episode 1574: reward = -149.00, steps = 150
    23:03:23 [INFO] train episode 1575: reward = -120.00, steps = 121
    23:03:23 [INFO] train episode 1576: reward = -180.00, steps = 181
    23:03:23 [INFO] train episode 1577: reward = -202.00, steps = 203
    23:03:23 [INFO] train episode 1578: reward = -126.00, steps = 127
    23:03:23 [INFO] train episode 1579: reward = -161.00, steps = 162
    23:03:24 [INFO] train episode 1580: reward = -131.00, steps = 132
    23:03:24 [INFO] train episode 1581: reward = -133.00, steps = 134
    23:03:24 [INFO] train episode 1582: reward = -208.00, steps = 209
    23:03:24 [INFO] train episode 1583: reward = -141.00, steps = 142
    23:03:24 [INFO] train episode 1584: reward = -141.00, steps = 142
    23:03:24 [INFO] train episode 1585: reward = -110.00, steps = 111
    23:03:24 [INFO] train episode 1586: reward = -152.00, steps = 153
    23:03:25 [INFO] train episode 1587: reward = -111.00, steps = 112
    23:03:25 [INFO] train episode 1588: reward = -174.00, steps = 175
    23:03:25 [INFO] train episode 1589: reward = -140.00, steps = 141
    23:03:25 [INFO] train episode 1590: reward = -123.00, steps = 124
    23:03:25 [INFO] train episode 1591: reward = -141.00, steps = 142
    23:03:25 [INFO] train episode 1592: reward = -112.00, steps = 113
    23:03:25 [INFO] train episode 1593: reward = -140.00, steps = 141
    23:03:26 [INFO] train episode 1594: reward = -143.00, steps = 144
    23:03:26 [INFO] train episode 1595: reward = -154.00, steps = 155
    23:03:26 [INFO] train episode 1596: reward = -168.00, steps = 169
    23:03:26 [INFO] train episode 1597: reward = -160.00, steps = 161
    23:03:26 [INFO] train episode 1598: reward = -154.00, steps = 155
    23:03:26 [INFO] train episode 1599: reward = -131.00, steps = 132
    23:03:27 [INFO] train episode 1600: reward = -167.00, steps = 168
    23:03:27 [INFO] train episode 1601: reward = -145.00, steps = 146
    23:03:27 [INFO] train episode 1602: reward = -160.00, steps = 161
    23:03:27 [INFO] train episode 1603: reward = -124.00, steps = 125
    23:03:27 [INFO] train episode 1604: reward = -122.00, steps = 123
    23:03:27 [INFO] train episode 1605: reward = -165.00, steps = 166
    23:03:28 [INFO] train episode 1606: reward = -173.00, steps = 174
    23:03:28 [INFO] train episode 1607: reward = -155.00, steps = 156
    23:03:28 [INFO] train episode 1608: reward = -171.00, steps = 172
    23:03:28 [INFO] train episode 1609: reward = -134.00, steps = 135
    23:03:28 [INFO] train episode 1610: reward = -174.00, steps = 175
    23:03:28 [INFO] train episode 1611: reward = -170.00, steps = 171
    23:03:29 [INFO] train episode 1612: reward = -180.00, steps = 181
    23:03:29 [INFO] train episode 1613: reward = -177.00, steps = 178
    23:03:29 [INFO] train episode 1614: reward = -136.00, steps = 137
    23:03:29 [INFO] train episode 1615: reward = -125.00, steps = 126
    23:03:29 [INFO] train episode 1616: reward = -112.00, steps = 113
    23:03:29 [INFO] train episode 1617: reward = -128.00, steps = 129
    23:03:29 [INFO] train episode 1618: reward = -112.00, steps = 113
    23:03:30 [INFO] train episode 1619: reward = -123.00, steps = 124
    23:03:30 [INFO] train episode 1620: reward = -133.00, steps = 134
    23:03:30 [INFO] train episode 1621: reward = -242.00, steps = 243
    23:03:30 [INFO] train episode 1622: reward = -184.00, steps = 185
    23:03:30 [INFO] train episode 1623: reward = -170.00, steps = 171
    23:03:30 [INFO] train episode 1624: reward = -161.00, steps = 162
    23:03:31 [INFO] train episode 1625: reward = -132.00, steps = 133
    23:03:31 [INFO] train episode 1626: reward = -127.00, steps = 128
    23:03:31 [INFO] train episode 1627: reward = -137.00, steps = 138
    23:03:31 [INFO] train episode 1628: reward = -140.00, steps = 141
    23:03:31 [INFO] train episode 1629: reward = -115.00, steps = 116
    23:03:31 [INFO] train episode 1630: reward = -134.00, steps = 135
    23:03:31 [INFO] train episode 1631: reward = -118.00, steps = 119
    23:03:31 [INFO] train episode 1632: reward = -135.00, steps = 136
    23:03:32 [INFO] train episode 1633: reward = -140.00, steps = 141
    23:03:32 [INFO] train episode 1634: reward = -132.00, steps = 133
    23:03:32 [INFO] train episode 1635: reward = -136.00, steps = 137
    23:03:32 [INFO] train episode 1636: reward = -138.00, steps = 139
    23:03:32 [INFO] train episode 1637: reward = -142.00, steps = 143
    23:03:32 [INFO] train episode 1638: reward = -127.00, steps = 128
    23:03:32 [INFO] train episode 1639: reward = -121.00, steps = 122
    23:03:33 [INFO] train episode 1640: reward = -112.00, steps = 113
    23:03:33 [INFO] train episode 1641: reward = -152.00, steps = 153
    23:03:33 [INFO] train episode 1642: reward = -192.00, steps = 193
    23:03:33 [INFO] train episode 1643: reward = -162.00, steps = 163
    23:03:33 [INFO] train episode 1644: reward = -141.00, steps = 142
    23:03:33 [INFO] train episode 1645: reward = -161.00, steps = 162
    23:03:34 [INFO] train episode 1646: reward = -226.00, steps = 227
    23:03:34 [INFO] train episode 1647: reward = -141.00, steps = 142
    23:03:34 [INFO] train episode 1648: reward = -143.00, steps = 144
    23:03:34 [INFO] train episode 1649: reward = -117.00, steps = 118
    23:03:34 [INFO] train episode 1650: reward = -141.00, steps = 142
    23:03:34 [INFO] train episode 1651: reward = -112.00, steps = 113
    23:03:35 [INFO] train episode 1652: reward = -122.00, steps = 123
    23:03:35 [INFO] train episode 1653: reward = -152.00, steps = 153
    23:03:35 [INFO] train episode 1654: reward = -139.00, steps = 140
    23:03:35 [INFO] train episode 1655: reward = -259.00, steps = 260
    23:03:35 [INFO] train episode 1656: reward = -128.00, steps = 129
    23:03:35 [INFO] train episode 1657: reward = -149.00, steps = 150
    23:03:36 [INFO] train episode 1658: reward = -114.00, steps = 115
    23:03:36 [INFO] train episode 1659: reward = -138.00, steps = 139
    23:03:36 [INFO] train episode 1660: reward = -153.00, steps = 154
    23:03:36 [INFO] train episode 1661: reward = -127.00, steps = 128
    23:03:36 [INFO] train episode 1662: reward = -117.00, steps = 118
    23:03:36 [INFO] train episode 1663: reward = -133.00, steps = 134
    23:03:36 [INFO] train episode 1664: reward = -137.00, steps = 138
    23:03:37 [INFO] train episode 1665: reward = -155.00, steps = 156
    23:03:37 [INFO] train episode 1666: reward = -132.00, steps = 133
    23:03:37 [INFO] train episode 1667: reward = -146.00, steps = 147
    23:03:37 [INFO] train episode 1668: reward = -139.00, steps = 140
    23:03:37 [INFO] train episode 1669: reward = -219.00, steps = 220
    23:03:37 [INFO] train episode 1670: reward = -193.00, steps = 194
    23:03:37 [INFO] train episode 1671: reward = -143.00, steps = 144
    23:03:38 [INFO] train episode 1672: reward = -153.00, steps = 154
    23:03:38 [INFO] train episode 1673: reward = -130.00, steps = 131
    23:03:38 [INFO] train episode 1674: reward = -149.00, steps = 150
    23:03:38 [INFO] train episode 1675: reward = -189.00, steps = 190
    23:03:38 [INFO] train episode 1676: reward = -161.00, steps = 162
    23:03:38 [INFO] train episode 1677: reward = -145.00, steps = 146
    23:03:39 [INFO] train episode 1678: reward = -123.00, steps = 124
    23:03:39 [INFO] train episode 1679: reward = -168.00, steps = 169
    23:03:39 [INFO] train episode 1680: reward = -140.00, steps = 141
    23:03:39 [INFO] train episode 1681: reward = -137.00, steps = 138
    23:03:39 [INFO] train episode 1682: reward = -117.00, steps = 118
    23:03:39 [INFO] train episode 1683: reward = -167.00, steps = 168
    23:03:39 [INFO] train episode 1684: reward = -129.00, steps = 130
    23:03:40 [INFO] train episode 1685: reward = -168.00, steps = 169
    23:03:40 [INFO] train episode 1686: reward = -129.00, steps = 130
    23:03:40 [INFO] train episode 1687: reward = -151.00, steps = 152
    23:03:40 [INFO] train episode 1688: reward = -147.00, steps = 148
    23:03:40 [INFO] train episode 1689: reward = -104.00, steps = 105
    23:03:40 [INFO] train episode 1690: reward = -138.00, steps = 139
    23:03:40 [INFO] train episode 1691: reward = -193.00, steps = 194
    23:03:41 [INFO] train episode 1692: reward = -122.00, steps = 123
    23:03:41 [INFO] train episode 1693: reward = -155.00, steps = 156
    23:03:41 [INFO] train episode 1694: reward = -135.00, steps = 136
    23:03:41 [INFO] train episode 1695: reward = -154.00, steps = 155
    23:03:41 [INFO] train episode 1696: reward = -165.00, steps = 166
    23:03:41 [INFO] train episode 1697: reward = -128.00, steps = 129
    23:03:42 [INFO] train episode 1698: reward = -157.00, steps = 158
    23:03:42 [INFO] train episode 1699: reward = -155.00, steps = 156
    23:03:42 [INFO] train episode 1700: reward = -169.00, steps = 170
    23:03:42 [INFO] train episode 1701: reward = -216.00, steps = 217
    23:03:42 [INFO] train episode 1702: reward = -135.00, steps = 136
    23:03:42 [INFO] train episode 1703: reward = -130.00, steps = 131
    23:03:43 [INFO] train episode 1704: reward = -124.00, steps = 125
    23:03:43 [INFO] train episode 1705: reward = -128.00, steps = 129
    23:03:43 [INFO] train episode 1706: reward = -129.00, steps = 130
    23:03:43 [INFO] train episode 1707: reward = -148.00, steps = 149
    23:03:43 [INFO] train episode 1708: reward = -168.00, steps = 169
    23:03:43 [INFO] train episode 1709: reward = -158.00, steps = 159
    23:03:43 [INFO] train episode 1710: reward = -116.00, steps = 117
    23:03:44 [INFO] train episode 1711: reward = -140.00, steps = 141
    23:03:44 [INFO] train episode 1712: reward = -134.00, steps = 135
    23:03:44 [INFO] train episode 1713: reward = -118.00, steps = 119
    23:03:44 [INFO] train episode 1714: reward = -133.00, steps = 134
    23:03:44 [INFO] train episode 1715: reward = -182.00, steps = 183
    23:03:44 [INFO] train episode 1716: reward = -183.00, steps = 184
    23:03:45 [INFO] train episode 1717: reward = -185.00, steps = 186
    23:03:45 [INFO] train episode 1718: reward = -158.00, steps = 159
    23:03:45 [INFO] train episode 1719: reward = -136.00, steps = 137
    23:03:45 [INFO] train episode 1720: reward = -165.00, steps = 166
    23:03:45 [INFO] train episode 1721: reward = -133.00, steps = 134
    23:03:45 [INFO] train episode 1722: reward = -118.00, steps = 119
    23:03:46 [INFO] train episode 1723: reward = -136.00, steps = 137
    23:03:46 [INFO] train episode 1724: reward = -119.00, steps = 120
    23:03:46 [INFO] train episode 1725: reward = -121.00, steps = 122
    23:03:46 [INFO] train episode 1726: reward = -164.00, steps = 165
    23:03:46 [INFO] train episode 1727: reward = -129.00, steps = 130
    23:03:46 [INFO] train episode 1728: reward = -148.00, steps = 149
    23:03:47 [INFO] train episode 1729: reward = -165.00, steps = 166
    23:03:47 [INFO] train episode 1730: reward = -181.00, steps = 182
    23:03:47 [INFO] train episode 1731: reward = -171.00, steps = 172
    23:03:47 [INFO] train episode 1732: reward = -131.00, steps = 132
    23:03:47 [INFO] train episode 1733: reward = -151.00, steps = 152
    23:03:47 [INFO] train episode 1734: reward = -273.00, steps = 274
    23:03:48 [INFO] train episode 1735: reward = -120.00, steps = 121
    23:03:48 [INFO] train episode 1736: reward = -159.00, steps = 160
    23:03:48 [INFO] train episode 1737: reward = -140.00, steps = 141
    23:03:48 [INFO] train episode 1738: reward = -148.00, steps = 149
    23:03:48 [INFO] train episode 1739: reward = -118.00, steps = 119
    23:03:48 [INFO] train episode 1740: reward = -157.00, steps = 158
    23:03:48 [INFO] train episode 1741: reward = -131.00, steps = 132
    23:03:49 [INFO] train episode 1742: reward = -124.00, steps = 125
    23:03:49 [INFO] train episode 1743: reward = -103.00, steps = 104
    23:03:49 [INFO] train episode 1744: reward = -153.00, steps = 154
    23:03:49 [INFO] train episode 1745: reward = -209.00, steps = 210
    23:03:49 [INFO] train episode 1746: reward = -135.00, steps = 136
    23:03:49 [INFO] train episode 1747: reward = -127.00, steps = 128
    23:03:50 [INFO] train episode 1748: reward = -216.00, steps = 217
    23:03:50 [INFO] train episode 1749: reward = -141.00, steps = 142
    23:03:50 [INFO] train episode 1750: reward = -144.00, steps = 145
    23:03:50 [INFO] train episode 1751: reward = -118.00, steps = 119
    23:03:50 [INFO] train episode 1752: reward = -112.00, steps = 113
    23:03:50 [INFO] train episode 1753: reward = -112.00, steps = 113
    23:03:50 [INFO] train episode 1754: reward = -195.00, steps = 196
    23:03:51 [INFO] train episode 1755: reward = -124.00, steps = 125
    23:03:51 [INFO] train episode 1756: reward = -146.00, steps = 147
    23:03:51 [INFO] train episode 1757: reward = -119.00, steps = 120
    23:03:51 [INFO] train episode 1758: reward = -137.00, steps = 138
    23:03:51 [INFO] train episode 1759: reward = -119.00, steps = 120
    23:03:51 [INFO] train episode 1760: reward = -159.00, steps = 160
    23:03:51 [INFO] train episode 1761: reward = -135.00, steps = 136
    23:03:51 [INFO] train episode 1762: reward = -99.00, steps = 100
    23:03:52 [INFO] train episode 1763: reward = -141.00, steps = 142
    23:03:52 [INFO] train episode 1764: reward = -177.00, steps = 178
    23:03:52 [INFO] train episode 1765: reward = -130.00, steps = 131
    23:03:52 [INFO] train episode 1766: reward = -103.00, steps = 104
    23:03:52 [INFO] train episode 1767: reward = -117.00, steps = 118
    23:03:52 [INFO] train episode 1768: reward = -160.00, steps = 161
    23:03:52 [INFO] train episode 1769: reward = -128.00, steps = 129
    23:03:53 [INFO] train episode 1770: reward = -104.00, steps = 105
    23:03:53 [INFO] train episode 1771: reward = -176.00, steps = 177
    23:03:53 [INFO] train episode 1772: reward = -115.00, steps = 116
    23:03:53 [INFO] train episode 1773: reward = -115.00, steps = 116
    23:03:53 [INFO] train episode 1774: reward = -140.00, steps = 141
    23:03:53 [INFO] train episode 1775: reward = -129.00, steps = 130
    23:03:53 [INFO] train episode 1776: reward = -120.00, steps = 121
    23:03:54 [INFO] train episode 1777: reward = -120.00, steps = 121
    23:03:54 [INFO] train episode 1778: reward = -112.00, steps = 113
    23:03:54 [INFO] train episode 1779: reward = -116.00, steps = 117
    23:03:54 [INFO] train episode 1780: reward = -120.00, steps = 121
    23:03:54 [INFO] train episode 1781: reward = -205.00, steps = 206
    23:03:54 [INFO] train episode 1782: reward = -112.00, steps = 113
    23:03:54 [INFO] train episode 1783: reward = -128.00, steps = 129
    23:03:55 [INFO] train episode 1784: reward = -137.00, steps = 138
    23:03:55 [INFO] train episode 1785: reward = -121.00, steps = 122
    23:03:55 [INFO] train episode 1786: reward = -138.00, steps = 139
    23:03:55 [INFO] train episode 1787: reward = -161.00, steps = 162
    23:03:55 [INFO] train episode 1788: reward = -129.00, steps = 130
    23:03:55 [INFO] train episode 1789: reward = -105.00, steps = 106
    23:03:55 [INFO] train episode 1790: reward = -121.00, steps = 122
    23:03:55 [INFO] train episode 1791: reward = -113.00, steps = 114
    23:03:56 [INFO] train episode 1792: reward = -151.00, steps = 152
    23:03:56 [INFO] train episode 1793: reward = -127.00, steps = 128
    23:03:56 [INFO] train episode 1794: reward = -147.00, steps = 148
    23:03:56 [INFO] train episode 1795: reward = -119.00, steps = 120
    23:03:56 [INFO] train episode 1796: reward = -111.00, steps = 112
    23:03:56 [INFO] train episode 1797: reward = -89.00, steps = 90
    23:03:56 [INFO] train episode 1798: reward = -96.00, steps = 97
    23:03:56 [INFO] ==== test ====
    23:03:56 [INFO] test episode 0: reward = -125.00, steps = 126
    23:03:57 [INFO] test episode 1: reward = -161.00, steps = 162
    23:03:57 [INFO] test episode 2: reward = -103.00, steps = 104
    23:03:57 [INFO] test episode 3: reward = -93.00, steps = 94
    23:03:57 [INFO] test episode 4: reward = -114.00, steps = 115
    23:03:57 [INFO] test episode 5: reward = -141.00, steps = 142
    23:03:57 [INFO] test episode 6: reward = -114.00, steps = 115
    23:03:57 [INFO] test episode 7: reward = -117.00, steps = 118
    23:03:57 [INFO] test episode 8: reward = -85.00, steps = 86
    23:03:57 [INFO] test episode 9: reward = -141.00, steps = 142
    23:03:57 [INFO] test episode 10: reward = -144.00, steps = 145
    23:03:57 [INFO] test episode 11: reward = -105.00, steps = 106
    23:03:57 [INFO] test episode 12: reward = -105.00, steps = 106
    23:03:58 [INFO] test episode 13: reward = -111.00, steps = 112
    23:03:58 [INFO] test episode 14: reward = -113.00, steps = 114
    23:03:58 [INFO] test episode 15: reward = -124.00, steps = 125
    23:03:58 [INFO] test episode 16: reward = -111.00, steps = 112
    23:03:58 [INFO] test episode 17: reward = -90.00, steps = 91
    23:03:58 [INFO] test episode 18: reward = -96.00, steps = 97
    23:03:58 [INFO] test episode 19: reward = -112.00, steps = 113
    23:03:58 [INFO] test episode 20: reward = -262.00, steps = 263
    23:03:58 [INFO] test episode 21: reward = -118.00, steps = 119
    23:03:58 [INFO] test episode 22: reward = -162.00, steps = 163
    23:03:59 [INFO] test episode 23: reward = -149.00, steps = 150
    23:03:59 [INFO] test episode 24: reward = -134.00, steps = 135
    23:03:59 [INFO] test episode 25: reward = -116.00, steps = 117
    23:03:59 [INFO] test episode 26: reward = -104.00, steps = 105
    23:03:59 [INFO] test episode 27: reward = -106.00, steps = 107
    23:03:59 [INFO] test episode 28: reward = -121.00, steps = 122
    23:03:59 [INFO] test episode 29: reward = -114.00, steps = 115
    23:03:59 [INFO] test episode 30: reward = -142.00, steps = 143
    23:03:59 [INFO] test episode 31: reward = -174.00, steps = 175
    23:03:59 [INFO] test episode 32: reward = -108.00, steps = 109
    23:03:59 [INFO] test episode 33: reward = -128.00, steps = 129
    23:04:00 [INFO] test episode 34: reward = -112.00, steps = 113
    23:04:00 [INFO] test episode 35: reward = -119.00, steps = 120
    23:04:00 [INFO] test episode 36: reward = -123.00, steps = 124
    23:04:00 [INFO] test episode 37: reward = -128.00, steps = 129
    23:04:00 [INFO] test episode 38: reward = -103.00, steps = 104
    23:04:00 [INFO] test episode 39: reward = -108.00, steps = 109
    23:04:00 [INFO] test episode 40: reward = -94.00, steps = 95
    23:04:00 [INFO] test episode 41: reward = -139.00, steps = 140
    23:04:00 [INFO] test episode 42: reward = -142.00, steps = 143
    23:04:00 [INFO] test episode 43: reward = -130.00, steps = 131
    23:04:00 [INFO] test episode 44: reward = -94.00, steps = 95
    23:04:01 [INFO] test episode 45: reward = -128.00, steps = 129
    23:04:01 [INFO] test episode 46: reward = -143.00, steps = 144
    23:04:01 [INFO] test episode 47: reward = -113.00, steps = 114
    23:04:01 [INFO] test episode 48: reward = -134.00, steps = 135
    23:04:01 [INFO] test episode 49: reward = -153.00, steps = 154
    23:04:01 [INFO] test episode 50: reward = -101.00, steps = 102
    23:04:01 [INFO] test episode 51: reward = -149.00, steps = 150
    23:04:01 [INFO] test episode 52: reward = -140.00, steps = 141
    23:04:01 [INFO] test episode 53: reward = -90.00, steps = 91
    23:04:01 [INFO] test episode 54: reward = -129.00, steps = 130
    23:04:01 [INFO] test episode 55: reward = -120.00, steps = 121
    23:04:02 [INFO] test episode 56: reward = -128.00, steps = 129
    23:04:02 [INFO] test episode 57: reward = -91.00, steps = 92
    23:04:02 [INFO] test episode 58: reward = -103.00, steps = 104
    23:04:02 [INFO] test episode 59: reward = -109.00, steps = 110
    23:04:02 [INFO] test episode 60: reward = -102.00, steps = 103
    23:04:02 [INFO] test episode 61: reward = -124.00, steps = 125
    23:04:02 [INFO] test episode 62: reward = -124.00, steps = 125
    23:04:02 [INFO] test episode 63: reward = -105.00, steps = 106
    23:04:02 [INFO] test episode 64: reward = -125.00, steps = 126
    23:04:02 [INFO] test episode 65: reward = -124.00, steps = 125
    23:04:02 [INFO] test episode 66: reward = -96.00, steps = 97
    23:04:02 [INFO] test episode 67: reward = -128.00, steps = 129
    23:04:03 [INFO] test episode 68: reward = -121.00, steps = 122
    23:04:03 [INFO] test episode 69: reward = -152.00, steps = 153
    23:04:03 [INFO] test episode 70: reward = -151.00, steps = 152
    23:04:03 [INFO] test episode 71: reward = -104.00, steps = 105
    23:04:03 [INFO] test episode 72: reward = -123.00, steps = 124
    23:04:03 [INFO] test episode 73: reward = -98.00, steps = 99
    23:04:03 [INFO] test episode 74: reward = -95.00, steps = 96
    23:04:03 [INFO] test episode 75: reward = -122.00, steps = 123
    23:04:03 [INFO] test episode 76: reward = -135.00, steps = 136
    23:04:03 [INFO] test episode 77: reward = -148.00, steps = 149
    23:04:03 [INFO] test episode 78: reward = -124.00, steps = 125
    23:04:04 [INFO] test episode 79: reward = -125.00, steps = 126
    23:04:04 [INFO] test episode 80: reward = -174.00, steps = 175
    23:04:04 [INFO] test episode 81: reward = -116.00, steps = 117
    23:04:04 [INFO] test episode 82: reward = -103.00, steps = 104
    23:04:04 [INFO] test episode 83: reward = -115.00, steps = 116
    23:04:04 [INFO] test episode 84: reward = -110.00, steps = 111
    23:04:04 [INFO] test episode 85: reward = -123.00, steps = 124
    23:04:04 [INFO] test episode 86: reward = -139.00, steps = 140
    23:04:04 [INFO] test episode 87: reward = -131.00, steps = 132
    23:04:04 [INFO] test episode 88: reward = -101.00, steps = 102
    23:04:04 [INFO] test episode 89: reward = -128.00, steps = 129
    23:04:05 [INFO] test episode 90: reward = -141.00, steps = 142
    23:04:05 [INFO] test episode 91: reward = -180.00, steps = 181
    23:04:05 [INFO] test episode 92: reward = -120.00, steps = 121
    23:04:05 [INFO] test episode 93: reward = -129.00, steps = 130
    23:04:05 [INFO] test episode 94: reward = -114.00, steps = 115
    23:04:05 [INFO] test episode 95: reward = -97.00, steps = 98
    23:04:05 [INFO] test episode 96: reward = -138.00, steps = 139
    23:04:05 [INFO] test episode 97: reward = -126.00, steps = 127
    23:04:05 [INFO] test episode 98: reward = -169.00, steps = 170
    23:04:05 [INFO] test episode 99: reward = -133.00, steps = 134
    23:04:05 [INFO] average episode reward = -123.79 ± 24.41

In [6]:

    env.close()