# Use Soft Q Learning to Play LunarLander-v2¶

PyTorch version

In [1]:

    %matplotlib inline

    import sys
    import logging
    import itertools
    import copy

    import numpy as np
    np.random.seed(0)
    import pandas as pd
    import gym
    import matplotlib.pyplot as plt
    import torch
    torch.manual_seed(0)
    import torch.nn as nn
    import torch.optim as optim
    import torch.distributions as distributions

    logging.basicConfig(level=logging.INFO,
            format='%(asctime)s [%(levelname)s] %(message)s',
            stream=sys.stdout, datefmt='%H:%M:%S')

In [2]:

    env = gym.make('LunarLander-v2')
    for key in vars(env):
        logging.info('%s: %s', key, vars(env)[key])
    for key in vars(env.spec):
        logging.info('%s: %s', key, vars(env.spec)[key])

    15:30:04 [INFO] env: <LunarLander<LunarLander-v2>>
    15:30:04 [INFO] action_space: Discrete(4)
    15:30:04 [INFO] observation_space: Box(-inf, inf, (8,), float32)
    15:30:04 [INFO] reward_range: (-inf, inf)
    15:30:04 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 50}
    15:30:04 [INFO] _max_episode_steps: 1000
    15:30:04 [INFO] _elapsed_steps: None
    15:30:04 [INFO] id: LunarLander-v2
    15:30:04 [INFO] entry_point: gym.envs.box2d:LunarLander
    15:30:04 [INFO] reward_threshold: 200
    15:30:04 [INFO] nondeterministic: False
    15:30:04 [INFO] max_episode_steps: 1000
    15:30:04 [INFO] _kwargs: {}
    15:30:04 [INFO] _env_name: LunarLander

In [3]:

    class DQNReplayer:
        def __init__(self, capacity):
            self.memory = pd.DataFrame(index=range(capacity),
                    columns=['state', 'action', 'reward', 'next_state', 'terminated'])
            self.i = 0
            self.count = 0
            self.capacity = capacity

        def store(self, *args):
            self.memory.loc[self.i] = np.asarray(args, dtype=object)
            self.i = (self.i + 1) % self.capacity
            self.count = min(self.count + 1, self.capacity)

        def sample(self, size):
            indices = np.random.choice(self.count, size=size)
            return (np.stack(self.memory.loc[indices, field]) for field in
                    self.memory.columns)

In [4]:

    class SQLAgent:
        def __init__(self, env):
            self.action_n = env.action_space.n
            self.gamma = 0.99

            self.replayer = DQNReplayer(10000)

            self.alpha = 0.02

            self.evaluate_net = self.build_net(
                    input_size=env.observation_space.shape[0],
                    hidden_sizes=[256, 256], output_size=self.action_n)
            self.optimizer = optim.Adam(self.evaluate_net.parameters(), lr=3e-4)
            self.loss = nn.MSELoss()

        def build_net(self, input_size, hidden_sizes, output_size):
            layers = []
            for input_size, output_size in zip(
                    [input_size,] + hidden_sizes, hidden_sizes + [output_size,]):
                layers.append(nn.Linear(input_size, output_size))
                layers.append(nn.ReLU())
            layers = layers[:-1]
            model = nn.Sequential(*layers)
            return model

        def reset(self, mode=None):
            self.mode = mode
            if self.mode == 'train':
                self.trajectory = []
                self.target_net = copy.deepcopy(self.evaluate_net)

        def step(self, observation, reward, terminated):
            state_tensor = torch.as_tensor(observation,
                    dtype=torch.float).squeeze(0)
            q_div_alpha_tensor = self.evaluate_net(state_tensor) / self.alpha
            v_div_alpha_tensor = torch.logsumexp(q_div_alpha_tensor, dim=-1,
                    keepdim=True)
            prob_tensor = (q_div_alpha_tensor - v_div_alpha_tensor).exp()
            action_tensor = distributions.Categorical(prob_tensor).sample()
            action = action_tensor.item()
            if self.mode == 'train':
                self.trajectory += [observation, reward, terminated, action]
                if len(self.trajectory) >= 8:
                    state, _, _, act, next_state, reward, terminated, _ = \
                            self.trajectory[-8:]
                    self.replayer.store(state, act, reward, next_state, terminated)
                if self.replayer.count >= 500:
                    self.learn()
            return action

        def close(self):
            pass

        def learn(self):
            # replay
            states, actions, rewards, next_states, terminateds = \
                    self.replayer.sample(128)
            state_tensor = torch.as_tensor(states, dtype=torch.float)
            action_tensor = torch.as_tensor(actions, dtype=torch.long)
            reward_tensor = torch.as_tensor(rewards, dtype=torch.float)
            next_state_tensor = torch.as_tensor(next_states, dtype=torch.float)
            terminated_tensor = torch.as_tensor(terminateds, dtype=torch.float)

            # update value net
            next_q_tensor = self.target_net(next_state_tensor)
            next_v_tensor = self.alpha * torch.logsumexp(next_q_tensor / self.alpha,
                    dim=-1)
            target_tensor = reward_tensor + self.gamma * (1. - terminated_tensor) * \
                    next_v_tensor
            pred_tensor = self.evaluate_net(state_tensor)
            q_tensor = pred_tensor.gather(1, action_tensor.unsqueeze(1)).squeeze(1)
            loss_tensor = self.loss(q_tensor, target_tensor.detach())
            self.optimizer.zero_grad()
            loss_tensor.backward()
            self.optimizer.step()

    agent = SQLAgent(env)

In [5]:

    def play_episode(env, agent, seed=None, mode=None, render=False):
        observation, _ = env.reset(seed=seed)
        reward, terminated, truncated = 0., False, False
        agent.reset(mode=mode)
        episode_reward, elapsed_steps = 0., 0
        while True:
            action = agent.step(observation, reward, terminated)
            if render:
                env.render()
            if terminated or truncated:
                break
            observation, reward, terminated, truncated, _ = env.step(action)
            episode_reward += reward
            elapsed_steps += 1
        agent.close()
        return episode_reward, elapsed_steps

    logging.info('==== train ====')
    episode_rewards = []
    for episode in itertools.count():
        episode_reward, elapsed_steps = play_episode(env, agent, seed=episode,
                mode='train')
        episode_rewards.append(episode_reward)
        logging.info('train episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
        if np.mean(episode_rewards[-10:]) > 250:
            break
    plt.plot(episode_rewards)

    logging.info('==== test ====')
    episode_rewards = []
    for episode in range(100):
        episode_reward, elapsed_steps = play_episode(env, agent)
        episode_rewards.append(episode_reward)
        logging.info('test episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
    logging.info('average episode reward = %.2f ± %.2f',
            np.mean(episode_rewards), np.std(episode_rewards))

    15:30:04 [INFO] ==== train ====
    15:30:04 [INFO] train episode 0: reward = -461.18, steps = 83
    15:30:04 [INFO] train episode 1: reward = -668.99, steps = 69
    15:30:04 [INFO] train episode 2: reward = -458.32, steps = 73
    15:30:04 [INFO] train episode 3: reward = -493.66, steps = 56
    15:30:04 [INFO] train episode 4: reward = -546.61, steps = 76
    15:30:04 [INFO] train episode 5: reward = -592.39, steps = 75
    15:30:04 [INFO] train episode 6: reward = -591.07, steps = 68
    15:30:05 [INFO] train episode 7: reward = 10.01, steps = 58
    15:30:06 [INFO] train episode 8: reward = -101.41, steps = 53
    15:30:06 [INFO] train episode 9: reward = -184.76, steps = 77
    15:30:07 [INFO] train episode 10: reward = -127.31, steps = 61
    15:30:08 [INFO] train episode 11: reward = -162.83, steps = 71
    15:30:09 [INFO] train episode 12: reward = -151.20, steps = 79
    15:30:09 [INFO] train episode 13: reward = -142.71, steps = 80
    15:30:10 [INFO] train episode 14: reward = -159.48, steps = 66
    15:30:11 [INFO] train episode 15: reward = -109.68, steps = 77
    15:30:12 [INFO] train episode 16: reward = -162.97, steps = 68
    15:30:13 [INFO] train episode 17: reward = -78.89, steps = 55
    15:30:14 [INFO] train episode 18: reward = -856.41, steps = 154
    15:30:16 [INFO] train episode 19: reward = -717.81, steps = 110
    15:30:16 [INFO] train episode 20: reward = -109.94, steps = 63
    15:30:17 [INFO] train episode 21: reward = -124.91, steps = 61
    15:30:18 [INFO] train episode 22: reward = -260.72, steps = 100
    15:30:19 [INFO] train episode 23: reward = -118.52, steps = 50
    15:30:20 [INFO] train episode 24: reward = -44.96, steps = 89
    15:30:21 [INFO] train episode 25: reward = -84.43, steps = 56
    15:30:22 [INFO] train episode 26: reward = -74.78, steps = 87
    15:30:24 [INFO] train episode 27: reward = -382.92, steps = 175
    15:30:25 [INFO] train episode 28: reward = -179.34, steps = 84
    15:30:26 [INFO] train episode 29: reward = -59.86, steps = 56
    15:30:27 [INFO] train episode 30: reward = -65.31, steps = 94
    15:30:28 [INFO] train episode 31: reward = -49.00, steps = 75
    15:30:29 [INFO] train episode 32: reward = 42.47, steps = 88
    15:30:30 [INFO] train episode 33: reward = -196.26, steps = 91
    15:30:32 [INFO] train episode 34: reward = -1080.69, steps = 159
    15:30:35 [INFO] train episode 35: reward = -346.48, steps = 190
    15:30:37 [INFO] train episode 36: reward = -272.09, steps = 217
    15:30:38 [INFO] train episode 37: reward = -108.51, steps = 102
    15:30:39 [INFO] train episode 38: reward = -422.14, steps = 80
    15:30:40 [INFO] train episode 39: reward = -469.50, steps = 95
    15:30:42 [INFO] train episode 40: reward = -349.59, steps = 190
    15:30:43 [INFO] train episode 41: reward = -212.93, steps = 104
    15:30:45 [INFO] train episode 42: reward = -257.79, steps = 128
    15:30:47 [INFO] train episode 43: reward = -293.18, steps = 215
    15:30:49 [INFO] train episode 44: reward = -199.20, steps = 150
    15:30:51 [INFO] train episode 45: reward = -223.86, steps = 158
    15:30:52 [INFO] train episode 46: reward = -204.57, steps = 134
    15:30:54 [INFO] train episode 47: reward = -198.53, steps = 174
    15:30:56 [INFO] train episode 48: reward = -358.08, steps = 155
    15:30:58 [INFO] train episode 49: reward = -216.80, steps = 155
    15:30:59 [INFO] train episode 50: reward = -131.84, steps = 171
    15:31:01 [INFO] train episode 51: reward = -224.78, steps = 128
    15:31:03 [INFO] train episode 52: reward = -138.28, steps = 160
    15:31:04 [INFO] train episode 53: reward = -94.56, steps = 150
    15:31:06 [INFO] train episode 54: reward = -73.09, steps = 156
    15:31:09 [INFO] train episode 55: reward = -381.66, steps = 242
    15:31:10 [INFO] train episode 56: reward = -280.63, steps = 149
    15:31:13 [INFO] train episode 57: reward = -262.54, steps = 236
    15:31:15 [INFO] train episode 58: reward = -494.19, steps = 168
    15:31:17 [INFO] train episode 59: reward = -168.34, steps = 149
    15:31:19 [INFO] train episode 60: reward = -319.37, steps = 232
    15:31:21 [INFO] train episode 61: reward = -172.28, steps = 137
    15:31:23 [INFO] train episode 62: reward = -217.15, steps = 143
    15:31:24 [INFO] train episode 63: reward = -216.08, steps = 154
    15:31:30 [INFO] train episode 64: reward = -192.72, steps = 434
    15:31:32 [INFO] train episode 65: reward = -186.88, steps = 149
    15:31:33 [INFO] train episode 66: reward = -183.78, steps = 139
    15:31:35 [INFO] train episode 67: reward = -352.77, steps = 167
    15:31:37 [INFO] train episode 68: reward = 15.43, steps = 149
    15:31:38 [INFO] train episode 69: reward = -253.02, steps = 124
    15:31:42 [INFO] train episode 70: reward = -259.30, steps = 325
    15:31:44 [INFO] train episode 71: reward = -118.72, steps = 189
    15:31:47 [INFO] train episode 72: reward = -279.75, steps = 231
    15:31:50 [INFO] train episode 73: reward = -291.15, steps = 217
    15:31:51 [INFO] train episode 74: reward = -136.00, steps = 140
    15:31:53 [INFO] train episode 75: reward = -189.93, steps = 153
    15:31:55 [INFO] train episode 76: reward = -196.52, steps = 154
    15:31:56 [INFO] train episode 77: reward = -101.84, steps = 160
    15:31:58 [INFO] train episode 78: reward = -180.87, steps = 146
    15:32:02 [INFO] train episode 79: reward = -271.79, steps = 283
    15:32:05 [INFO] train episode 80: reward = -444.22, steps = 248
    15:32:10 [INFO] train episode 81: reward = -197.07, steps = 343
    15:32:12 [INFO] train episode 82: reward = -102.25, steps = 140
    15:32:17 [INFO] train episode 83: reward = -220.68, steps = 253
    15:32:18 [INFO] train episode 84: reward = -187.62, steps = 101
    15:32:21 [INFO] train episode 85: reward = -233.20, steps = 206
    15:32:26 [INFO] train episode 86: reward = -266.51, steps = 369
    15:32:31 [INFO] train episode 87: reward = -232.73, steps = 340
    15:32:35 [INFO] train episode 88: reward = -165.14, steps = 301
    15:32:52 [INFO] train episode 89: reward = -167.00, steps = 1000
    15:32:56 [INFO] train episode 90: reward = -237.51, steps = 219
    15:33:13 [INFO] train episode 91: reward = -201.66, steps = 1000
    15:33:15 [INFO] train episode 92: reward = -180.10, steps = 188
    15:33:31 [INFO] train episode 93: reward = -148.77, steps = 1000
    15:33:47 [INFO] train episode 94: reward = -150.70, steps = 1000
    15:34:02 [INFO] train episode 95: reward = -181.40, steps = 1000
    15:34:04 [INFO] train episode 96: reward = -100.86, steps = 141
    15:34:08 [INFO] train episode 97: reward = -154.23, steps = 246
    15:34:23 [INFO] train episode 98: reward = -161.37, steps = 1000
    15:34:24 [INFO] train episode 99: reward = -291.60, steps = 95
    15:34:40 [INFO] train episode 100: reward = -149.28, steps = 1000
    15:34:56 [INFO] train episode 101: reward = -140.16, steps = 1000
    15:35:12 [INFO] train episode 102: reward = -201.76, steps = 1000
    15:35:29 [INFO] train episode 103: reward = -156.56, steps = 1000
    15:35:47 [INFO] train episode 104: reward = -130.02, steps = 1000
    15:36:03 [INFO] train episode 105: reward = -165.13, steps = 1000
    15:36:21 [INFO] train episode 106: reward = -209.76, steps = 1000
    15:36:36 [INFO] train episode 107: reward = -161.81, steps = 1000
    15:36:37 [INFO] train episode 108: reward = -145.19, steps = 83
    15:36:52 [INFO] train episode 109: reward = -179.72, steps = 1000
    15:37:07 [INFO] train episode 110: reward = -192.56, steps = 1000
    15:37:22 [INFO] train episode 111: reward = -202.50, steps = 1000
    15:37:38 [INFO] train episode 112: reward = -167.27, steps = 1000
    15:37:56 [INFO] train episode 113: reward = -178.42, steps = 1000
    15:38:11 [INFO] train episode 114: reward = -175.40, steps = 1000
    15:38:26 [INFO] train episode 115: reward = -205.94, steps = 1000
    15:38:42 [INFO] train episode 116: reward = -165.85, steps = 1000
    15:38:58 [INFO] train episode 117: reward = -183.95, steps = 1000
    15:39:14 [INFO] train episode 118: reward = -201.14, steps = 1000
    15:39:30 [INFO] train episode 119: reward = -155.56, steps = 1000
    15:39:45 [INFO] train episode 120: reward = -139.84, steps = 1000
    15:40:02 [INFO] train episode 121: reward = -141.59, steps = 1000
    15:40:18 [INFO] train episode 122: reward = -167.62, steps = 1000
    15:40:34 [INFO] train episode 123: reward = -148.34, steps = 1000
    15:40:48 [INFO] train episode 124: reward = -98.38, steps = 1000
    15:41:06 [INFO] train episode 125: reward = -173.27, steps = 1000
    15:41:22 [INFO] train episode 126: reward = -134.38, steps = 1000
    15:41:39 [INFO] train episode 127: reward = -123.90, steps = 1000
    15:41:41 [INFO] train episode 128: reward = -260.99, steps = 135
    15:41:58 [INFO] train episode 129: reward = -205.37, steps = 1000
    15:42:15 [INFO] train episode 130: reward = -161.72, steps = 1000
    15:42:31 [INFO] train episode 131: reward = -171.49, steps = 1000
    15:42:47 [INFO] train episode 132: reward = -145.85, steps = 1000
    15:43:03 [INFO] train episode 133: reward = -140.45, steps = 1000
    15:43:18 [INFO] train episode 134: reward = -144.73, steps = 1000
    15:43:37 [INFO] train episode 135: reward = -145.92, steps = 1000
    15:43:54 [INFO] train episode 136: reward = -131.38, steps = 1000
    15:44:13 [INFO] train episode 137: reward = -159.54, steps = 1000
    15:44:30 [INFO] train episode 138: reward = -164.33, steps = 1000
    15:44:51 [INFO] train episode 139: reward = -151.81, steps = 1000
    15:45:10 [INFO] train episode 140: reward = -159.38, steps = 1000
    15:45:29 [INFO] train episode 141: reward = -119.14, steps = 1000
    15:45:46 [INFO] train episode 142: reward = -149.16, steps = 1000
    15:46:03 [INFO] train episode 143: reward = -134.57, steps = 1000
    15:46:08 [INFO] train episode 144: reward = -172.85, steps = 329
    15:46:27 [INFO] train episode 145: reward = -146.75, steps = 1000
    15:46:40 [INFO] train episode 146: reward = -228.53, steps = 671
    15:46:57 [INFO] train episode 147: reward = -133.78, steps = 1000
    15:47:17 [INFO] train episode 148: reward = -170.46, steps = 1000
    15:47:34 [INFO] train episode 149: reward = -132.13, steps = 1000
    15:47:51 [INFO] train episode 150: reward = -155.12, steps = 1000
    15:48:09 [INFO] train episode 151: reward = -115.96, steps = 1000
    15:48:26 [INFO] train episode 152: reward = -128.12, steps = 1000
    15:48:46 [INFO] train episode 153: reward = -136.80, steps = 1000
    15:49:06 [INFO] train episode 154: reward = -130.60, steps = 1000
    15:49:24 [INFO] train episode 155: reward = -104.23, steps = 1000
    15:49:51 [INFO] train episode 156: reward = -154.83, steps = 1000
    15:50:10 [INFO] train episode 157: reward = -132.27, steps = 1000
    15:50:14 [INFO] train episode 158: reward = -141.06, steps = 257
    15:50:30 [INFO] train episode 159: reward = -129.47, steps = 1000
    15:50:31 [INFO] train episode 160: reward = -299.50, steps = 80
    15:50:48 [INFO] train episode 161: reward = -140.84, steps = 1000
    15:51:00 [INFO] train episode 162: reward = -244.77, steps = 816
    15:51:16 [INFO] train episode 163: reward = -91.54, steps = 1000
    15:51:28 [INFO] train episode 164: reward = -195.96, steps = 743
    15:51:30 [INFO] train episode 165: reward = -124.06, steps = 139
    15:51:46 [INFO] train episode 166: reward = -132.26, steps = 1000
    15:51:50 [INFO] train episode 167: reward = -77.08, steps = 289
    15:51:53 [INFO] train episode 168: reward = -75.81, steps = 231
    15:52:08 [INFO] train episode 169: reward = -118.95, steps = 1000
    15:52:15 [INFO] train episode 170: reward = -234.10, steps = 453
    15:52:32 [INFO] train episode 171: reward = -128.45, steps = 1000
    15:52:41 [INFO] train episode 172: reward = -331.69, steps = 600
    15:52:58 [INFO] train episode 173: reward = -73.57, steps = 1000
    15:53:15 [INFO] train episode 174: reward = -114.12, steps = 1000
    15:53:32 [INFO] train episode 175: reward = -160.87, steps = 1000
    15:53:50 [INFO] train episode 176: reward = -134.43, steps = 1000
    15:54:06 [INFO] train episode 177: reward = -116.93, steps = 1000
    15:54:22 [INFO] train episode 178: reward = -178.51, steps = 1000
    15:54:39 [INFO] train episode 179: reward = -84.95, steps = 1000
    15:54:54 [INFO] train episode 180: reward = -117.83, steps = 1000
    15:55:11 [INFO] train episode 181: reward = -127.32, steps = 1000
    15:55:26 [INFO] train episode 182: reward = -105.13, steps = 1000
    15:55:43 [INFO] train episode 183: reward = -117.50, steps = 1000
    15:55:58 [INFO] train episode 184: reward = -107.08, steps = 1000
    15:56:16 [INFO] train episode 185: reward = -111.60, steps = 1000
    15:56:32 [INFO] train episode 186: reward = -101.00, steps = 1000
    15:56:48 [INFO] train episode 187: reward = -106.90, steps = 1000
    15:57:04 [INFO] train episode 188: reward = -116.72, steps = 1000
    15:57:21 [INFO] train episode 189: reward = -145.83, steps = 1000
    15:57:38 [INFO] train episode 190: reward = -94.56, steps = 1000
    15:57:56 [INFO] train episode 191: reward = -158.11, steps = 1000
    15:58:14 [INFO] train episode 192: reward = -159.91, steps = 1000
    15:58:32 [INFO] train episode 193: reward = -119.21, steps = 1000
    15:58:47 [INFO] train episode 194: reward = -115.40, steps = 1000
    15:59:04 [INFO] train episode 195: reward = -68.14, steps = 1000
    15:59:23 [INFO] train episode 196: reward = -132.07, steps = 1000
    15:59:40 [INFO] train episode 197: reward = -144.48, steps = 1000
    15:59:57 [INFO] train episode 198: reward = -116.21, steps = 1000
    15:59:58 [INFO] train episode 199: reward = -102.59, steps = 58
    16:00:14 [INFO] train episode 200: reward = -103.85, steps = 1000
    16:00:31 [INFO] train episode 201: reward = -131.40, steps = 1000
    16:00:52 [INFO] train episode 202: reward = -70.19, steps = 1000
    16:01:09 [INFO] train episode 203: reward = -142.31, steps = 1000
    16:01:25 [INFO] train episode 204: reward = -145.11, steps = 1000
    16:01:41 [INFO] train episode 205: reward = -124.93, steps = 1000
    16:01:56 [INFO] train episode 206: reward = -74.29, steps = 1000
    16:02:12 [INFO] train episode 207: reward = -33.20, steps = 1000
    16:02:28 [INFO] train episode 208: reward = -89.19, steps = 1000
    16:02:44 [INFO] train episode 209: reward = -67.87, steps = 1000
    16:03:00 [INFO] train episode 210: reward = -90.71, steps = 1000
    16:03:16 [INFO] train episode 211: reward = -110.03, steps = 1000
    16:03:32 [INFO] train episode 212: reward = -66.46, steps = 1000
    16:03:49 [INFO] train episode 213: reward = -98.37, steps = 1000
    16:04:05 [INFO] train episode 214: reward = -56.01, steps = 1000
    16:04:21 [INFO] train episode 215: reward = -89.63, steps = 1000
    16:04:38 [INFO] train episode 216: reward = -84.69, steps = 1000
    16:04:54 [INFO] train episode 217: reward = -67.74, steps = 1000
    16:05:11 [INFO] train episode 218: reward = -105.08, steps = 1000
    16:05:12 [INFO] train episode 219: reward = -91.82, steps = 52
    16:05:28 [INFO] train episode 220: reward = -55.65, steps = 1000
    16:05:43 [INFO] train episode 221: reward = -86.73, steps = 1000
    16:05:58 [INFO] train episode 222: reward = -45.91, steps = 1000
    16:06:16 [INFO] train episode 223: reward = -49.99, steps = 1000
    16:06:32 [INFO] train episode 224: reward = -32.55, steps = 1000
    16:06:49 [INFO] train episode 225: reward = -78.87, steps = 1000
    16:07:06 [INFO] train episode 226: reward = -44.40, steps = 1000
    16:07:22 [INFO] train episode 227: reward = -54.29, steps = 1000
    16:07:38 [INFO] train episode 228: reward = -82.28, steps = 1000
    16:07:54 [INFO] train episode 229: reward = -68.36, steps = 1000
    16:08:09 [INFO] train episode 230: reward = -26.31, steps = 1000
    16:08:24 [INFO] train episode 231: reward = -86.26, steps = 1000
    16:08:40 [INFO] train episode 232: reward = -46.15, steps = 1000
    16:08:56 [INFO] train episode 233: reward = -155.39, steps = 1000
    16:09:12 [INFO] train episode 234: reward = -41.64, steps = 1000
    16:09:29 [INFO] train episode 235: reward = -88.05, steps = 1000
    16:09:46 [INFO] train episode 236: reward = -87.17, steps = 1000
    16:10:01 [INFO] train episode 237: reward = -168.74, steps = 991
    16:10:14 [INFO] train episode 238: reward = -159.35, steps = 821
    16:10:31 [INFO] train episode 239: reward = -89.67, steps = 1000
    16:10:47 [INFO] train episode 240: reward = -229.81, steps = 944
    16:11:05 [INFO] train episode 241: reward = -48.93, steps = 1000
    16:11:20 [INFO] train episode 242: reward = -30.57, steps = 1000
    16:11:38 [INFO] train episode 243: reward = -98.84, steps = 1000
    16:11:54 [INFO] train episode 244: reward = -59.22, steps = 1000
    16:12:11 [INFO] train episode 245: reward = -47.49, steps = 1000
    16:12:29 [INFO] train episode 246: reward = -49.54, steps = 1000
    16:12:45 [INFO] train episode 247: reward = -55.58, steps = 1000
    16:13:01 [INFO] train episode 248: reward = -261.72, steps = 969
    16:13:19 [INFO] train episode 249: reward = -40.42, steps = 1000
    16:13:35 [INFO] train episode 250: reward = -32.94, steps = 1000
    16:13:52 [INFO] train episode 251: reward = -24.56, steps = 1000
    16:14:08 [INFO] train episode 252: reward = -16.88, steps = 1000
    16:14:26 [INFO] train episode 253: reward = -56.63, steps = 1000
    16:14:42 [INFO] train episode 254: reward = -38.38, steps = 1000
    16:14:59 [INFO] train episode 255: reward = -54.97, steps = 1000
    16:15:17 [INFO] train episode 256: reward = -79.20, steps = 1000
    16:15:35 [INFO] train episode 257: reward = -76.28, steps = 1000
    16:15:52 [INFO] train episode 258: reward = -59.60, steps = 1000
    16:16:09 [INFO] train episode 259: reward = -35.59, steps = 1000
    16:16:27 [INFO] train episode 260: reward = -51.68, steps = 1000
    16:16:44 [INFO] train episode 261: reward = -51.66, steps = 1000
    16:17:01 [INFO] train episode 262: reward = -32.21, steps = 1000
    16:17:19 [INFO] train episode 263: reward = -74.43, steps = 1000
    16:17:36 [INFO] train episode 264: reward = -60.47, steps = 1000
    16:17:54 [INFO] train episode 265: reward = -34.44, steps = 1000
    16:18:11 [INFO] train episode 266: reward = -17.53, steps = 1000
    16:18:28 [INFO] train episode 267: reward = -50.44, steps = 1000
    16:18:47 [INFO] train episode 268: reward = -31.32, steps = 1000
    16:19:05 [INFO] train episode 269: reward = -74.84, steps = 1000
    16:19:23 [INFO] train episode 270: reward = -71.76, steps = 1000
    16:19:42 [INFO] train episode 271: reward = -26.28, steps = 1000
    16:20:00 [INFO] train episode 272: reward = -78.65, steps = 1000
    16:20:19 [INFO] train episode 273: reward = -54.83, steps = 1000
    16:20:36 [INFO] train episode 274: reward = -20.61, steps = 1000
    16:20:53 [INFO] train episode 275: reward = -57.88, steps = 1000
    16:21:12 [INFO] train episode 276: reward = -44.64, steps = 1000
    16:21:28 [INFO] train episode 277: reward = -27.04, steps = 1000
    16:21:45 [INFO] train episode 278: reward = -39.71, steps = 1000
    16:22:05 [INFO] train episode 279: reward = -55.80, steps = 1000
    16:22:23 [INFO] train episode 280: reward = -67.25, steps = 1000
    16:22:39 [INFO] train episode 281: reward = -33.43, steps = 1000
    16:22:59 [INFO] train episode 282: reward = -36.89, steps = 1000
    16:23:17 [INFO] train episode 283: reward = -109.18, steps = 1000
    16:23:23 [INFO] train episode 284: reward = -637.89, steps = 368
    16:23:41 [INFO] train episode 285: reward = -56.64, steps = 1000
    16:23:58 [INFO] train episode 286: reward = -60.65, steps = 1000
    16:24:14 [INFO] train episode 287: reward = -23.12, steps = 1000
    16:24:31 [INFO] train episode 288: reward = -67.32, steps = 1000
    16:24:46 [INFO] train episode 289: reward = -70.14, steps = 1000
    16:25:03 [INFO] train episode 290: reward = -11.99, steps = 1000
    16:25:21 [INFO] train episode 291: reward = -82.50, steps = 1000
    16:25:38 [INFO] train episode 292: reward = -24.19, steps = 1000
    16:25:55 [INFO] train episode 293: reward = -75.30, steps = 1000
    16:26:11 [INFO] train episode 294: reward = -55.99, steps = 1000
    16:26:27 [INFO] train episode 295: reward = 10.52, steps = 1000
    16:26:43 [INFO] train episode 296: reward = -49.16, steps = 1000
    16:27:02 [INFO] train episode 297: reward = -57.18, steps = 1000
    16:27:18 [INFO] train episode 298: reward = -35.64, steps = 1000
    16:27:35 [INFO] train episode 299: reward = -661.19, steps = 1000
    16:27:52 [INFO] train episode 300: reward = -66.01, steps = 1000
    16:27:52 [INFO] train episode 301: reward = -109.61, steps = 53
    16:28:08 [INFO] train episode 302: reward = -184.58, steps = 1000
    16:28:24 [INFO] train episode 303: reward = -61.92, steps = 1000
    16:28:41 [INFO] train episode 304: reward = -71.44, steps = 1000
    16:28:43 [INFO] train episode 305: reward = -105.97, steps = 180
    16:29:00 [INFO] train episode 306: reward = -5.41, steps = 1000
    16:29:16 [INFO] train episode 307: reward = -89.25, steps = 1000
    16:29:33 [INFO] train episode 308: reward = -80.65, steps = 1000
    16:29:50 [INFO] train episode 309: reward = -35.80, steps = 1000
    16:30:07 [INFO] train episode 310: reward = -41.52, steps = 1000
    16:30:23 [INFO] train episode 311: reward = -82.70, steps = 1000
    16:30:25 [INFO] train episode 312: reward = -120.78, steps = 131
    16:30:30 [INFO] train episode 313: reward = -174.65, steps = 297
    16:30:32 [INFO] train episode 314: reward = -60.49, steps = 185
    16:30:48 [INFO] train episode 315: reward = -33.68, steps = 1000
    16:30:50 [INFO] train episode 316: reward = -327.84, steps = 130
    16:30:52 [INFO] train episode 317: reward = -368.02, steps = 134
    16:30:55 [INFO] train episode 318: reward = -374.26, steps = 207
    16:30:57 [INFO] train episode 319: reward = -258.82, steps = 108
    16:31:13 [INFO] train episode 320: reward = -31.46, steps = 1000
    16:31:30 [INFO] train episode 321: reward = -45.66, steps = 1000
    16:31:46 [INFO] train episode 322: reward = -34.57, steps = 1000
    16:31:52 [INFO] train episode 323: reward = -209.25, steps = 403
    16:32:08 [INFO] train episode 324: reward = -39.94, steps = 1000
    16:32:11 [INFO] train episode 325: reward = -135.46, steps = 163
    16:32:27 [INFO] train episode 326: reward = -10.92, steps = 1000
    16:32:43 [INFO] train episode 327: reward = -45.30, steps = 1000
    16:33:00 [INFO] train episode 328: reward = -51.49, steps = 1000
    16:33:09 [INFO] train episode 329: reward = -22.21, steps = 524
    16:33:25 [INFO] train episode 330: reward = -32.08, steps = 1000
    16:33:32 [INFO] train episode 331: reward = -189.42, steps = 447
    16:33:35 [INFO] train episode 332: reward = -78.06, steps = 209
    16:33:42 [INFO] train episode 333: reward = -208.59, steps = 487
    16:33:59 [INFO] train episode 334: reward = -12.92, steps = 1000
    16:34:15 [INFO] train episode 335: reward = -26.31, steps = 1000
    16:34:32 [INFO] train episode 336: reward = -78.34, steps = 1000
    16:34:48 [INFO] train episode 337: reward = 21.48, steps = 1000
    16:35:06 [INFO] train episode 338: reward = -58.45, steps = 1000
    16:35:22 [INFO] train episode 339: reward = 11.60, steps = 1000
    16:35:38 [INFO] train episode 340: reward = 10.61, steps = 1000
    16:35:53 [INFO] train episode 341: reward = 3.54, steps = 1000
    16:36:09 [INFO] train episode 342: reward = 15.60, steps = 1000
    16:36:09 [INFO] train episode 343: reward = -119.59, steps = 52
    16:36:24 [INFO] train episode 344: reward = -32.67, steps = 1000
    16:36:40 [INFO] train episode 345: reward = 10.68, steps = 1000
    16:36:57 [INFO] train episode 346: reward = -35.37, steps = 1000
    16:37:12 [INFO] train episode 347: reward = 9.28, steps = 1000
    16:37:28 [INFO] train episode 348: reward = -22.10, steps = 1000
    16:37:44 [INFO] train episode 349: reward = 24.38, steps = 1000
    16:38:00 [INFO] train episode 350: reward = 8.58, steps = 1000
    16:38:17 [INFO] train episode 351: reward = 1.40, steps = 1000
    16:38:35 [INFO] train episode 352: reward = -51.02, steps = 1000
    16:38:52 [INFO] train episode 353: reward = -68.66, steps = 1000
    16:39:08 [INFO] train episode 354: reward = 13.89, steps = 1000
    16:39:24 [INFO] train episode 355: reward = 8.85, steps = 1000
    16:39:40 [INFO] train episode 356: reward = -37.26, steps = 1000
    16:39:56 [INFO] train episode 357: reward = 12.02, steps = 1000
    16:40:12 [INFO] train episode 358: reward = -47.81, steps = 1000
    16:40:28 [INFO] train episode 359: reward = -21.64, steps = 1000
    16:40:45 [INFO] train episode 360: reward = 4.33, steps = 1000
    16:41:02 [INFO] train episode 361: reward = -14.15, steps = 1000
    16:41:18 [INFO] train episode 362: reward = -0.50, steps = 1000
    16:41:35 [INFO] train episode 363: reward = -32.45, steps = 1000
    16:41:52 [INFO] train episode 364: reward = -44.04, steps = 1000
    16:42:08 [INFO] train episode 365: reward = -7.90, steps = 1000
    16:42:26 [INFO] train episode 366: reward = -42.57, steps = 1000
    16:42:42 [INFO] train episode 367: reward = -22.23, steps = 1000
    16:43:00 [INFO] train episode 368: reward = -26.27, steps = 1000
    16:43:16 [INFO] train episode 369: reward = -16.67, steps = 1000
    16:43:34 [INFO] train episode 370: reward = -12.73, steps = 1000
    16:43:51 [INFO] train episode 371: reward = -13.88, steps = 1000
    16:44:08 [INFO] train episode 372: reward = -13.60, steps = 1000
    16:44:24 [INFO] train episode 373: reward = 20.07, steps = 1000
    16:44:43 [INFO] train episode 374: reward = -39.98, steps = 1000
    16:44:59 [INFO] train episode 375: reward = 1.83, steps = 1000
    16:45:16 [INFO] train episode 376: reward = 8.48, steps = 1000
    16:45:32 [INFO] train episode 377: reward = -35.34, steps = 1000
    16:45:49 [INFO] train episode 378: reward = -15.22, steps = 1000
    16:46:05 [INFO] train episode 379: reward = -11.67, steps = 1000
    16:46:07 [INFO] train episode 380: reward = -91.93, steps = 135
    16:46:23 [INFO] train episode 381: reward = 3.62, steps = 1000
    16:46:39 [INFO] train episode 382: reward = -17.76, steps = 1000
    16:46:56 [INFO] train episode 383: reward = -45.99, steps = 1000
    16:47:13 [INFO] train episode 384: reward = 14.77, steps = 1000
    16:47:29 [INFO] train episode 385: reward = 7.12, steps = 1000
    16:47:36 [INFO] train episode 386: reward = -497.05, steps = 434
    16:47:52 [INFO] train episode 387: reward = -8.96, steps = 1000
    16:48:10 [INFO] train episode 388: reward = -62.27, steps = 1000
    16:48:26 [INFO] train episode 389: reward = -9.82, steps = 1000
    16:48:43 [INFO] train episode 390: reward = -13.88, steps = 1000
    16:48:59 [INFO] train episode 391: reward = -50.66, steps = 1000
    16:49:17 [INFO] train episode 392: reward = 20.18, steps = 1000
    16:49:33 [INFO] train episode 393: reward = -36.30, steps = 1000
    16:49:51 [INFO] train episode 394: reward = -6.69, steps = 1000
    16:50:08 [INFO] train episode 395: reward = -41.34, steps = 1000
    16:50:24 [INFO] train episode 396: reward = 10.24, steps = 1000
    16:50:41 [INFO] train episode 397: reward = -3.72, steps = 1000
    16:50:59 [INFO] train episode 398: reward = 9.63, steps = 1000
    16:51:15 [INFO] train episode 399: reward = 3.34, steps = 1000
    16:51:33 [INFO] train episode 400: reward = -26.98, steps = 1000
    16:51:50 [INFO] train episode 401: reward = -10.55, steps = 1000
    16:52:08 [INFO] train episode 402: reward = -61.39, steps = 1000
    16:52:25 [INFO] train episode 403: reward = -3.59, steps = 1000
    16:52:41 [INFO] train episode 404: reward = -11.97, steps = 1000
    16:52:59 [INFO] train episode 405: reward = 18.61, steps = 1000
    16:53:15 [INFO] train episode 406: reward = -3.39, steps = 1000
    16:53:34 [INFO] train episode 407: reward = -15.20, steps = 1000
    16:53:51 [INFO] train episode 408: reward = -45.63, steps = 1000
    16:54:07 [INFO] train episode 409: reward = -18.62, steps = 1000
    16:54:24 [INFO] train episode 410: reward = 5.25, steps = 1000
    16:54:41 [INFO] train episode 411: reward = -11.82, steps = 1000
    16:54:58 [INFO] train episode 412: reward = 8.86, steps = 1000
    16:55:15 [INFO] train episode 413: reward = 1.93, steps = 1000
    16:55:33 [INFO] train episode 414: reward = -2.19, steps = 1000
    16:55:50 [INFO] train episode 415: reward = -9.91, steps = 1000
    16:56:08 [INFO] train episode 416: reward = -13.03, steps = 1000
    16:56:25 [INFO] train episode 417: reward = -60.89, steps = 1000
    16:56:42 [INFO] train episode 418: reward = -30.01, steps = 1000
    16:57:00 [INFO] train episode 419: reward = -22.01, steps = 1000
    16:57:17 [INFO] train episode 420: reward = -30.74, steps = 1000
    16:57:35 [INFO] train episode 421: reward = 25.21, steps = 1000
    16:57:52 [INFO] train episode 422: reward = -28.84, steps = 1000
    16:58:09 [INFO] train episode 423: reward = -16.14, steps = 1000
    16:58:26 [INFO] train episode 424: reward = -12.96, steps = 1000
    16:58:43 [INFO] train episode 425: reward = -6.68, steps = 1000
    16:59:00 [INFO] train episode 426: reward = -6.48, steps = 1000
    16:59:18 [INFO] train episode 427: reward = 20.64, steps = 1000
    16:59:35 [INFO] train episode 428: reward = -36.57, steps = 1000
    16:59:46 [INFO] train episode 429: reward = 179.74, steps = 641
    16:59:59 [INFO] train episode 430: reward = -258.31, steps = 750
    17:00:16 [INFO] train episode 431: reward = -37.56, steps = 1000
    17:00:33 [INFO] train episode 432: reward = -9.76, steps = 1000
    17:00:45 [INFO] train episode 433: reward = -380.50, steps = 759
    17:01:02 [INFO] train episode 434: reward = -113.16, steps = 1000
    17:01:11 [INFO] train episode 435: reward = 192.68, steps = 634
    17:01:28 [INFO] train episode 436: reward = -16.30, steps = 1000
    17:01:37 [INFO] train episode 437: reward = 225.65, steps = 604
    17:01:48 [INFO] train episode 438: reward = -92.96, steps = 670
    17:02:06 [INFO] train episode 439: reward = 0.91, steps = 1000
    17:02:14 [INFO] train episode 440: reward = -199.07, steps = 533
    17:02:27 [INFO] train episode 441: reward = 148.87, steps = 734
    17:02:41 [INFO] train episode 442: reward = 123.26, steps = 890
    17:02:58 [INFO] train episode 443: reward = -81.29, steps = 1000
    17:03:04 [INFO] train episode 444: reward = 197.81, steps = 401
    17:03:21 [INFO] train episode 445: reward = 12.10, steps = 1000
    17:03:32 [INFO] train episode 446: reward = 171.38, steps = 632
    17:03:48 [INFO] train episode 447: reward = 0.39, steps = 1000
    17:03:56 [INFO] train episode 448: reward = -50.86, steps = 515
    17:04:12 [INFO] train episode 449: reward = -50.66, steps = 1000
    17:04:25 [INFO] train episode 450: reward = -119.16, steps = 793
    17:04:36 [INFO] train episode 451: reward = -162.38, steps = 723
    17:04:47 [INFO] train episode 452: reward = 106.41, steps = 674
    17:05:04 [INFO] train episode 453: reward = 18.17, steps = 1000
    17:05:21 [INFO] train episode 454: reward = -76.30, steps = 1000
    17:05:37 [INFO] train episode 455: reward = -228.40, steps = 963
    17:05:54 [INFO] train episode 456: reward = -4.35, steps = 1000
    17:06:11 [INFO] train episode 457: reward = 118.99, steps = 997
    17:06:23 [INFO] train episode 458: reward = 171.42, steps = 770
    17:06:36 [INFO] train episode 459: reward = 170.09, steps = 784
    17:06:54 [INFO] train episode 460: reward = -48.00, steps = 1000
    17:07:01 [INFO] train episode 461: reward = 161.24, steps = 483
    17:07:08 [INFO] train episode 462: reward = -1011.54, steps = 434
    17:07:24 [INFO] train episode 463: reward = 27.88, steps = 1000
    17:07:40 [INFO] train episode 464: reward = -55.39, steps = 1000
    17:07:57 [INFO] train episode 465: reward = -11.84, steps = 1000
    17:08:13 [INFO] train episode 466: reward = -16.58, steps = 1000
    17:08:31 [INFO] train episode 467: reward = -48.79, steps = 1000
    17:08:44 [INFO] train episode 468: reward = 75.98, steps = 803
    17:09:01 [INFO] train episode 469: reward = 73.27, steps = 984
    17:09:18 [INFO] train episode 470: reward = 17.57, steps = 1000
    17:09:35 [INFO] train episode 471: reward = -40.56, steps = 1000
    17:09:46 [INFO] train episode 472: reward = 153.79, steps = 692
    17:10:03 [INFO] train episode 473: reward = -31.45, steps = 1000
    17:10:21 [INFO] train episode 474: reward = -11.71, steps = 1000
    17:10:31 [INFO] train episode 475: reward = 173.09, steps = 627
    17:10:47 [INFO] train episode 476: reward = -41.60, steps = 1000
    17:10:56 [INFO] train episode 477: reward = 189.14, steps = 554
    17:11:12 [INFO] train episode 478: reward = -28.12, steps = 1000
    17:11:29 [INFO] train episode 479: reward = 33.71, steps = 1000
    17:11:37 [INFO] train episode 480: reward = 248.13, steps = 464
    17:11:47 [INFO] train episode 481: reward = -178.47, steps = 601
    17:12:04 [INFO] train episode 482: reward = 34.76, steps = 1000
    17:12:10 [INFO] train episode 483: reward = 222.54, steps = 386
    17:12:27 [INFO] train episode 484: reward = -5.07, steps = 1000
    17:12:34 [INFO] train episode 485: reward = 237.54, steps = 418
    17:12:51 [INFO] train episode 486: reward = -422.23, steps = 986
    17:13:09 [INFO] train episode 487: reward = 0.66, steps = 1000
    17:13:25 [INFO] train episode 488: reward = 22.06, steps = 1000
    17:13:42 [INFO] train episode 489: reward = 4.54, steps = 1000
    17:13:59 [INFO] train episode 490: reward = -49.61, steps = 1000
    17:14:16 [INFO] train episode 491: reward = -24.64, steps = 1000
    17:14:23 [INFO] train episode 492: reward = 227.28, steps = 411
    17:14:40 [INFO] train episode 493: reward = -62.41, steps = 1000
    17:14:53 [INFO] train episode 494: reward = 140.25, steps = 790
    17:15:10 [INFO] train episode 495: reward = 7.08, steps = 1000
    17:15:28 [INFO] train episode 496: reward = 37.71, steps = 1000
    17:15:45 [INFO] train episode 497: reward = 2.57, steps = 1000
    17:15:51 [INFO] train episode 498: reward = 238.87, steps = 400
    17:16:08 [INFO] train episode 499: reward = -4.74, steps = 1000
    17:16:17 [INFO] train episode 500: reward = 215.58, steps = 544
    17:16:34 [INFO] train episode 501: reward = 26.46, steps = 1000
    17:16:51 [INFO] train episode 502: reward = 20.71, steps = 1000
    17:17:09 [INFO] train episode 503: reward = 23.39, steps = 1000
    17:17:26 [INFO] train episode 504: reward = 10.38, steps = 1000
    17:17:35 [INFO] train episode 505: reward = 212.92, steps = 620
    17:17:42 [INFO] train episode 506: reward = 214.52, steps = 421
    17:17:47 [INFO] train episode 507: reward = -604.26, steps = 304
    17:17:51 [INFO] train episode 508: reward = 220.46, steps = 304
    17:18:07 [INFO] train episode 509: reward = 19.86, steps = 1000
    17:18:24 [INFO] train episode 510: reward = 69.63, steps = 1000
    17:18:42 [INFO] train episode 511: reward = 39.31, steps = 1000
    17:18:53 [INFO] train episode 512: reward = 177.56, steps = 684
    17:18:55 [INFO] train episode 513: reward = -302.84, steps = 152
    17:19:12 [INFO] train episode 514: reward = 18.61, steps = 1000
    17:19:14 [INFO] train episode 515: reward = -466.55, steps = 125
    17:19:20 [INFO] train episode 516: reward = -415.81, steps = 408
    17:19:23 [INFO] train episode 517: reward = -340.35, steps = 184
    17:19:38 [INFO] train episode 518: reward = -458.65, steps = 926
    17:19:42 [INFO] train episode 519: reward = 281.32, steps = 289
    17:19:59 [INFO] train episode 520: reward = 5.11, steps = 1000
    17:20:17 [INFO] train episode 521: reward = 39.16, steps = 1000
    17:20:33 [INFO] train episode 522: reward = -30.10, steps = 1000
    17:20:50 [INFO] train episode 523: reward = 21.39, steps = 1000
    17:21:07 [INFO] train episode 524: reward = 26.32, steps = 1000
    17:21:09 [INFO] train episode 525: reward = -57.20, steps = 135
    17:21:16 [INFO] train episode 526: reward = -432.65, steps = 437
    17:21:18 [INFO] train episode 527: reward = -295.68, steps = 102
    17:21:19 [INFO] train episode 528: reward = -414.96, steps = 108
    17:21:24 [INFO] train episode 529: reward = 236.73, steps = 304
    17:21:41 [INFO] train episode 530: reward = -12.81, steps = 1000
    17:21:44 [INFO] train episode 531: reward = -136.83, steps = 200
    17:22:01 [INFO] train episode 532: reward = 126.82, steps = 932
    17:22:19 [INFO] train episode 533: reward = -967.64, steps = 999
    17:22:36 [INFO] train episode 534: reward = 64.86, steps = 1000
    17:22:53 [INFO] train episode 535: reward = 85.45, steps = 1000
    17:23:09 [INFO] train episode 536: reward = -13.24, steps = 1000
    17:23:27 [INFO] train episode 537: reward = 41.51, steps = 1000
    17:23:44 [INFO] train episode 538: reward = 35.68, steps = 1000
    17:24:01 [INFO] train episode 539: reward = -11.86, steps = 1000
    17:24:09 [INFO] train episode 540: reward = -549.14, steps = 509
    17:24:10 [INFO] train episode 541: reward = -53.61, steps = 74
    17:24:28 [INFO] train episode 542: reward = -25.94, steps = 1000
    17:24:45 [INFO] train episode 543: reward = -13.76, steps = 1000
    17:24:52 [INFO] train episode 544: reward = -610.65, steps = 391
    17:25:07 [INFO] train episode 545: reward = -133.54, steps = 905
    17:25:25 [INFO] train episode 546: reward = -2.68, steps = 1000
    17:25:42 [INFO] train episode 547: reward = 32.10, steps = 1000
    17:25:43 [INFO] train episode 548: reward = 22.49, steps = 96
    17:25:45 [INFO] train episode 549: reward = 33.30, steps = 102
    17:26:02 [INFO] train episode 550: reward = -39.32, steps = 1000
    17:26:19 [INFO] train episode 551: reward = -37.75, steps = 1000
    17:26:38 [INFO] train episode 552: reward = 30.53, steps = 1000
    17:26:55 [INFO] train episode 553: reward = -15.91, steps = 1000
    17:27:12 [INFO] train episode 554: reward = -4.25, steps = 1000
    17:27:29 [INFO] train episode 555: reward = -6.59, steps = 1000
    17:27:46 [INFO] train episode 556: reward = 28.12, steps = 1000
    17:28:03 [INFO] train episode 557: reward = 41.68, steps = 1000
    17:28:20 [INFO] train episode 558: reward = 71.26, steps = 1000
    17:28:37 [INFO] train episode 559: reward = 22.88, steps = 1000
    17:28:52 [INFO] train episode 560: reward = 195.52, steps = 892
    17:29:09 [INFO] train episode 561: reward = 24.32, steps = 1000
    17:29:26 [INFO] train episode 562: reward = -19.14, steps = 1000
    17:29:44 [INFO] train episode 563: reward = 56.24, steps = 1000
    17:30:01 [INFO] train episode 564: reward = 85.08, steps = 1000
    17:30:15 [INFO] train episode 565: reward = 174.13, steps = 824
    17:30:17 [INFO] train episode 566: reward = -178.52, steps = 168
    17:30:35 [INFO] train episode 567: reward = -2.73, steps = 1000
    17:30:52 [INFO] train episode 568: reward = 37.31, steps = 1000
    17:30:59 [INFO] train episode 569: reward = 215.90, steps = 448
    17:31:16 [INFO] train episode 570: reward = -5.80, steps = 1000
    17:31:17 [INFO] train episode 571: reward = -51.08, steps = 84
    17:31:34 [INFO] train episode 572: reward = 37.96, steps = 1000
    17:31:52 [INFO] train episode 573: reward = 38.76, steps = 1000
    17:32:08 [INFO] train episode 574: reward = 12.21, steps = 1000
    17:32:10 [INFO] train episode 575: reward = -82.36, steps = 96
    17:32:16 [INFO] train episode 576: reward = 217.38, steps = 405
    17:32:25 [INFO] train episode 577: reward = 203.25, steps = 565
    17:32:36 [INFO] train episode 578: reward = 221.50, steps = 640
    17:32:38 [INFO] train episode 579: reward = 76.11, steps = 159
    17:32:55 [INFO] train episode 580: reward = 132.48, steps = 977
    17:33:05 [INFO] train episode 581: reward = 219.43, steps = 614
    17:33:21 [INFO] train episode 582: reward = -9.71, steps = 1000
    17:33:27 [INFO] train episode 583: reward = 240.53, steps = 365
    17:33:44 [INFO] train episode 584: reward = 2.31, steps = 1000
    17:33:45 [INFO] train episode 585: reward = 32.22, steps = 89
    17:33:47 [INFO] train episode 586: reward = -27.33, steps = 86
    17:33:51 [INFO] train episode 587: reward = 258.94, steps = 308
    17:34:00 [INFO] train episode 588: reward = 241.40, steps = 566
    17:34:18 [INFO] train episode 589: reward = 59.84, steps = 1000
    17:34:20 [INFO] train episode 590: reward = -59.60, steps = 108
    17:34:21 [INFO] train episode 591: reward = -34.47, steps = 57
    17:34:37 [INFO] train episode 592: reward = 31.24, steps = 1000
    17:34:51 [INFO] train episode 593: reward = 187.36, steps = 800
    17:35:03 [INFO] train episode 594: reward = 197.85, steps = 713
    17:35:21 [INFO] train episode 595: reward = 10.25, steps = 1000
    17:35:38 [INFO] train episode 596: reward = 17.86, steps = 1000
    17:35:55 [INFO] train episode 597: reward = 41.72, steps = 1000
    17:35:57 [INFO] train episode 598: reward = 2.79, steps = 121
    17:36:14 [INFO] train episode 599: reward = 53.77, steps = 1000
    17:36:31 [INFO] train episode 600: reward = -1.95, steps = 1000
    17:36:41 [INFO] train episode 601: reward = 207.04, steps = 608
    17:36:58 [INFO] train episode 602: reward = 42.99, steps = 1000
    17:37:16 [INFO] train episode 603: reward = 25.57, steps = 1000
    17:37:24 [INFO] train episode 604: reward = 202.82, steps = 509
    17:37:40 [INFO] train episode 605: reward = 45.00, steps = 1000
    17:37:58 [INFO] train episode 606: reward = 38.45, steps = 1000
    17:38:15 [INFO] train episode 607: reward = 41.12, steps = 1000
    17:38:16 [INFO] train episode 608: reward = -14.02, steps = 106
    17:38:33 [INFO] train episode 609: reward = 20.66, steps = 1000
    17:38:51 [INFO] train episode 610: reward = 21.48, steps = 1000
    17:39:04 [INFO] train episode 611: reward = 175.96, steps = 788
    17:39:21 [INFO] train episode 612: reward = 51.15, steps = 1000
    17:39:38 [INFO] train episode 613: reward = 1.04, steps = 1000
    17:39:55 [INFO] train episode 614: reward = 114.43, steps = 1000
    17:40:00 [INFO] train episode 615: reward = 238.75, steps = 321
    17:40:17 [INFO] train episode 616: reward = 105.67, steps = 1000
    17:40:33 [INFO] train episode 617: reward = 109.79, steps = 1000
    17:40:49 [INFO] train episode 618: reward = 176.95, steps = 990
    17:41:06 [INFO] train episode 619: reward = 54.18, steps = 1000
    17:41:17 [INFO] train episode 620: reward = 214.27, steps = 640
    17:41:23 [INFO] train episode 621: reward = 204.66, steps = 389
    17:41:25 [INFO] train episode 622: reward = 7.88, steps = 119
    17:41:41 [INFO] train episode 623: reward = 165.24, steps = 1000
    17:41:57 [INFO] train episode 624: reward = 167.14, steps = 1000
    17:41:58 [INFO] train episode 625: reward = 7.88, steps = 72
    17:42:00 [INFO] train episode 626: reward = -128.61, steps = 91
    17:42:01 [INFO] train episode 627: reward = -16.84, steps = 74
    17:42:17 [INFO] train episode 628: reward = 155.34, steps = 1000
    17:42:33 [INFO] train episode 629: reward = 100.81, steps = 1000
    17:42:35 [INFO] train episode 630: reward = -115.57, steps = 117
    17:42:51 [INFO] train episode 631: reward = 138.67, steps = 1000
    17:43:09 [INFO] train episode 632: reward = 14.39, steps = 1000
    17:43:26 [INFO] train episode 633: reward = 102.26, steps = 1000
    17:43:40 [INFO] train episode 634: reward = 181.22, steps = 861
    17:43:56 [INFO] train episode 635: reward = 164.01, steps = 1000
    17:44:01 [INFO] train episode 636: reward = 235.82, steps = 343
    17:44:17 [INFO] train episode 637: reward = 162.08, steps = 1000
    17:44:19 [INFO] train episode 638: reward = 10.27, steps = 128
    17:44:36 [INFO] train episode 639: reward = 117.35, steps = 1000
    17:44:38 [INFO] train episode 640: reward = 53.56, steps = 111
    17:44:47 [INFO] train episode 641: reward = 256.07, steps = 595
    17:45:06 [INFO] train episode 642: reward = 101.01, steps = 1000
    17:45:23 [INFO] train episode 643: reward = 122.99, steps = 1000
    17:45:25 [INFO] train episode 644: reward = 73.47, steps = 120
    17:45:42 [INFO] train episode 645: reward = 142.42, steps = 1000
    17:45:44 [INFO] train episode 646: reward = 66.60, steps = 104
    17:45:53 [INFO] train episode 647: reward = 220.20, steps = 564
    17:46:10 [INFO] train episode 648: reward = 127.84, steps = 1000
    17:46:26 [INFO] train episode 649: reward = 120.23, steps = 1000
    17:46:33 [INFO] train episode 650: reward = 223.96, steps = 461
    17:46:49 [INFO] train episode 651: reward = 154.65, steps = 1000
    17:47:06 [INFO] train episode 652: reward = 143.88, steps = 1000
    17:47:22 [INFO] train episode 653: reward = 150.57, steps = 1000
    17:47:38 [INFO] train episode 654: reward = 27.69, steps = 1000
    17:47:42 [INFO] train episode 655: reward = -43.86, steps = 229
    17:47:57 [INFO] train episode 656: reward = 89.57, steps = 1000
    17:48:14 [INFO] train episode 657: reward = 168.64, steps = 1000
    17:48:24 [INFO] train episode 658: reward = 207.00, steps = 642
    17:48:40 [INFO] train episode 659: reward = 182.23, steps = 1000
    17:48:57 [INFO] train episode 660: reward = 115.71, steps = 1000
    17:49:01 [INFO] train episode 661: reward = -160.52, steps = 281
    17:49:17 [INFO] train episode 662: reward = 157.85, steps = 1000
    17:49:33 [INFO] train episode 663: reward = 130.31, steps = 1000
    17:49:49 [INFO] train episode 664: reward = 130.32, steps = 1000
    17:49:56 [INFO] train episode 665: reward = 277.95, steps = 410
    17:50:12 [INFO] train episode 666: reward = 154.16, steps = 1000
    17:50:18 [INFO] train episode 667: reward = 196.92, steps = 423
    17:50:34 [INFO] train episode 668: reward = 135.52, steps = 1000
    17:50:50 [INFO] train episode 669: reward = 106.39, steps = 1000
    17:50:52 [INFO] train episode 670: reward = -24.90, steps = 104
    17:51:08 [INFO] train episode 671: reward = 154.71, steps = 1000
    17:51:15 [INFO] train episode 672: reward = 226.01, steps = 468
    17:51:31 [INFO] train episode 673: reward = 132.32, steps = 1000
    17:51:47 [INFO] train episode 674: reward = 138.69, steps = 1000
    17:52:00 [INFO] train episode 675: reward = 232.37, steps = 797
    17:52:13 [INFO] train episode 676: reward = 249.46, steps = 817
    17:52:29 [INFO] train episode 677: reward = 112.30, steps = 1000
    17:52:45 [INFO] train episode 678: reward = 104.20, steps = 1000
    17:52:51 [INFO] train episode 679: reward = -187.01, steps = 379
    17:53:08 [INFO] train episode 680: reward = 136.29, steps = 1000
    17:53:13 [INFO] train episode 681: reward = -486.16, steps = 360
    17:53:20 [INFO] train episode 682: reward = -749.46, steps = 446
    17:53:36 [INFO] train episode 683: reward = 109.68, steps = 1000
    17:53:52 [INFO] train episode 684: reward = 154.85, steps = 1000
    17:54:09 [INFO] train episode 685: reward = 156.93, steps = 1000
    17:54:12 [INFO] train episode 686: reward = -406.34, steps = 210
    17:54:17 [INFO] train episode 687: reward = -401.19, steps = 300
    17:54:21 [INFO] train episode 688: reward = -82.41, steps = 291
    17:54:37 [INFO] train episode 689: reward = 115.55, steps = 1000
    17:54:54 [INFO] train episode 690: reward = 121.94, steps = 1000
    17:55:02 [INFO] train episode 691: reward = 183.99, steps = 519
    17:55:19 [INFO] train episode 692: reward = 76.99, steps = 1000
    17:55:37 [INFO] train episode 693: reward = 57.91, steps = 1000
    17:55:53 [INFO] train episode 694: reward = 123.04, steps = 1000
    17:56:10 [INFO] train episode 695: reward = 38.86, steps = 1000
    17:56:25 [INFO] train episode 696: reward = 134.54, steps = 1000
    17:56:32 [INFO] train episode 697: reward = -130.50, steps = 430
    17:56:49 [INFO] train episode 698: reward = 124.60, steps = 1000
    17:56:53 [INFO] train episode 699: reward = 267.28, steps = 276
    17:56:59 [INFO] train episode 700: reward = 209.36, steps = 389
    17:57:15 [INFO] train episode 701: reward = 159.69, steps = 1000
    17:57:18 [INFO] train episode 702: reward = -7.29, steps = 189
    17:57:25 [INFO] train episode 703: reward = 222.08, steps = 413
    17:57:36 [INFO] train episode 704: reward = 232.52, steps = 686
    17:57:40 [INFO] train episode 705: reward = 10.55, steps = 215
    17:57:57 [INFO] train episode 706: reward = 142.65, steps = 1000
    17:58:14 [INFO] train episode 707: reward = 171.00, steps = 1000
    17:58:32 [INFO] train episode 708: reward = 128.44, steps = 1000
    17:58:36 [INFO] train episode 709: reward = 4.20, steps = 241
    17:58:56 [INFO] train episode 710: reward = -51.23, steps = 1000
    17:59:02 [INFO] train episode 711: reward = 244.43, steps = 382
    17:59:09 [INFO] train episode 712: reward = -198.80, steps = 386
    17:59:26 [INFO] train episode 713: reward = 120.89, steps = 1000
    17:59:43 [INFO] train episode 714: reward = 89.99, steps = 1000
    17:59:47 [INFO] train episode 715: reward = 217.96, steps = 308
    18:00:00 [INFO] train episode 716: reward = 127.40, steps = 741
    18:00:08 [INFO] train episode 717: reward = 256.36, steps = 516
    18:00:22 [INFO] train episode 718: reward = 177.99, steps = 759
    18:00:33 [INFO] train episode 719: reward = 247.88, steps = 732
    18:00:51 [INFO] train episode 720: reward = 130.01, steps = 1000
    18:00:54 [INFO] train episode 721: reward = -313.62, steps = 177
    18:01:04 [INFO] train episode 722: reward = 250.17, steps = 519
    18:01:24 [INFO] train episode 723: reward = 80.82, steps = 1000
    18:01:28 [INFO] train episode 724: reward = 40.78, steps = 188
    18:01:48 [INFO] train episode 725: reward = 99.39, steps = 1000
    18:02:06 [INFO] train episode 726: reward = 143.09, steps = 1000
    18:02:25 [INFO] train episode 727: reward = 153.73, steps = 1000
    18:02:42 [INFO] train episode 728: reward = 147.26, steps = 1000
    18:02:52 [INFO] train episode 729: reward = 238.17, steps = 618
    18:03:09 [INFO] train episode 730: reward = 79.20, steps = 1000
    18:03:27 [INFO] train episode 731: reward = 24.03, steps = 1000
    18:03:31 [INFO] train episode 732: reward = 242.16, steps = 235
    18:03:47 [INFO] train episode 733: reward = 78.30, steps = 1000
    18:04:03 [INFO] train episode 734: reward = 97.28, steps = 1000
    18:04:20 [INFO] train episode 735: reward = 90.98, steps = 1000
    18:04:38 [INFO] train episode 736: reward = 79.91, steps = 1000
    18:04:50 [INFO] train episode 737: reward = -159.03, steps = 692
    18:05:08 [INFO] train episode 738: reward = 85.60, steps = 1000
    18:05:10 [INFO] train episode 739: reward = 43.50, steps = 171
    18:05:13 [INFO] train episode 740: reward = 30.27, steps = 171
    18:05:16 [INFO] train episode 741: reward = -454.75, steps = 188
    18:05:17 [INFO] train episode 742: reward = -212.17, steps = 70
    18:05:33 [INFO] train episode 743: reward = 219.78, steps = 982
    18:05:50 [INFO] train episode 744: reward = 149.90, steps = 1000
    18:06:07 [INFO] train episode 745: reward = 128.33, steps = 1000
    18:06:15 [INFO] train episode 746: reward = 208.50, steps = 468
    18:06:20 [INFO] train episode 747: reward = -88.29, steps = 306
    18:06:23 [INFO] train episode 748: reward = 255.11, steps = 195
    18:06:36 [INFO] train episode 749: reward = 231.50, steps = 765
    18:06:38 [INFO] train episode 750: reward = 33.01, steps = 145
    18:06:56 [INFO] train episode 751: reward = -55.89, steps = 1000
    18:07:01 [INFO] train episode 752: reward = -7.38, steps = 328
    18:07:06 [INFO] train episode 753: reward = -133.72, steps = 330
    18:07:22 [INFO] train episode 754: reward = 251.94, steps = 940
    18:07:38 [INFO] train episode 755: reward = 138.53, steps = 1000
    18:07:40 [INFO] train episode 756: reward = 11.25, steps = 119
    18:07:56 [INFO] train episode 757: reward = 151.37, steps = 1000
    18:08:04 [INFO] train episode 758: reward = 241.25, steps = 475
    18:08:21 [INFO] train episode 759: reward = -39.54, steps = 1000
    18:08:37 [INFO] train episode 760: reward = 144.65, steps = 1000
    18:08:49 [INFO] train episode 761: reward = 211.23, steps = 696
    18:09:06 [INFO] train episode 762: reward = 108.09, steps = 1000
    18:09:23 [INFO] train episode 763: reward = -86.33, steps = 1000
    18:09:34 [INFO] train episode 764: reward = -99.00, steps = 704
    18:09:45 [INFO] train episode 765: reward = 184.86, steps = 652
    18:09:53 [INFO] train episode 766: reward = 256.11, steps = 444
    18:09:56 [INFO] train episode 767: reward = 263.82, steps = 220
    18:10:13 [INFO] train episode 768: reward = 158.73, steps = 1000
    18:10:30 [INFO] train episode 769: reward = 79.91, steps = 1000
    18:10:48 [INFO] train episode 770: reward = 222.30, steps = 982
    18:10:52 [INFO] train episode 771: reward = 294.06, steps = 263
    18:11:11 [INFO] train episode 772: reward = 130.29, steps = 1000
    18:11:14 [INFO] train episode 773: reward = -66.40, steps = 179
    18:11:30 [INFO] train episode 774: reward = 102.93, steps = 1000
    18:11:48 [INFO] train episode 775: reward = 115.97, steps = 1000
    18:12:05 [INFO] train episode 776: reward = 107.86, steps = 1000
    18:12:19 [INFO] train episode 777: reward = 254.59, steps = 793
    18:12:35 [INFO] train episode 778: reward = 104.27, steps = 1000
    18:12:52 [INFO] train episode 779: reward = 158.35, steps = 1000
    18:12:59 [INFO] train episode 780: reward = 227.04, steps = 399
    18:13:16 [INFO] train episode 781: reward = 171.72, steps = 1000
    18:13:21 [INFO] train episode 782: reward = 252.99, steps = 329
    18:13:36 [INFO] train episode 783: reward = 261.01, steps = 894
    18:13:52 [INFO] train episode 784: reward = 255.66, steps = 925
    18:14:09 [INFO] train episode 785: reward = 133.93, steps = 1000
    18:14:26 [INFO] train episode 786: reward = 155.62, steps = 1000
    18:14:43 [INFO] train episode 787: reward = 141.15, steps = 1000
    18:14:49 [INFO] train episode 788: reward = 234.30, steps = 377
    18:14:54 [INFO] train episode 789: reward = 253.35, steps = 258
    18:15:12 [INFO] train episode 790: reward = 84.78, steps = 1000
    18:15:30 [INFO] train episode 791: reward = 117.51, steps = 1000
    18:15:47 [INFO] train episode 792: reward = 93.79, steps = 1000
    18:15:49 [INFO] train episode 793: reward = -185.16, steps = 129
    18:16:05 [INFO] train episode 794: reward = 162.98, steps = 1000
    18:16:09 [INFO] train episode 795: reward = 256.94, steps = 258
    18:16:25 [INFO] train episode 796: reward = 134.97, steps = 1000
    18:16:26 [INFO] train episode 797: reward = -231.05, steps = 79
    18:16:43 [INFO] train episode 798: reward = 151.98, steps = 1000
    18:16:59 [INFO] train episode 799: reward = 130.34, steps = 1000
    18:17:15 [INFO] train episode 800: reward = 154.68, steps = 1000
    18:17:19 [INFO] train episode 801: reward = -170.08, steps = 214
    18:17:35 [INFO] train episode 802: reward = 81.76, steps = 1000
    18:17:51 [INFO] train episode 803: reward = 140.37, steps = 1000
    18:17:58 [INFO] train episode 804: reward = 215.17, steps = 409
    18:18:01 [INFO] train episode 805: reward = 278.40, steps = 211
    18:18:18 [INFO] train episode 806: reward = 93.31, steps = 1000
    18:18:34 [INFO] train episode 807: reward = 166.68, steps = 1000
    18:18:50 [INFO] train episode 808: reward = 132.64, steps = 1000
    18:18:54 [INFO] train episode 809: reward = 240.51, steps = 272
    18:18:58 [INFO] train episode 810: reward = 251.70, steps = 217
    18:19:14 [INFO] train episode 811: reward = 127.53, steps = 1000
    18:19:31 [INFO] train episode 812: reward = 95.43, steps = 1000
    18:19:48 [INFO] train episode 813: reward = 135.73, steps = 1000
    18:19:58 [INFO] train episode 814: reward = 268.12, steps = 661
    18:20:15 [INFO] train episode 815: reward = 147.78, steps = 1000
    18:20:31 [INFO] train episode 816: reward = 170.47, steps = 1000
    18:20:48 [INFO] train episode 817: reward = 130.99, steps = 1000
    18:20:52 [INFO] train episode 818: reward = 262.83, steps = 274
    18:21:09 [INFO] train episode 819: reward = 110.39, steps = 1000
    18:21:15 [INFO] train episode 820: reward = 239.54, steps = 378
    18:21:31 [INFO] train episode 821: reward = 133.82, steps = 1000
    18:21:48 [INFO] train episode 822: reward = 137.69, steps = 1000
    18:22:05 [INFO] train episode 823: reward = 149.98, steps = 1000
    18:22:21 [INFO] train episode 824: reward = 174.66, steps = 1000
    18:22:23 [INFO] train episode 825: reward = 23.42, steps = 111
    18:22:29 [INFO] train episode 826: reward = 256.38, steps = 376
    18:22:43 [INFO] train episode 827: reward = 170.66, steps = 1000
    18:22:58 [INFO] train episode 828: reward = 144.07, steps = 1000
    18:23:12 [INFO] train episode 829: reward = 141.22, steps = 1000
    18:23:15 [INFO] train episode 830: reward = 268.09, steps = 214
    18:23:21 [INFO] train episode 831: reward = 197.00, steps = 489
    18:23:28 [INFO] train episode 832: reward = 248.00, steps = 492
    18:23:42 [INFO] train episode 833: reward = 125.99, steps = 1000
    18:23:56 [INFO] train episode 834: reward = 136.15, steps = 1000
    18:23:59 [INFO] train episode 835: reward = 269.43, steps = 251
    18:24:14 [INFO] train episode 836: reward = 125.02, steps = 1000
    18:24:28 [INFO] train episode 837: reward = 110.55, steps = 1000
    18:24:41 [INFO] train episode 838: reward = 129.61, steps = 1000
    18:24:44 [INFO] train episode 839: reward = 271.42, steps = 227
    18:24:46 [INFO] train episode 840: reward = 19.81, steps = 139
    18:25:00 [INFO] train episode 841: reward = 131.76, steps = 1000
    18:25:06 [INFO] train episode 842: reward = 294.54, steps = 406
    18:25:20 [INFO] train episode 843: reward = 104.78, steps = 1000
    18:25:34 [INFO] train episode 844: reward = 99.10, steps = 1000
    18:25:48 [INFO] train episode 845: reward = 115.60, steps = 1000
    18:26:02 [INFO] train episode 846: reward = 140.14, steps = 1000
    18:26:17 [INFO] train episode 847: reward = 145.72, steps = 1000
    18:26:31 [INFO] train episode 848: reward = 165.32, steps = 1000
    18:26:46 [INFO] train episode 849: reward = 132.30, steps = 1000
    18:27:01 [INFO] train episode 850: reward = 139.42, steps = 1000
    18:27:05 [INFO] train episode 851: reward = -866.85, steps = 260
    18:27:11 [INFO] train episode 852: reward = -1546.72, steps = 403
    18:27:16 [INFO] train episode 853: reward = -161.36, steps = 358
    18:27:21 [INFO] train episode 854: reward = 212.41, steps = 378
    18:27:22 [INFO] train episode 855: reward = -604.72, steps = 92
    18:27:30 [INFO] train episode 856: reward = 255.88, steps = 556
    18:27:32 [INFO] train episode 857: reward = 4.19, steps = 188
    18:27:34 [INFO] train episode 858: reward = 261.39, steps = 156
    18:27:36 [INFO] train episode 859: reward = -7.95, steps = 86
    18:27:44 [INFO] train episode 860: reward = 218.55, steps = 613
    18:27:47 [INFO] train episode 861: reward = 289.51, steps = 241
    18:27:50 [INFO] train episode 862: reward = 246.36, steps = 255
    18:27:51 [INFO] train episode 863: reward = 26.10, steps = 75
    18:28:05 [INFO] train episode 864: reward = 157.62, steps = 1000
    18:28:19 [INFO] train episode 865: reward = 115.23, steps = 1000
    18:28:22 [INFO] train episode 866: reward = -328.90, steps = 274
    18:28:26 [INFO] train episode 867: reward = 230.80, steps = 280
    18:28:32 [INFO] train episode 868: reward = 259.59, steps = 456
    18:28:36 [INFO] train episode 869: reward = -585.37, steps = 340
    18:28:38 [INFO] train episode 870: reward = 28.53, steps = 149
    18:28:41 [INFO] train episode 871: reward = 284.25, steps = 244
    18:28:55 [INFO] train episode 872: reward = 137.13, steps = 1000
    18:28:56 [INFO] train episode 873: reward = -52.76, steps = 75
    18:29:00 [INFO] train episode 874: reward = -44.45, steps = 263
    18:29:02 [INFO] train episode 875: reward = 53.58, steps = 148
    18:29:16 [INFO] train episode 876: reward = 113.73, steps = 1000
    18:29:18 [INFO] train episode 877: reward = -65.56, steps = 218
    18:29:19 [INFO] train episode 878: reward = -87.05, steps = 68
    18:29:24 [INFO] train episode 879: reward = 253.74, steps = 325
    18:29:30 [INFO] train episode 880: reward = -179.31, steps = 483
    18:29:33 [INFO] train episode 881: reward = 249.55, steps = 205
    18:29:35 [INFO] train episode 882: reward = -5.75, steps = 202
    18:29:40 [INFO] train episode 883: reward = 238.89, steps = 340
    18:29:46 [INFO] train episode 884: reward = 228.81, steps = 456
    18:29:47 [INFO] train episode 885: reward = 13.06, steps = 115
    18:29:56 [INFO] train episode 886: reward = 243.02, steps = 598
    18:30:01 [INFO] train episode 887: reward = 209.74, steps = 400
    18:30:04 [INFO] train episode 888: reward = 279.38, steps = 234
    18:30:18 [INFO] train episode 889: reward = 128.43, steps = 1000
    18:30:33 [INFO] train episode 890: reward = 85.24, steps = 1000
    18:30:48 [INFO] train episode 891: reward = 111.26, steps = 1000
    18:31:03 [INFO] train episode 892: reward = 157.65, steps = 1000
    18:31:11 [INFO] train episode 893: reward = -24.41, steps = 577
    18:31:19 [INFO] train episode 894: reward = 218.77, steps = 587
    18:31:23 [INFO] train episode 895: reward = 218.10, steps = 283
    18:31:37 [INFO] train episode 896: reward = 108.42, steps = 1000
    18:31:46 [INFO] train episode 897: reward = -63.21, steps = 600
    18:31:51 [INFO] train episode 898: reward = 192.27, steps = 346
    18:31:57 [INFO] train episode 899: reward = 248.43, steps = 459
    18:32:01 [INFO] train episode 900: reward = 269.86, steps = 267
    18:32:03 [INFO] train episode 901: reward = 63.01, steps = 141
    18:32:09 [INFO] train episode 902: reward = 250.85, steps = 484
    18:32:12 [INFO] train episode 903: reward = -231.04, steps = 230
    18:32:18 [INFO] train episode 904: reward = 231.72, steps = 440
    18:32:29 [INFO] train episode 905: reward = 225.02, steps = 811
    18:32:35 [INFO] train episode 906: reward = 253.08, steps = 421
    18:32:49 [INFO] train episode 907: reward = 130.34, steps = 1000
    18:32:59 [INFO] train episode 908: reward = 213.74, steps = 704
    18:33:14 [INFO] train episode 909: reward = 91.86, steps = 1000
    18:33:18 [INFO] train episode 910: reward = 295.56, steps = 299
    18:33:23 [INFO] train episode 911: reward = 196.36, steps = 395
    18:33:38 [INFO] train episode 912: reward = 164.99, steps = 1000
    18:33:52 [INFO] train episode 913: reward = 131.58, steps = 1000
    18:34:06 [INFO] train episode 914: reward = 135.81, steps = 1000
    18:34:20 [INFO] train episode 915: reward = 152.15, steps = 1000
    18:34:34 [INFO] train episode 916: reward = 183.03, steps = 1000
    18:34:48 [INFO] train episode 917: reward = 133.81, steps = 1000
    18:35:02 [INFO] train episode 918: reward = 130.18, steps = 1000
    18:35:09 [INFO] train episode 919: reward = 254.73, steps = 509
    18:35:23 [INFO] train episode 920: reward = 109.03, steps = 1000
    18:35:38 [INFO] train episode 921: reward = 166.15, steps = 1000
    18:35:52 [INFO] train episode 922: reward = 142.53, steps = 1000
    18:36:07 [INFO] train episode 923: reward = 142.92, steps = 1000
    18:36:22 [INFO] train episode 924: reward = 167.44, steps = 1000
    18:36:30 [INFO] train episode 925: reward = 188.15, steps = 505
    18:36:43 [INFO] train episode 926: reward = 215.21, steps = 874
    18:36:58 [INFO] train episode 927: reward = 143.08, steps = 1000
    18:37:13 [INFO] train episode 928: reward = 112.91, steps = 1000
    18:37:28 [INFO] train episode 929: reward = 81.78, steps = 1000
    18:37:42 [INFO] train episode 930: reward = 140.03, steps = 1000
    18:37:55 [INFO] train episode 931: reward = 163.42, steps = 1000
    18:38:09 [INFO] train episode 932: reward = 98.82, steps = 1000
    18:38:23 [INFO] train episode 933: reward = 120.41, steps = 1000
    18:38:37 [INFO] train episode 934: reward = 183.47, steps = 1000
    18:38:52 [INFO] train episode 935: reward = 137.67, steps = 1000
    18:38:59 [INFO] train episode 936: reward = 266.40, steps = 543
    18:39:03 [INFO] train episode 937: reward = 297.45, steps = 288
    18:39:07 [INFO] train episode 938: reward = 260.39, steps = 280
    18:39:15 [INFO] train episode 939: reward = 207.47, steps = 613
    18:39:17 [INFO] train episode 940: reward = -28.38, steps = 104
    18:39:22 [INFO] train episode 941: reward = 257.70, steps = 377
    18:39:28 [INFO] train episode 942: reward = 254.55, steps = 472
    18:39:42 [INFO] train episode 943: reward = 149.70, steps = 1000
    18:39:56 [INFO] train episode 944: reward = 145.37, steps = 1000
    18:40:01 [INFO] train episode 945: reward = 258.16, steps = 300
    18:40:03 [INFO] train episode 946: reward = 72.58, steps = 151
    18:40:11 [INFO] train episode 947: reward = 265.94, steps = 506
    18:40:14 [INFO] train episode 948: reward = 287.80, steps = 271
    18:40:19 [INFO] train episode 949: reward = 271.69, steps = 364
    18:40:21 [INFO] train episode 950: reward = 9.70, steps = 129
    18:40:35 [INFO] train episode 951: reward = 223.55, steps = 970
    18:40:41 [INFO] train episode 952: reward = 242.70, steps = 404
    18:40:56 [INFO] train episode 953: reward = 137.20, steps = 1000
    18:41:10 [INFO] train episode 954: reward = 131.76, steps = 1000
    18:41:24 [INFO] train episode 955: reward = 161.45, steps = 1000
    18:41:27 [INFO] train episode 956: reward = 284.07, steps = 204
    18:41:42 [INFO] train episode 957: reward = 145.33, steps = 1000
    18:41:58 [INFO] train episode 958: reward = 138.87, steps = 1000
    18:42:11 [INFO] train episode 959: reward = 240.89, steps = 874
    18:42:16 [INFO] train episode 960: reward = 247.68, steps = 373
    18:42:24 [INFO] train episode 961: reward = 275.73, steps = 607
    18:42:40 [INFO] train episode 962: reward = 147.51, steps = 1000
    18:42:56 [INFO] train episode 963: reward = 130.93, steps = 1000
    18:43:10 [INFO] train episode 964: reward = 143.72, steps = 1000
    18:43:13 [INFO] train episode 965: reward = 10.94, steps = 185
    18:43:16 [INFO] train episode 966: reward = 279.68, steps = 229
    18:43:30 [INFO] train episode 967: reward = 116.36, steps = 1000
    18:43:44 [INFO] train episode 968: reward = 116.62, steps = 1000
    18:43:59 [INFO] train episode 969: reward = 110.02, steps = 1000
    18:44:02 [INFO] train episode 970: reward = -42.73, steps = 198
    18:44:13 [INFO] train episode 971: reward = 270.28, steps = 778
    18:44:28 [INFO] train episode 972: reward = 143.24, steps = 1000
    18:44:41 [INFO] train episode 973: reward = 238.53, steps = 834
    18:44:43 [INFO] train episode 974: reward = 25.61, steps = 120
    18:44:58 [INFO] train episode 975: reward = 114.87, steps = 1000
    18:45:00 [INFO] train episode 976: reward = 246.41, steps = 182
    18:45:04 [INFO] train episode 977: reward = 272.81, steps = 274
    18:45:07 [INFO] train episode 978: reward = 284.30, steps = 156
    18:45:09 [INFO] train episode 979: reward = 275.49, steps = 181
    18:45:11 [INFO] train episode 980: reward = -188.34, steps = 128
    18:45:26 [INFO] train episode 981: reward = 148.86, steps = 1000
    18:45:34 [INFO] train episode 982: reward = 213.56, steps = 496
    18:45:49 [INFO] train episode 983: reward = 109.34, steps = 1000
    18:45:52 [INFO] train episode 984: reward = 234.93, steps = 235
    18:46:07 [INFO] train episode 985: reward = 130.47, steps = 1000
    18:46:11 [INFO] train episode 986: reward = 248.10, steps = 298
    18:46:13 [INFO] train episode 987: reward = 3.70, steps = 143
    18:46:14 [INFO] train episode 988: reward = 20.89, steps = 137
    18:46:18 [INFO] train episode 989: reward = 222.76, steps = 261
    18:46:24 [INFO] train episode 990: reward = 288.13, steps = 429
    18:46:28 [INFO] train episode 991: reward = 244.69, steps = 295
    18:46:32 [INFO] train episode 992: reward = 262.47, steps = 337
    18:46:35 [INFO] train episode 993: reward = 281.84, steps = 226
    18:46:39 [INFO] train episode 994: reward = 261.27, steps = 296
    18:46:53 [INFO] train episode 995: reward = 172.76, steps = 1000
    18:47:08 [INFO] train episode 996: reward = 129.99, steps = 1000
    18:47:22 [INFO] train episode 997: reward = 146.06, steps = 1000
    18:47:36 [INFO] train episode 998: reward = 145.55, steps = 1000
    18:47:50 [INFO] train episode 999: reward = 166.10, steps = 1000
    18:47:53 [INFO] train episode 1000: reward = 271.57, steps = 228
    18:47:55 [INFO] train episode 1001: reward = 56.52, steps = 134
    18:48:01 [INFO] train episode 1002: reward = 271.93, steps = 501
    18:48:05 [INFO] train episode 1003: reward = 285.02, steps = 295
    18:48:20 [INFO] train episode 1004: reward = 133.74, steps = 1000
    18:48:34 [INFO] train episode 1005: reward = 113.48, steps = 1000
    18:48:38 [INFO] train episode 1006: reward = 222.72, steps = 323
    18:48:45 [INFO] train episode 1007: reward = 279.40, steps = 535
    18:49:00 [INFO] train episode 1008: reward = 128.84, steps = 1000
    18:49:03 [INFO] train episode 1009: reward = 246.11, steps = 217
    18:49:17 [INFO] train episode 1010: reward = 132.16, steps = 1000
    18:49:20 [INFO] train episode 1011: reward = -199.85, steps = 209
    18:49:23 [INFO] train episode 1012: reward = 257.34, steps = 227
    18:49:28 [INFO] train episode 1013: reward = 278.25, steps = 371
    18:49:32 [INFO] train episode 1014: reward = 243.59, steps = 336
    18:49:34 [INFO] train episode 1015: reward = -37.41, steps = 83
    18:49:41 [INFO] train episode 1016: reward = 244.25, steps = 571
    18:49:56 [INFO] train episode 1017: reward = 171.17, steps = 1000
    18:49:57 [INFO] train episode 1018: reward = 24.91, steps = 110
    18:49:59 [INFO] train episode 1019: reward = 38.19, steps = 154
    18:50:14 [INFO] train episode 1020: reward = 103.89, steps = 1000
    18:50:18 [INFO] train episode 1021: reward = 279.42, steps = 298
    18:50:32 [INFO] train episode 1022: reward = 171.37, steps = 1000
    18:50:47 [INFO] train episode 1023: reward = 141.96, steps = 1000
    18:51:01 [INFO] train episode 1024: reward = 131.08, steps = 1000
    18:51:03 [INFO] train episode 1025: reward = 70.47, steps = 113
    18:51:07 [INFO] train episode 1026: reward = 244.15, steps = 326
    18:51:22 [INFO] train episode 1027: reward = 86.42, steps = 1000
    18:51:25 [INFO] train episode 1028: reward = 225.56, steps = 233
    18:51:27 [INFO] train episode 1029: reward = -229.27, steps = 117
    18:51:33 [INFO] train episode 1030: reward = 248.30, steps = 471
    18:51:47 [INFO] train episode 1031: reward = 141.96, steps = 1000
    18:52:02 [INFO] train episode 1032: reward = 158.02, steps = 1000
    18:52:16 [INFO] train episode 1033: reward = 126.78, steps = 1000
    18:52:17 [INFO] train episode 1034: reward = -54.74, steps = 58
    18:52:31 [INFO] train episode 1035: reward = 130.50, steps = 1000
    18:52:45 [INFO] train episode 1036: reward = 134.02, steps = 1000
    18:52:50 [INFO] train episode 1037: reward = 273.50, steps = 326
    18:53:05 [INFO] train episode 1038: reward = 113.90, steps = 1000
    18:53:10 [INFO] train episode 1039: reward = 231.13, steps = 383
    18:53:12 [INFO] train episode 1040: reward = -33.05, steps = 92
    18:53:14 [INFO] train episode 1041: reward = 19.81, steps = 172
    18:53:18 [INFO] train episode 1042: reward = 254.22, steps = 332
    18:53:19 [INFO] train episode 1043: reward = -31.29, steps = 75
    18:53:30 [INFO] train episode 1044: reward = 246.31, steps = 757
    18:53:37 [INFO] train episode 1045: reward = 210.33, steps = 495
    18:53:51 [INFO] train episode 1046: reward = 117.26, steps = 1000
    18:54:05 [INFO] train episode 1047: reward = 129.29, steps = 1000
    18:54:20 [INFO] train episode 1048: reward = 142.76, steps = 1000
    18:54:21 [INFO] train episode 1049: reward = -51.24, steps = 99
    18:54:26 [INFO] train episode 1050: reward = 254.33, steps = 338
    18:54:33 [INFO] train episode 1051: reward = 280.18, steps = 516
    18:54:35 [INFO] train episode 1052: reward = 221.16, steps = 188
    18:54:40 [INFO] train episode 1053: reward = 257.07, steps = 340
    18:54:42 [INFO] train episode 1054: reward = -42.04, steps = 129
    18:54:44 [INFO] train episode 1055: reward = 18.48, steps = 172
    18:54:45 [INFO] train episode 1056: reward = -77.88, steps = 76
    18:54:52 [INFO] train episode 1057: reward = 203.52, steps = 481
    18:54:53 [INFO] train episode 1058: reward = -45.13, steps = 78
    18:55:08 [INFO] train episode 1059: reward = 41.83, steps = 1000
    18:55:13 [INFO] train episode 1060: reward = 265.92, steps = 388
    18:55:17 [INFO] train episode 1061: reward = 239.85, steps = 258
    18:55:19 [INFO] train episode 1062: reward = -23.84, steps = 164
    18:55:23 [INFO] train episode 1063: reward = 197.61, steps = 281
    18:55:38 [INFO] train episode 1064: reward = 138.22, steps = 1000
    18:55:40 [INFO] train episode 1065: reward = -54.58, steps = 128
    18:55:42 [INFO] train episode 1066: reward = 17.46, steps = 159
    18:55:46 [INFO] train episode 1067: reward = 177.45, steps = 295
    18:56:00 [INFO] train episode 1068: reward = 87.30, steps = 1000
    18:56:02 [INFO] train episode 1069: reward = -35.31, steps = 108
    18:56:05 [INFO] train episode 1070: reward = 245.48, steps = 263
    18:56:07 [INFO] train episode 1071: reward = -54.59, steps = 108
    18:56:11 [INFO] train episode 1072: reward = 263.13, steps = 312
    18:56:14 [INFO] train episode 1073: reward = 253.31, steps = 236
    18:56:21 [INFO] train episode 1074: reward = 260.72, steps = 466
    18:56:35 [INFO] train episode 1075: reward = 133.72, steps = 1000
    18:56:39 [INFO] train episode 1076: reward = 262.66, steps = 251
    18:56:40 [INFO] train episode 1077: reward = -89.85, steps = 138
    18:56:42 [INFO] train episode 1078: reward = -450.04, steps = 117
    18:56:46 [INFO] train episode 1079: reward = 275.63, steps = 274
    18:56:48 [INFO] train episode 1080: reward = -65.11, steps = 136
    18:57:02 [INFO] train episode 1081: reward = 140.64, steps = 1000
    18:57:17 [INFO] train episode 1082: reward = 136.91, steps = 1000
    18:57:21 [INFO] train episode 1083: reward = 271.82, steps = 256
    18:57:24 [INFO] train episode 1084: reward = 283.92, steps = 272
    18:57:26 [INFO] train episode 1085: reward = -60.55, steps = 104
    18:57:41 [INFO] train episode 1086: reward = 106.55, steps = 1000
    18:57:45 [INFO] train episode 1087: reward = 237.74, steps = 255
    18:57:48 [INFO] train episode 1088: reward = 263.52, steps = 211
    18:57:51 [INFO] train episode 1089: reward = 232.83, steps = 199
    18:57:55 [INFO] train episode 1090: reward = 244.00, steps = 297
    18:57:59 [INFO] train episode 1091: reward = 279.19, steps = 262
    18:58:03 [INFO] train episode 1092: reward = 203.34, steps = 344
    18:58:08 [INFO] train episode 1093: reward = 287.05, steps = 295
    18:58:15 [INFO] train episode 1094: reward = 235.25, steps = 491
    18:58:19 [INFO] train episode 1095: reward = 250.99, steps = 247
    18:58:23 [INFO] train episode 1096: reward = 289.25, steps = 248
    18:58:23 [INFO] ==== test ====
    18:58:23 [INFO] test episode 0: reward = 279.16, steps = 244
    18:58:23 [INFO] test episode 1: reward = 238.80, steps = 277
    18:58:24 [INFO] test episode 2: reward = 256.58, steps = 407
    18:58:24 [INFO] test episode 3: reward = 260.11, steps = 247
    18:58:25 [INFO] test episode 4: reward = 253.51, steps = 295
    18:58:25 [INFO] test episode 5: reward = 244.07, steps = 277
    18:58:27 [INFO] test episode 6: reward = 107.44, steps = 1000
    18:58:28 [INFO] test episode 7: reward = 256.76, steps = 358
    18:58:28 [INFO] test episode 8: reward = 253.51, steps = 216
    18:58:28 [INFO] test episode 9: reward = 273.04, steps = 241
    18:58:28 [INFO] test episode 10: reward = 217.87, steps = 237
    18:58:30 [INFO] test episode 11: reward = 104.29, steps = 1000
    18:58:31 [INFO] test episode 12: reward = 215.63, steps = 226
    18:58:31 [INFO] test episode 13: reward = 260.16, steps = 264
    18:58:31 [INFO] test episode 14: reward = 266.09, steps = 357
    18:58:32 [INFO] test episode 15: reward = 257.40, steps = 334
    18:58:32 [INFO] test episode 16: reward = 246.24, steps = 337
    18:58:33 [INFO] test episode 17: reward = 257.08, steps = 353
    18:58:33 [INFO] test episode 18: reward = 255.00, steps = 274
    18:58:35 [INFO] test episode 19: reward = 151.24, steps = 1000
    18:58:36 [INFO] test episode 20: reward = 257.41, steps = 552
    18:58:38 [INFO] test episode 21: reward = 107.49, steps = 1000
    18:58:38 [INFO] test episode 22: reward = 225.74, steps = 224
    18:58:40 [INFO] test episode 23: reward = 141.09, steps = 1000
    18:58:42 [INFO] test episode 24: reward = 142.38, steps = 1000
    18:58:42 [INFO] test episode 25: reward = 238.57, steps = 257
    18:58:42 [INFO] test episode 26: reward = 262.03, steps = 258
    18:58:43 [INFO] test episode 27: reward = 253.45, steps = 206
    18:58:43 [INFO] test episode 28: reward = 241.28, steps = 345
    18:58:43 [INFO] test episode 29: reward = 221.80, steps = 244
    18:58:45 [INFO] test episode 30: reward = 158.97, steps = 1000
    18:58:46 [INFO] test episode 31: reward = 211.46, steps = 377
    18:58:46 [INFO] test episode 32: reward = 263.97, steps = 348
    18:58:47 [INFO] test episode 33: reward = 243.19, steps = 256
    18:58:47 [INFO] test episode 34: reward = 258.01, steps = 364
    18:58:47 [INFO] test episode 35: reward = 279.04, steps = 253
    18:58:48 [INFO] test episode 36: reward = 268.28, steps = 285
    18:58:48 [INFO] test episode 37: reward = 220.07, steps = 228
    18:58:48 [INFO] test episode 38: reward = 253.82, steps = 428
    18:58:49 [INFO] test episode 39: reward = 259.66, steps = 309
    18:58:51 [INFO] test episode 40: reward = 194.73, steps = 912
    18:58:51 [INFO] test episode 41: reward = 266.65, steps = 241
    18:58:52 [INFO] test episode 42: reward = 197.65, steps = 513
    18:58:52 [INFO] test episode 43: reward = 266.24, steps = 391
    18:58:53 [INFO] test episode 44: reward = 253.45, steps = 243
    18:58:53 [INFO] test episode 45: reward = 263.09, steps = 365
    18:58:55 [INFO] test episode 46: reward = 101.22, steps = 1000
    18:58:56 [INFO] test episode 47: reward = 258.37, steps = 471
    18:58:56 [INFO] test episode 48: reward = 252.55, steps = 256
    18:58:58 [INFO] test episode 49: reward = 168.53, steps = 894
    18:58:58 [INFO] test episode 50: reward = 253.71, steps = 439
    18:58:59 [INFO] test episode 51: reward = 232.88, steps = 207
    18:59:01 [INFO] test episode 52: reward = 210.76, steps = 908
    18:59:01 [INFO] test episode 53: reward = 263.08, steps = 343
    18:59:02 [INFO] test episode 54: reward = 224.02, steps = 315
    18:59:02 [INFO] test episode 55: reward = 232.40, steps = 311
    18:59:02 [INFO] test episode 56: reward = 256.63, steps = 305
    18:59:03 [INFO] test episode 57: reward = 248.26, steps = 372
    18:59:03 [INFO] test episode 58: reward = 273.76, steps = 386
    18:59:05 [INFO] test episode 59: reward = 135.52, steps = 1000
    18:59:07 [INFO] test episode 60: reward = 148.69, steps = 1000
    18:59:07 [INFO] test episode 61: reward = 251.73, steps = 266
    18:59:08 [INFO] test episode 62: reward = 251.96, steps = 180
    18:59:08 [INFO] test episode 63: reward = 246.56, steps = 313
    18:59:08 [INFO] test episode 64: reward = 254.47, steps = 326
    18:59:09 [INFO] test episode 65: reward = -2.94, steps = 246
    18:59:09 [INFO] test episode 66: reward = 257.42, steps = 354
    18:59:09 [INFO] test episode 67: reward = 256.20, steps = 336
    18:59:10 [INFO] test episode 68: reward = 246.34, steps = 293
    18:59:12 [INFO] test episode 69: reward = 118.35, steps = 1000
    18:59:12 [INFO] test episode 70: reward = 221.84, steps = 334
    18:59:13 [INFO] test episode 71: reward = 242.60, steps = 349
    18:59:13 [INFO] test episode 72: reward = 265.19, steps = 244
    18:59:13 [INFO] test episode 73: reward = 272.73, steps = 262
    18:59:14 [INFO] test episode 74: reward = 240.03, steps = 443
    18:59:14 [INFO] test episode 75: reward = 263.69, steps = 227
    18:59:15 [INFO] test episode 76: reward = 195.27, steps = 387
    18:59:15 [INFO] test episode 77: reward = 238.01, steps = 432
    18:59:16 [INFO] test episode 78: reward = 245.46, steps = 343
    18:59:16 [INFO] test episode 79: reward = 237.64, steps = 302
    18:59:16 [INFO] test episode 80: reward = 265.55, steps = 278
    18:59:17 [INFO] test episode 81: reward = 271.83, steps = 404
    18:59:17 [INFO] test episode 82: reward = 255.63, steps = 233
    18:59:17 [INFO] test episode 83: reward = 255.71, steps = 434
    18:59:18 [INFO] test episode 84: reward = 247.68, steps = 444
    18:59:18 [INFO] test episode 85: reward = 202.10, steps = 326
    18:59:19 [INFO] test episode 86: reward = 268.52, steps = 255
    18:59:19 [INFO] test episode 87: reward = 247.50, steps = 287
    18:59:19 [INFO] test episode 88: reward = 276.68, steps = 358
    18:59:21 [INFO] test episode 89: reward = 139.25, steps = 1000
    18:59:23 [INFO] test episode 90: reward = 94.53, steps = 1000
    18:59:23 [INFO] test episode 91: reward = 286.10, steps = 323
    18:59:23 [INFO] test episode 92: reward = 263.29, steps = 320
    18:59:25 [INFO] test episode 93: reward = 108.22, steps = 1000
    18:59:27 [INFO] test episode 94: reward = 146.27, steps = 1000
    18:59:27 [INFO] test episode 95: reward = 235.82, steps = 322
    18:59:29 [INFO] test episode 96: reward = 143.85, steps = 1000
    18:59:30 [INFO] test episode 97: reward = 241.33, steps = 455
    18:59:30 [INFO] test episode 98: reward = 287.49, steps = 258
    18:59:32 [INFO] test episode 99: reward = 140.57, steps = 1000
    18:59:32 [INFO] average episode reward = 225.44 ± 54.79