# MoutainCar-v0¶

In [1]:

    %matplotlib inline

    import sys
    import logging
    import itertools

    import numpy as np
    np.random.seed(0)
    import gym
    import matplotlib.pyplot as plt

    logging.basicConfig(level=logging.INFO,
            format='%(asctime)s [%(levelname)s] %(message)s',
            stream=sys.stdout, datefmt='%H:%M:%S')

### Use Environment¶

In [2]:

    env = gym.make('MountainCar-v0')
    logging.info('observation space = %s', env.observation_space)
    logging.info('action space = %s', env.action_space)
    logging.info('range of positions = %s ~ %s', env.min_position, env.max_position)
    logging.info('range of speeds = %s ~ %s', -env.max_speed, env.max_speed)
    logging.info('goal position = %s', env.goal_position)
    logging.info('reward threshold = %s', env.spec.reward_threshold)
    logging.info('max episode steps = %s', env.spec.max_episode_steps)

    00:01:54 [INFO] observation space = Box([-1.2  -0.07], [0.6  0.07], (2,), float32)
    00:01:54 [INFO] action space = Discrete(3)
    00:01:54 [INFO] range of positions = -1.2 ~ 0.6
    00:01:54 [INFO] range of speeds = -0.07 ~ 0.07
    00:01:54 [INFO] goal position = 0.5
    00:01:54 [INFO] reward threshold = -110.0
    00:01:54 [INFO] max episode steps = 200

### Stubborn Agent¶

Simply pushing right does not work

In [3]:

    positions, velocities = [], []
    observation, _ = env.reset()
    while True:
        positions.append(observation[0])
        velocities.append(observation[1])
        next_observation, reward, terminated, truncated, _ = env.step(2)
        if terminated or truncated:
            break
        observation = next_observation

    if next_observation[0] > 0.5:
        logging.info('succeed')
    else:
        logging.info('fail')

    # plot positions and velocities
    fig, ax = plt.subplots()
    ax.plot(positions, label='position')
    ax.plot(velocities, label='velocity')
    ax.legend()

    00:01:54 [INFO] fail

Out[3]:

    <matplotlib.legend.Legend at 0x271f7a91ea0>

### SARSA with Tile Coding¶

Tile Coder

In [4]:

    class TileCoder:
        def __init__(self, layer_count, feature_count):
            self.layer_count = layer_count
            self.feature_count = feature_count
            self.codebook = {}

        def get_feature(self, codeword):
            if codeword in self.codebook:
                return self.codebook[codeword]
            count = len(self.codebook)
            if count >= self.feature_count:  # resolve conflicts
                return hash(codeword) % self.feature_count
            self.codebook[codeword] = count
            return count

        def __call__(self, floats=(), ints=()):
            dim = len(floats)
            scaled_floats = tuple(f * (self.layer_count ** 2) for f in floats)
            features = []
            for layer in range(self.layer_count):
                codeword = (layer,) + tuple(
                        int((f + (1 + dim * i) * layer) / self.layer_count)
                        for i, f in enumerate(scaled_floats)) + ints
                feature = self.get_feature(codeword)
                features.append(feature)
            return features

SARSA with Linear Approximation

In [5]:

    class SARSAAgent:
        def __init__(self, env):
            self.action_n = env.action_space.n
            self.obs_low = env.observation_space.low
            self.obs_scale = env.observation_space.high - \
                    env.observation_space.low
            self.encoder = TileCoder(8, 1896)
            self.w = np.zeros(self.encoder.feature_count)
            self.gamma = 1.
            self.learning_rate = 0.03

        def encode(self, observation, action):
            states = tuple((observation - self.obs_low) / self.obs_scale)
            actions = (action,)
            return self.encoder(states, actions)

        def get_q(self, observation, action):  # action value
            features = self.encode(observation, action)
            return self.w[features].sum()

        def reset(self, mode=None):
            self.mode = mode
            if self.mode == 'train':
                self.trajectory = []

        def step(self, observation, reward, terminated):
            if self.mode == 'train' and np.random.rand() < 0.001:
                action = np.random.randint(self.action_n)
            else:
                qs = [self.get_q(observation, action) for action in
                        range(self.action_n)]
                action = np.argmax(qs)
            if self.mode == 'train':
                self.trajectory += [observation, reward, terminated, action]
                if len(self.trajectory) >= 8:
                    self.learn()
            return action

        def close(self):
            pass

        def learn(self):
            observation, _, _, action, next_observation, reward, terminated, \
                    next_action = self.trajectory[-8:]
            target = reward + (1. - terminated) * self.gamma * \
                    self.get_q(next_observation, next_action)
            td_error = target - self.get_q(observation, action)
            features = self.encode(observation, action)
            self.w[features] += (self.learning_rate * td_error)

    agent = SARSAAgent(env)

Train & test

In [6]:

    def play_episode(env, agent, seed=None, mode=None, render=False):
        observation, _ = env.reset(seed=seed)
        reward, terminated, truncated = 0., False, False
        agent.reset(mode=mode)
        episode_reward, elapsed_steps = 0., 0
        while True:
            action = agent.step(observation, reward, terminated)
            if render:
                env.render()
            if terminated or truncated:
                break
            observation, reward, terminated, truncated, _ = env.step(action)
            episode_reward += reward
            elapsed_steps += 1
        agent.close()
        return episode_reward, elapsed_steps

    logging.info('==== train ====')
    episode_rewards = []
    for episode in itertools.count():
        episode_reward, elapsed_steps = play_episode(env, agent, seed=episode,
                mode='train')
        episode_rewards.append(episode_reward)
        logging.info('train episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
        if np.mean(episode_rewards[-100:]) > -110:
            break
    plt.plot(episode_rewards)

    logging.info('==== test ====')
    episode_rewards = []
    for episode in range(100):
        episode_reward, elapsed_steps = play_episode(env, agent)
        episode_rewards.append(episode_reward)
        logging.info('test episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
    logging.info('average episode reward = %.2f ± %.2f',
            np.mean(episode_rewards), np.std(episode_rewards))

    00:01:55 [INFO] ==== train ====
    00:01:55 [INFO] train episode 0: reward = -200.00, steps = 200
    00:01:55 [INFO] train episode 1: reward = -200.00, steps = 200
    00:01:55 [INFO] train episode 2: reward = -200.00, steps = 200
    00:01:55 [INFO] train episode 3: reward = -200.00, steps = 200
    00:01:55 [INFO] train episode 4: reward = -200.00, steps = 200
    00:01:55 [INFO] train episode 5: reward = -200.00, steps = 200
    00:01:55 [INFO] train episode 6: reward = -200.00, steps = 200
    00:01:56 [INFO] train episode 7: reward = -200.00, steps = 200
    00:01:56 [INFO] train episode 8: reward = -200.00, steps = 200
    00:01:56 [INFO] train episode 9: reward = -200.00, steps = 200
    00:01:56 [INFO] train episode 10: reward = -200.00, steps = 200
    00:01:56 [INFO] train episode 11: reward = -200.00, steps = 200
    00:01:56 [INFO] train episode 12: reward = -200.00, steps = 200
    00:01:56 [INFO] train episode 13: reward = -200.00, steps = 200
    00:01:56 [INFO] train episode 14: reward = -200.00, steps = 200
    00:01:57 [INFO] train episode 15: reward = -200.00, steps = 200
    00:01:57 [INFO] train episode 16: reward = -200.00, steps = 200
    00:01:57 [INFO] train episode 17: reward = -200.00, steps = 200
    00:01:57 [INFO] train episode 18: reward = -200.00, steps = 200
    00:01:57 [INFO] train episode 19: reward = -200.00, steps = 200
    00:01:57 [INFO] train episode 20: reward = -200.00, steps = 200
    00:01:57 [INFO] train episode 21: reward = -200.00, steps = 200
    00:01:57 [INFO] train episode 22: reward = -200.00, steps = 200
    00:01:58 [INFO] train episode 23: reward = -200.00, steps = 200
    00:01:58 [INFO] train episode 24: reward = -200.00, steps = 200
    00:01:58 [INFO] train episode 25: reward = -200.00, steps = 200
    00:01:58 [INFO] train episode 26: reward = -200.00, steps = 200
    00:01:58 [INFO] train episode 27: reward = -200.00, steps = 200
    00:01:58 [INFO] train episode 28: reward = -200.00, steps = 200
    00:01:58 [INFO] train episode 29: reward = -200.00, steps = 200
    00:01:58 [INFO] train episode 30: reward = -200.00, steps = 200
    00:01:59 [INFO] train episode 31: reward = -200.00, steps = 200
    00:01:59 [INFO] train episode 32: reward = -200.00, steps = 200
    00:01:59 [INFO] train episode 33: reward = -200.00, steps = 200
    00:01:59 [INFO] train episode 34: reward = -200.00, steps = 200
    00:01:59 [INFO] train episode 35: reward = -200.00, steps = 200
    00:01:59 [INFO] train episode 36: reward = -200.00, steps = 200
    00:01:59 [INFO] train episode 37: reward = -200.00, steps = 200
    00:01:59 [INFO] train episode 38: reward = -200.00, steps = 200
    00:01:59 [INFO] train episode 39: reward = -200.00, steps = 200
    00:02:00 [INFO] train episode 40: reward = -200.00, steps = 200
    00:02:00 [INFO] train episode 41: reward = -200.00, steps = 200
    00:02:00 [INFO] train episode 42: reward = -169.00, steps = 169
    00:02:00 [INFO] train episode 43: reward = -200.00, steps = 200
    00:02:00 [INFO] train episode 44: reward = -200.00, steps = 200
    00:02:01 [INFO] train episode 45: reward = -200.00, steps = 200
    00:02:01 [INFO] train episode 46: reward = -200.00, steps = 200
    00:02:01 [INFO] train episode 47: reward = -200.00, steps = 200
    00:02:01 [INFO] train episode 48: reward = -200.00, steps = 200
    00:02:01 [INFO] train episode 49: reward = -200.00, steps = 200
    00:02:01 [INFO] train episode 50: reward = -200.00, steps = 200
    00:02:01 [INFO] train episode 51: reward = -200.00, steps = 200
    00:02:01 [INFO] train episode 52: reward = -200.00, steps = 200
    00:02:02 [INFO] train episode 53: reward = -200.00, steps = 200
    00:02:02 [INFO] train episode 54: reward = -200.00, steps = 200
    00:02:02 [INFO] train episode 55: reward = -200.00, steps = 200
    00:02:02 [INFO] train episode 56: reward = -167.00, steps = 167
    00:02:02 [INFO] train episode 57: reward = -200.00, steps = 200
    00:02:02 [INFO] train episode 58: reward = -200.00, steps = 200
    00:02:02 [INFO] train episode 59: reward = -156.00, steps = 156
    00:02:02 [INFO] train episode 60: reward = -200.00, steps = 200
    00:02:02 [INFO] train episode 61: reward = -200.00, steps = 200
    00:02:03 [INFO] train episode 62: reward = -174.00, steps = 174
    00:02:03 [INFO] train episode 63: reward = -200.00, steps = 200
    00:02:03 [INFO] train episode 64: reward = -172.00, steps = 172
    00:02:03 [INFO] train episode 65: reward = -132.00, steps = 132
    00:02:03 [INFO] train episode 66: reward = -173.00, steps = 173
    00:02:03 [INFO] train episode 67: reward = -200.00, steps = 200
    00:02:03 [INFO] train episode 68: reward = -200.00, steps = 200
    00:02:03 [INFO] train episode 69: reward = -200.00, steps = 200
    00:02:03 [INFO] train episode 70: reward = -200.00, steps = 200
    00:02:04 [INFO] train episode 71: reward = -191.00, steps = 191
    00:02:04 [INFO] train episode 72: reward = -200.00, steps = 200
    00:02:04 [INFO] train episode 73: reward = -179.00, steps = 179
    00:02:04 [INFO] train episode 74: reward = -200.00, steps = 200
    00:02:04 [INFO] train episode 75: reward = -187.00, steps = 187
    00:02:04 [INFO] train episode 76: reward = -155.00, steps = 155
    00:02:04 [INFO] train episode 77: reward = -164.00, steps = 164
    00:02:04 [INFO] train episode 78: reward = -200.00, steps = 200
    00:02:05 [INFO] train episode 79: reward = -200.00, steps = 200
    00:02:05 [INFO] train episode 80: reward = -200.00, steps = 200
    00:02:05 [INFO] train episode 81: reward = -149.00, steps = 149
    00:02:05 [INFO] train episode 82: reward = -164.00, steps = 164
    00:02:05 [INFO] train episode 83: reward = -200.00, steps = 200
    00:02:05 [INFO] train episode 84: reward = -160.00, steps = 160
    00:02:05 [INFO] train episode 85: reward = -195.00, steps = 195
    00:02:05 [INFO] train episode 86: reward = -200.00, steps = 200
    00:02:05 [INFO] train episode 87: reward = -188.00, steps = 188
    00:02:06 [INFO] train episode 88: reward = -194.00, steps = 194
    00:02:06 [INFO] train episode 89: reward = -200.00, steps = 200
    00:02:06 [INFO] train episode 90: reward = -183.00, steps = 183
    00:02:06 [INFO] train episode 91: reward = -190.00, steps = 190
    00:02:06 [INFO] train episode 92: reward = -173.00, steps = 173
    00:02:06 [INFO] train episode 93: reward = -172.00, steps = 172
    00:02:06 [INFO] train episode 94: reward = -141.00, steps = 141
    00:02:06 [INFO] train episode 95: reward = -200.00, steps = 200
    00:02:06 [INFO] train episode 96: reward = -154.00, steps = 154
    00:02:07 [INFO] train episode 97: reward = -155.00, steps = 155
    00:02:07 [INFO] train episode 98: reward = -163.00, steps = 163
    00:02:07 [INFO] train episode 99: reward = -179.00, steps = 179
    00:02:07 [INFO] train episode 100: reward = -155.00, steps = 155
    00:02:07 [INFO] train episode 101: reward = -155.00, steps = 155
    00:02:07 [INFO] train episode 102: reward = -200.00, steps = 200
    00:02:07 [INFO] train episode 103: reward = -195.00, steps = 195
    00:02:07 [INFO] train episode 104: reward = -167.00, steps = 167
    00:02:07 [INFO] train episode 105: reward = -153.00, steps = 153
    00:02:07 [INFO] train episode 106: reward = -171.00, steps = 171
    00:02:08 [INFO] train episode 107: reward = -151.00, steps = 151
    00:02:08 [INFO] train episode 108: reward = -161.00, steps = 161
    00:02:08 [INFO] train episode 109: reward = -155.00, steps = 155
    00:02:08 [INFO] train episode 110: reward = -161.00, steps = 161
    00:02:08 [INFO] train episode 111: reward = -198.00, steps = 198
    00:02:08 [INFO] train episode 112: reward = -197.00, steps = 197
    00:02:08 [INFO] train episode 113: reward = -200.00, steps = 200
    00:02:08 [INFO] train episode 114: reward = -154.00, steps = 154
    00:02:08 [INFO] train episode 115: reward = -150.00, steps = 150
    00:02:09 [INFO] train episode 116: reward = -190.00, steps = 190
    00:02:09 [INFO] train episode 117: reward = -155.00, steps = 155
    00:02:09 [INFO] train episode 118: reward = -154.00, steps = 154
    00:02:09 [INFO] train episode 119: reward = -152.00, steps = 152
    00:02:09 [INFO] train episode 120: reward = -153.00, steps = 153
    00:02:09 [INFO] train episode 121: reward = -162.00, steps = 162
    00:02:09 [INFO] train episode 122: reward = -161.00, steps = 161
    00:02:09 [INFO] train episode 123: reward = -154.00, steps = 154
    00:02:09 [INFO] train episode 124: reward = -200.00, steps = 200
    00:02:09 [INFO] train episode 125: reward = -166.00, steps = 166
    00:02:10 [INFO] train episode 126: reward = -115.00, steps = 115
    00:02:10 [INFO] train episode 127: reward = -197.00, steps = 197
    00:02:10 [INFO] train episode 128: reward = -156.00, steps = 156
    00:02:10 [INFO] train episode 129: reward = -195.00, steps = 195
    00:02:10 [INFO] train episode 130: reward = -168.00, steps = 168
    00:02:10 [INFO] train episode 131: reward = -151.00, steps = 151
    00:02:10 [INFO] train episode 132: reward = -128.00, steps = 128
    00:02:10 [INFO] train episode 133: reward = -162.00, steps = 162
    00:02:10 [INFO] train episode 134: reward = -155.00, steps = 155
    00:02:10 [INFO] train episode 135: reward = -162.00, steps = 162
    00:02:11 [INFO] train episode 136: reward = -200.00, steps = 200
    00:02:11 [INFO] train episode 137: reward = -168.00, steps = 168
    00:02:11 [INFO] train episode 138: reward = -151.00, steps = 151
    00:02:11 [INFO] train episode 139: reward = -177.00, steps = 177
    00:02:11 [INFO] train episode 140: reward = -151.00, steps = 151
    00:02:11 [INFO] train episode 141: reward = -158.00, steps = 158
    00:02:11 [INFO] train episode 142: reward = -121.00, steps = 121
    00:02:11 [INFO] train episode 143: reward = -200.00, steps = 200
    00:02:11 [INFO] train episode 144: reward = -156.00, steps = 156
    00:02:11 [INFO] train episode 145: reward = -150.00, steps = 150
    00:02:11 [INFO] train episode 146: reward = -152.00, steps = 152
    00:02:12 [INFO] train episode 147: reward = -152.00, steps = 152
    00:02:12 [INFO] train episode 148: reward = -141.00, steps = 141
    00:02:12 [INFO] train episode 149: reward = -200.00, steps = 200
    00:02:12 [INFO] train episode 150: reward = -119.00, steps = 119
    00:02:12 [INFO] train episode 151: reward = -151.00, steps = 151
    00:02:12 [INFO] train episode 152: reward = -200.00, steps = 200
    00:02:12 [INFO] train episode 153: reward = -154.00, steps = 154
    00:02:12 [INFO] train episode 154: reward = -147.00, steps = 147
    00:02:12 [INFO] train episode 155: reward = -114.00, steps = 114
    00:02:12 [INFO] train episode 156: reward = -121.00, steps = 121
    00:02:12 [INFO] train episode 157: reward = -116.00, steps = 116
    00:02:13 [INFO] train episode 158: reward = -86.00, steps = 86
    00:02:13 [INFO] train episode 159: reward = -159.00, steps = 159
    00:02:13 [INFO] train episode 160: reward = -148.00, steps = 148
    00:02:13 [INFO] train episode 161: reward = -158.00, steps = 158
    00:02:13 [INFO] train episode 162: reward = -123.00, steps = 123
    00:02:13 [INFO] train episode 163: reward = -156.00, steps = 156
    00:02:13 [INFO] train episode 164: reward = -120.00, steps = 120
    00:02:13 [INFO] train episode 165: reward = -163.00, steps = 163
    00:02:13 [INFO] train episode 166: reward = -147.00, steps = 147
    00:02:13 [INFO] train episode 167: reward = -141.00, steps = 141
    00:02:13 [INFO] train episode 168: reward = -116.00, steps = 116
    00:02:13 [INFO] train episode 169: reward = -115.00, steps = 115
    00:02:14 [INFO] train episode 170: reward = -149.00, steps = 149
    00:02:14 [INFO] train episode 171: reward = -113.00, steps = 113
    00:02:14 [INFO] train episode 172: reward = -119.00, steps = 119
    00:02:14 [INFO] train episode 173: reward = -116.00, steps = 116
    00:02:14 [INFO] train episode 174: reward = -158.00, steps = 158
    00:02:14 [INFO] train episode 175: reward = -144.00, steps = 144
    00:02:14 [INFO] train episode 176: reward = -160.00, steps = 160
    00:02:14 [INFO] train episode 177: reward = -144.00, steps = 144
    00:02:14 [INFO] train episode 178: reward = -148.00, steps = 148
    00:02:14 [INFO] train episode 179: reward = -147.00, steps = 147
    00:02:14 [INFO] train episode 180: reward = -144.00, steps = 144
    00:02:14 [INFO] train episode 181: reward = -162.00, steps = 162
    00:02:15 [INFO] train episode 182: reward = -120.00, steps = 120
    00:02:15 [INFO] train episode 183: reward = -159.00, steps = 159
    00:02:15 [INFO] train episode 184: reward = -117.00, steps = 117
    00:02:15 [INFO] train episode 185: reward = -116.00, steps = 116
    00:02:15 [INFO] train episode 186: reward = -151.00, steps = 151
    00:02:15 [INFO] train episode 187: reward = -200.00, steps = 200
    00:02:15 [INFO] train episode 188: reward = -122.00, steps = 122
    00:02:15 [INFO] train episode 189: reward = -115.00, steps = 115
    00:02:15 [INFO] train episode 190: reward = -145.00, steps = 145
    00:02:15 [INFO] train episode 191: reward = -147.00, steps = 147
    00:02:15 [INFO] train episode 192: reward = -151.00, steps = 151
    00:02:16 [INFO] train episode 193: reward = -157.00, steps = 157
    00:02:16 [INFO] train episode 194: reward = -163.00, steps = 163
    00:02:16 [INFO] train episode 195: reward = -118.00, steps = 118
    00:02:16 [INFO] train episode 196: reward = -182.00, steps = 182
    00:02:16 [INFO] train episode 197: reward = -166.00, steps = 166
    00:02:16 [INFO] train episode 198: reward = -149.00, steps = 149
    00:02:16 [INFO] train episode 199: reward = -151.00, steps = 151
    00:02:16 [INFO] train episode 200: reward = -151.00, steps = 151
    00:02:16 [INFO] train episode 201: reward = -163.00, steps = 163
    00:02:16 [INFO] train episode 202: reward = -117.00, steps = 117
    00:02:16 [INFO] train episode 203: reward = -158.00, steps = 158
    00:02:17 [INFO] train episode 204: reward = -155.00, steps = 155
    00:02:17 [INFO] train episode 205: reward = -162.00, steps = 162
    00:02:17 [INFO] train episode 206: reward = -168.00, steps = 168
    00:02:17 [INFO] train episode 207: reward = -117.00, steps = 117
    00:02:17 [INFO] train episode 208: reward = -92.00, steps = 92
    00:02:17 [INFO] train episode 209: reward = -118.00, steps = 118
    00:02:17 [INFO] train episode 210: reward = -163.00, steps = 163
    00:02:17 [INFO] train episode 211: reward = -111.00, steps = 111
    00:02:17 [INFO] train episode 212: reward = -163.00, steps = 163
    00:02:17 [INFO] train episode 213: reward = -154.00, steps = 154
    00:02:17 [INFO] train episode 214: reward = -126.00, steps = 126
    00:02:17 [INFO] train episode 215: reward = -154.00, steps = 154
    00:02:18 [INFO] train episode 216: reward = -153.00, steps = 153
    00:02:18 [INFO] train episode 217: reward = -158.00, steps = 158
    00:02:18 [INFO] train episode 218: reward = -151.00, steps = 151
    00:02:18 [INFO] train episode 219: reward = -155.00, steps = 155
    00:02:18 [INFO] train episode 220: reward = -164.00, steps = 164
    00:02:18 [INFO] train episode 221: reward = -143.00, steps = 143
    00:02:18 [INFO] train episode 222: reward = -152.00, steps = 152
    00:02:18 [INFO] train episode 223: reward = -153.00, steps = 153
    00:02:18 [INFO] train episode 224: reward = -145.00, steps = 145
    00:02:18 [INFO] train episode 225: reward = -153.00, steps = 153
    00:02:18 [INFO] train episode 226: reward = -151.00, steps = 151
    00:02:19 [INFO] train episode 227: reward = -173.00, steps = 173
    00:02:19 [INFO] train episode 228: reward = -148.00, steps = 148
    00:02:19 [INFO] train episode 229: reward = -152.00, steps = 152
    00:02:19 [INFO] train episode 230: reward = -119.00, steps = 119
    00:02:19 [INFO] train episode 231: reward = -156.00, steps = 156
    00:02:19 [INFO] train episode 232: reward = -145.00, steps = 145
    00:02:19 [INFO] train episode 233: reward = -124.00, steps = 124
    00:02:19 [INFO] train episode 234: reward = -148.00, steps = 148
    00:02:19 [INFO] train episode 235: reward = -192.00, steps = 192
    00:02:19 [INFO] train episode 236: reward = -168.00, steps = 168
    00:02:19 [INFO] train episode 237: reward = -109.00, steps = 109
    00:02:20 [INFO] train episode 238: reward = -149.00, steps = 149
    00:02:20 [INFO] train episode 239: reward = -110.00, steps = 110
    00:02:20 [INFO] train episode 240: reward = -157.00, steps = 157
    00:02:20 [INFO] train episode 241: reward = -132.00, steps = 132
    00:02:20 [INFO] train episode 242: reward = -117.00, steps = 117
    00:02:20 [INFO] train episode 243: reward = -108.00, steps = 108
    00:02:20 [INFO] train episode 244: reward = -136.00, steps = 136
    00:02:20 [INFO] train episode 245: reward = -111.00, steps = 111
    00:02:20 [INFO] train episode 246: reward = -111.00, steps = 111
    00:02:20 [INFO] train episode 247: reward = -150.00, steps = 150
    00:02:20 [INFO] train episode 248: reward = -148.00, steps = 148
    00:02:20 [INFO] train episode 249: reward = -153.00, steps = 153
    00:02:20 [INFO] train episode 250: reward = -119.00, steps = 119
    00:02:21 [INFO] train episode 251: reward = -154.00, steps = 154
    00:02:21 [INFO] train episode 252: reward = -148.00, steps = 148
    00:02:21 [INFO] train episode 253: reward = -111.00, steps = 111
    00:02:21 [INFO] train episode 254: reward = -150.00, steps = 150
    00:02:21 [INFO] train episode 255: reward = -148.00, steps = 148
    00:02:21 [INFO] train episode 256: reward = -145.00, steps = 145
    00:02:21 [INFO] train episode 257: reward = -147.00, steps = 147
    00:02:21 [INFO] train episode 258: reward = -149.00, steps = 149
    00:02:21 [INFO] train episode 259: reward = -147.00, steps = 147
    00:02:21 [INFO] train episode 260: reward = -154.00, steps = 154
    00:02:21 [INFO] train episode 261: reward = -148.00, steps = 148
    00:02:22 [INFO] train episode 262: reward = -148.00, steps = 148
    00:02:22 [INFO] train episode 263: reward = -115.00, steps = 115
    00:02:22 [INFO] train episode 264: reward = -151.00, steps = 151
    00:02:22 [INFO] train episode 265: reward = -118.00, steps = 118
    00:02:22 [INFO] train episode 266: reward = -109.00, steps = 109
    00:02:22 [INFO] train episode 267: reward = -146.00, steps = 146
    00:02:22 [INFO] train episode 268: reward = -151.00, steps = 151
    00:02:22 [INFO] train episode 269: reward = -145.00, steps = 145
    00:02:22 [INFO] train episode 270: reward = -148.00, steps = 148
    00:02:22 [INFO] train episode 271: reward = -149.00, steps = 149
    00:02:22 [INFO] train episode 272: reward = -152.00, steps = 152
    00:02:22 [INFO] train episode 273: reward = -119.00, steps = 119
    00:02:23 [INFO] train episode 274: reward = -163.00, steps = 163
    00:02:23 [INFO] train episode 275: reward = -151.00, steps = 151
    00:02:23 [INFO] train episode 276: reward = -149.00, steps = 149
    00:02:23 [INFO] train episode 277: reward = -145.00, steps = 145
    00:02:23 [INFO] train episode 278: reward = -149.00, steps = 149
    00:02:23 [INFO] train episode 279: reward = -113.00, steps = 113
    00:02:23 [INFO] train episode 280: reward = -153.00, steps = 153
    00:02:23 [INFO] train episode 281: reward = -145.00, steps = 145
    00:02:23 [INFO] train episode 282: reward = -144.00, steps = 144
    00:02:23 [INFO] train episode 283: reward = -148.00, steps = 148
    00:02:23 [INFO] train episode 284: reward = -148.00, steps = 148
    00:02:24 [INFO] train episode 285: reward = -185.00, steps = 185
    00:02:24 [INFO] train episode 286: reward = -141.00, steps = 141
    00:02:24 [INFO] train episode 287: reward = -149.00, steps = 149
    00:02:24 [INFO] train episode 288: reward = -156.00, steps = 156
    00:02:24 [INFO] train episode 289: reward = -159.00, steps = 159
    00:02:24 [INFO] train episode 290: reward = -150.00, steps = 150
    00:02:24 [INFO] train episode 291: reward = -139.00, steps = 139
    00:02:24 [INFO] train episode 292: reward = -160.00, steps = 160
    00:02:24 [INFO] train episode 293: reward = -151.00, steps = 151
    00:02:24 [INFO] train episode 294: reward = -158.00, steps = 158
    00:02:24 [INFO] train episode 295: reward = -117.00, steps = 117
    00:02:25 [INFO] train episode 296: reward = -145.00, steps = 145
    00:02:25 [INFO] train episode 297: reward = -119.00, steps = 119
    00:02:25 [INFO] train episode 298: reward = -144.00, steps = 144
    00:02:25 [INFO] train episode 299: reward = -143.00, steps = 143
    00:02:25 [INFO] train episode 300: reward = -142.00, steps = 142
    00:02:25 [INFO] train episode 301: reward = -148.00, steps = 148
    00:02:25 [INFO] train episode 302: reward = -150.00, steps = 150
    00:02:25 [INFO] train episode 303: reward = -137.00, steps = 137
    00:02:25 [INFO] train episode 304: reward = -163.00, steps = 163
    00:02:25 [INFO] train episode 305: reward = -144.00, steps = 144
    00:02:25 [INFO] train episode 306: reward = -124.00, steps = 124
    00:02:25 [INFO] train episode 307: reward = -148.00, steps = 148
    00:02:26 [INFO] train episode 308: reward = -143.00, steps = 143
    00:02:26 [INFO] train episode 309: reward = -116.00, steps = 116
    00:02:26 [INFO] train episode 310: reward = -142.00, steps = 142
    00:02:26 [INFO] train episode 311: reward = -140.00, steps = 140
    00:02:26 [INFO] train episode 312: reward = -147.00, steps = 147
    00:02:26 [INFO] train episode 313: reward = -147.00, steps = 147
    00:02:26 [INFO] train episode 314: reward = -155.00, steps = 155
    00:02:26 [INFO] train episode 315: reward = -154.00, steps = 154
    00:02:26 [INFO] train episode 316: reward = -86.00, steps = 86
    00:02:26 [INFO] train episode 317: reward = -90.00, steps = 90
    00:02:26 [INFO] train episode 318: reward = -85.00, steps = 85
    00:02:26 [INFO] train episode 319: reward = -151.00, steps = 151
    00:02:27 [INFO] train episode 320: reward = -158.00, steps = 158
    00:02:27 [INFO] train episode 321: reward = -156.00, steps = 156
    00:02:27 [INFO] train episode 322: reward = -148.00, steps = 148
    00:02:27 [INFO] train episode 323: reward = -118.00, steps = 118
    00:02:27 [INFO] train episode 324: reward = -116.00, steps = 116
    00:02:27 [INFO] train episode 325: reward = -108.00, steps = 108
    00:02:27 [INFO] train episode 326: reward = -112.00, steps = 112
    00:02:27 [INFO] train episode 327: reward = -162.00, steps = 162
    00:02:27 [INFO] train episode 328: reward = -165.00, steps = 165
    00:02:27 [INFO] train episode 329: reward = -135.00, steps = 135
    00:02:27 [INFO] train episode 330: reward = -84.00, steps = 84
    00:02:27 [INFO] train episode 331: reward = -142.00, steps = 142
    00:02:27 [INFO] train episode 332: reward = -108.00, steps = 108
    00:02:28 [INFO] train episode 333: reward = -85.00, steps = 85
    00:02:28 [INFO] train episode 334: reward = -154.00, steps = 154
    00:02:28 [INFO] train episode 335: reward = -146.00, steps = 146
    00:02:28 [INFO] train episode 336: reward = -87.00, steps = 87
    00:02:28 [INFO] train episode 337: reward = -114.00, steps = 114
    00:02:28 [INFO] train episode 338: reward = -92.00, steps = 92
    00:02:28 [INFO] train episode 339: reward = -138.00, steps = 138
    00:02:28 [INFO] train episode 340: reward = -88.00, steps = 88
    00:02:28 [INFO] train episode 341: reward = -164.00, steps = 164
    00:02:28 [INFO] train episode 342: reward = -156.00, steps = 156
    00:02:28 [INFO] train episode 343: reward = -90.00, steps = 90
    00:02:28 [INFO] train episode 344: reward = -98.00, steps = 98
    00:02:28 [INFO] train episode 345: reward = -163.00, steps = 163
    00:02:29 [INFO] train episode 346: reward = -144.00, steps = 144
    00:02:29 [INFO] train episode 347: reward = -109.00, steps = 109
    00:02:29 [INFO] train episode 348: reward = -112.00, steps = 112
    00:02:29 [INFO] train episode 349: reward = -141.00, steps = 141
    00:02:29 [INFO] train episode 350: reward = -96.00, steps = 96
    00:02:29 [INFO] train episode 351: reward = -161.00, steps = 161
    00:02:29 [INFO] train episode 352: reward = -141.00, steps = 141
    00:02:29 [INFO] train episode 353: reward = -96.00, steps = 96
    00:02:29 [INFO] train episode 354: reward = -151.00, steps = 151
    00:02:29 [INFO] train episode 355: reward = -112.00, steps = 112
    00:02:29 [INFO] train episode 356: reward = -107.00, steps = 107
    00:02:29 [INFO] train episode 357: reward = -109.00, steps = 109
    00:02:29 [INFO] train episode 358: reward = -144.00, steps = 144
    00:02:30 [INFO] train episode 359: reward = -139.00, steps = 139
    00:02:30 [INFO] train episode 360: reward = -159.00, steps = 159
    00:02:30 [INFO] train episode 361: reward = -138.00, steps = 138
    00:02:30 [INFO] train episode 362: reward = -123.00, steps = 123
    00:02:30 [INFO] train episode 363: reward = -159.00, steps = 159
    00:02:30 [INFO] train episode 364: reward = -109.00, steps = 109
    00:02:30 [INFO] train episode 365: reward = -158.00, steps = 158
    00:02:30 [INFO] train episode 366: reward = -120.00, steps = 120
    00:02:30 [INFO] train episode 367: reward = -149.00, steps = 149
    00:02:30 [INFO] train episode 368: reward = -111.00, steps = 111
    00:02:30 [INFO] train episode 369: reward = -200.00, steps = 200
    00:02:30 [INFO] train episode 370: reward = -90.00, steps = 90
    00:02:31 [INFO] train episode 371: reward = -145.00, steps = 145
    00:02:31 [INFO] train episode 372: reward = -162.00, steps = 162
    00:02:31 [INFO] train episode 373: reward = -88.00, steps = 88
    00:02:31 [INFO] train episode 374: reward = -110.00, steps = 110
    00:02:31 [INFO] train episode 375: reward = -87.00, steps = 87
    00:02:31 [INFO] train episode 376: reward = -153.00, steps = 153
    00:02:31 [INFO] train episode 377: reward = -149.00, steps = 149
    00:02:31 [INFO] train episode 378: reward = -149.00, steps = 149
    00:02:31 [INFO] train episode 379: reward = -148.00, steps = 148
    00:02:31 [INFO] train episode 380: reward = -133.00, steps = 133
    00:02:31 [INFO] train episode 381: reward = -198.00, steps = 198
    00:02:31 [INFO] train episode 382: reward = -105.00, steps = 105
    00:02:32 [INFO] train episode 383: reward = -107.00, steps = 107
    00:02:32 [INFO] train episode 384: reward = -145.00, steps = 145
    00:02:32 [INFO] train episode 385: reward = -140.00, steps = 140
    00:02:32 [INFO] train episode 386: reward = -108.00, steps = 108
    00:02:32 [INFO] train episode 387: reward = -144.00, steps = 144
    00:02:32 [INFO] train episode 388: reward = -110.00, steps = 110
    00:02:32 [INFO] train episode 389: reward = -111.00, steps = 111
    00:02:32 [INFO] train episode 390: reward = -161.00, steps = 161
    00:02:32 [INFO] train episode 391: reward = -105.00, steps = 105
    00:02:32 [INFO] train episode 392: reward = -146.00, steps = 146
    00:02:32 [INFO] train episode 393: reward = -145.00, steps = 145
    00:02:32 [INFO] train episode 394: reward = -94.00, steps = 94
    00:02:32 [INFO] train episode 395: reward = -108.00, steps = 108
    00:02:33 [INFO] train episode 396: reward = -148.00, steps = 148
    00:02:33 [INFO] train episode 397: reward = -144.00, steps = 144
    00:02:33 [INFO] train episode 398: reward = -90.00, steps = 90
    00:02:33 [INFO] train episode 399: reward = -97.00, steps = 97
    00:02:33 [INFO] train episode 400: reward = -107.00, steps = 107
    00:02:33 [INFO] train episode 401: reward = -95.00, steps = 95
    00:02:33 [INFO] train episode 402: reward = -142.00, steps = 142
    00:02:33 [INFO] train episode 403: reward = -106.00, steps = 106
    00:02:33 [INFO] train episode 404: reward = -147.00, steps = 147
    00:02:33 [INFO] train episode 405: reward = -86.00, steps = 86
    00:02:33 [INFO] train episode 406: reward = -108.00, steps = 108
    00:02:33 [INFO] train episode 407: reward = -109.00, steps = 109
    00:02:33 [INFO] train episode 408: reward = -171.00, steps = 171
    00:02:33 [INFO] train episode 409: reward = -149.00, steps = 149
    00:02:34 [INFO] train episode 410: reward = -141.00, steps = 141
    00:02:34 [INFO] train episode 411: reward = -117.00, steps = 117
    00:02:34 [INFO] train episode 412: reward = -95.00, steps = 95
    00:02:34 [INFO] train episode 413: reward = -109.00, steps = 109
    00:02:34 [INFO] train episode 414: reward = -140.00, steps = 140
    00:02:34 [INFO] train episode 415: reward = -91.00, steps = 91
    00:02:34 [INFO] train episode 416: reward = -113.00, steps = 113
    00:02:34 [INFO] train episode 417: reward = -87.00, steps = 87
    00:02:34 [INFO] train episode 418: reward = -140.00, steps = 140
    00:02:34 [INFO] train episode 419: reward = -84.00, steps = 84
    00:02:34 [INFO] train episode 420: reward = -91.00, steps = 91
    00:02:34 [INFO] train episode 421: reward = -145.00, steps = 145
    00:02:34 [INFO] train episode 422: reward = -166.00, steps = 166
    00:02:35 [INFO] train episode 423: reward = -167.00, steps = 167
    00:02:35 [INFO] train episode 424: reward = -93.00, steps = 93
    00:02:35 [INFO] train episode 425: reward = -120.00, steps = 120
    00:02:35 [INFO] train episode 426: reward = -157.00, steps = 157
    00:02:35 [INFO] train episode 427: reward = -150.00, steps = 150
    00:02:35 [INFO] train episode 428: reward = -147.00, steps = 147
    00:02:35 [INFO] train episode 429: reward = -142.00, steps = 142
    00:02:35 [INFO] train episode 430: reward = -145.00, steps = 145
    00:02:35 [INFO] train episode 431: reward = -111.00, steps = 111
    00:02:35 [INFO] train episode 432: reward = -144.00, steps = 144
    00:02:35 [INFO] train episode 433: reward = -148.00, steps = 148
    00:02:35 [INFO] train episode 434: reward = -141.00, steps = 141
    00:02:36 [INFO] train episode 435: reward = -152.00, steps = 152
    00:02:36 [INFO] train episode 436: reward = -107.00, steps = 107
    00:02:36 [INFO] train episode 437: reward = -108.00, steps = 108
    00:02:36 [INFO] train episode 438: reward = -89.00, steps = 89
    00:02:36 [INFO] train episode 439: reward = -86.00, steps = 86
    00:02:36 [INFO] train episode 440: reward = -84.00, steps = 84
    00:02:36 [INFO] train episode 441: reward = -110.00, steps = 110
    00:02:36 [INFO] train episode 442: reward = -144.00, steps = 144
    00:02:36 [INFO] train episode 443: reward = -109.00, steps = 109
    00:02:36 [INFO] train episode 444: reward = -88.00, steps = 88
    00:02:36 [INFO] train episode 445: reward = -150.00, steps = 150
    00:02:36 [INFO] train episode 446: reward = -96.00, steps = 96
    00:02:36 [INFO] train episode 447: reward = -162.00, steps = 162
    00:02:36 [INFO] train episode 448: reward = -107.00, steps = 107
    00:02:36 [INFO] train episode 449: reward = -86.00, steps = 86
    00:02:37 [INFO] train episode 450: reward = -150.00, steps = 150
    00:02:37 [INFO] train episode 451: reward = -138.00, steps = 138
    00:02:37 [INFO] train episode 452: reward = -84.00, steps = 84
    00:02:37 [INFO] train episode 453: reward = -141.00, steps = 141
    00:02:37 [INFO] train episode 454: reward = -107.00, steps = 107
    00:02:37 [INFO] train episode 455: reward = -143.00, steps = 143
    00:02:37 [INFO] train episode 456: reward = -135.00, steps = 135
    00:02:37 [INFO] train episode 457: reward = -139.00, steps = 139
    00:02:37 [INFO] train episode 458: reward = -158.00, steps = 158
    00:02:37 [INFO] train episode 459: reward = -87.00, steps = 87
    00:02:37 [INFO] train episode 460: reward = -108.00, steps = 108
    00:02:37 [INFO] train episode 461: reward = -87.00, steps = 87
    00:02:37 [INFO] train episode 462: reward = -87.00, steps = 87
    00:02:37 [INFO] train episode 463: reward = -85.00, steps = 85
    00:02:38 [INFO] train episode 464: reward = -167.00, steps = 167
    00:02:38 [INFO] train episode 465: reward = -85.00, steps = 85
    00:02:38 [INFO] train episode 466: reward = -105.00, steps = 105
    00:02:38 [INFO] train episode 467: reward = -153.00, steps = 153
    00:02:38 [INFO] train episode 468: reward = -89.00, steps = 89
    00:02:38 [INFO] train episode 469: reward = -153.00, steps = 153
    00:02:38 [INFO] train episode 470: reward = -150.00, steps = 150
    00:02:38 [INFO] train episode 471: reward = -145.00, steps = 145
    00:02:38 [INFO] train episode 472: reward = -153.00, steps = 153
    00:02:38 [INFO] train episode 473: reward = -90.00, steps = 90
    00:02:38 [INFO] train episode 474: reward = -90.00, steps = 90
    00:02:38 [INFO] train episode 475: reward = -111.00, steps = 111
    00:02:38 [INFO] train episode 476: reward = -107.00, steps = 107
    00:02:39 [INFO] train episode 477: reward = -88.00, steps = 88
    00:02:39 [INFO] train episode 478: reward = -147.00, steps = 147
    00:02:39 [INFO] train episode 479: reward = -138.00, steps = 138
    00:02:39 [INFO] train episode 480: reward = -96.00, steps = 96
    00:02:39 [INFO] train episode 481: reward = -89.00, steps = 89
    00:02:39 [INFO] train episode 482: reward = -197.00, steps = 197
    00:02:39 [INFO] train episode 483: reward = -110.00, steps = 110
    00:02:39 [INFO] train episode 484: reward = -106.00, steps = 106
    00:02:39 [INFO] train episode 485: reward = -159.00, steps = 159
    00:02:39 [INFO] train episode 486: reward = -90.00, steps = 90
    00:02:39 [INFO] train episode 487: reward = -151.00, steps = 151
    00:02:39 [INFO] train episode 488: reward = -105.00, steps = 105
    00:02:39 [INFO] train episode 489: reward = -145.00, steps = 145
    00:02:40 [INFO] train episode 490: reward = -138.00, steps = 138
    00:02:40 [INFO] train episode 491: reward = -105.00, steps = 105
    00:02:40 [INFO] train episode 492: reward = -140.00, steps = 140
    00:02:40 [INFO] train episode 493: reward = -97.00, steps = 97
    00:02:40 [INFO] train episode 494: reward = -90.00, steps = 90
    00:02:40 [INFO] train episode 495: reward = -145.00, steps = 145
    00:02:40 [INFO] train episode 496: reward = -93.00, steps = 93
    00:02:40 [INFO] train episode 497: reward = -137.00, steps = 137
    00:02:40 [INFO] train episode 498: reward = -164.00, steps = 164
    00:02:40 [INFO] train episode 499: reward = -85.00, steps = 85
    00:02:40 [INFO] train episode 500: reward = -158.00, steps = 158
    00:02:40 [INFO] train episode 501: reward = -86.00, steps = 86
    00:02:40 [INFO] train episode 502: reward = -131.00, steps = 131
    00:02:41 [INFO] train episode 503: reward = -148.00, steps = 148
    00:02:41 [INFO] train episode 504: reward = -108.00, steps = 108
    00:02:41 [INFO] train episode 505: reward = -87.00, steps = 87
    00:02:41 [INFO] train episode 506: reward = -146.00, steps = 146
    00:02:41 [INFO] train episode 507: reward = -134.00, steps = 134
    00:02:41 [INFO] train episode 508: reward = -89.00, steps = 89
    00:02:41 [INFO] train episode 509: reward = -147.00, steps = 147
    00:02:41 [INFO] train episode 510: reward = -161.00, steps = 161
    00:02:41 [INFO] train episode 511: reward = -97.00, steps = 97
    00:02:41 [INFO] train episode 512: reward = -122.00, steps = 122
    00:02:41 [INFO] train episode 513: reward = -147.00, steps = 147
    00:02:41 [INFO] train episode 514: reward = -175.00, steps = 175
    00:02:41 [INFO] train episode 515: reward = -100.00, steps = 100
    00:02:42 [INFO] train episode 516: reward = -107.00, steps = 107
    00:02:42 [INFO] train episode 517: reward = -91.00, steps = 91
    00:02:42 [INFO] train episode 518: reward = -137.00, steps = 137
    00:02:42 [INFO] train episode 519: reward = -135.00, steps = 135
    00:02:42 [INFO] train episode 520: reward = -103.00, steps = 103
    00:02:42 [INFO] train episode 521: reward = -91.00, steps = 91
    00:02:42 [INFO] train episode 522: reward = -108.00, steps = 108
    00:02:42 [INFO] train episode 523: reward = -106.00, steps = 106
    00:02:42 [INFO] train episode 524: reward = -118.00, steps = 118
    00:02:42 [INFO] train episode 525: reward = -94.00, steps = 94
    00:02:42 [INFO] train episode 526: reward = -147.00, steps = 147
    00:02:42 [INFO] train episode 527: reward = -163.00, steps = 163
    00:02:42 [INFO] train episode 528: reward = -90.00, steps = 90
    00:02:42 [INFO] train episode 529: reward = -137.00, steps = 137
    00:02:43 [INFO] train episode 530: reward = -149.00, steps = 149
    00:02:43 [INFO] train episode 531: reward = -90.00, steps = 90
    00:02:43 [INFO] train episode 532: reward = -132.00, steps = 132
    00:02:43 [INFO] train episode 533: reward = -144.00, steps = 144
    00:02:43 [INFO] train episode 534: reward = -142.00, steps = 142
    00:02:43 [INFO] train episode 535: reward = -84.00, steps = 84
    00:02:43 [INFO] train episode 536: reward = -129.00, steps = 129
    00:02:43 [INFO] train episode 537: reward = -136.00, steps = 136
    00:02:43 [INFO] train episode 538: reward = -145.00, steps = 145
    00:02:43 [INFO] train episode 539: reward = -104.00, steps = 104
    00:02:43 [INFO] train episode 540: reward = -138.00, steps = 138
    00:02:43 [INFO] train episode 541: reward = -141.00, steps = 141
    00:02:43 [INFO] train episode 542: reward = -88.00, steps = 88
    00:02:44 [INFO] train episode 543: reward = -134.00, steps = 134
    00:02:44 [INFO] train episode 544: reward = -133.00, steps = 133
    00:02:44 [INFO] train episode 545: reward = -104.00, steps = 104
    00:02:44 [INFO] train episode 546: reward = -170.00, steps = 170
    00:02:44 [INFO] train episode 547: reward = -105.00, steps = 105
    00:02:44 [INFO] train episode 548: reward = -104.00, steps = 104
    00:02:44 [INFO] train episode 549: reward = -104.00, steps = 104
    00:02:44 [INFO] train episode 550: reward = -147.00, steps = 147
    00:02:44 [INFO] train episode 551: reward = -87.00, steps = 87
    00:02:44 [INFO] train episode 552: reward = -108.00, steps = 108
    00:02:44 [INFO] train episode 553: reward = -95.00, steps = 95
    00:02:44 [INFO] train episode 554: reward = -88.00, steps = 88
    00:02:44 [INFO] train episode 555: reward = -105.00, steps = 105
    00:02:44 [INFO] train episode 556: reward = -93.00, steps = 93
    00:02:45 [INFO] train episode 557: reward = -88.00, steps = 88
    00:02:45 [INFO] train episode 558: reward = -101.00, steps = 101
    00:02:45 [INFO] train episode 559: reward = -89.00, steps = 89
    00:02:45 [INFO] train episode 560: reward = -154.00, steps = 154
    00:02:45 [INFO] train episode 561: reward = -98.00, steps = 98
    00:02:45 [INFO] train episode 562: reward = -169.00, steps = 169
    00:02:45 [INFO] train episode 563: reward = -145.00, steps = 145
    00:02:45 [INFO] train episode 564: reward = -154.00, steps = 154
    00:02:45 [INFO] train episode 565: reward = -151.00, steps = 151
    00:02:45 [INFO] train episode 566: reward = -85.00, steps = 85
    00:02:45 [INFO] train episode 567: reward = -101.00, steps = 101
    00:02:45 [INFO] train episode 568: reward = -92.00, steps = 92
    00:02:45 [INFO] train episode 569: reward = -90.00, steps = 90
    00:02:45 [INFO] train episode 570: reward = -93.00, steps = 93
    00:02:46 [INFO] train episode 571: reward = -95.00, steps = 95
    00:02:46 [INFO] train episode 572: reward = -182.00, steps = 182
    00:02:46 [INFO] train episode 573: reward = -135.00, steps = 135
    00:02:46 [INFO] train episode 574: reward = -156.00, steps = 156
    00:02:46 [INFO] train episode 575: reward = -126.00, steps = 126
    00:02:46 [INFO] train episode 576: reward = -89.00, steps = 89
    00:02:46 [INFO] train episode 577: reward = -101.00, steps = 101
    00:02:46 [INFO] train episode 578: reward = -134.00, steps = 134
    00:02:46 [INFO] train episode 579: reward = -132.00, steps = 132
    00:02:46 [INFO] train episode 580: reward = -110.00, steps = 110
    00:02:46 [INFO] train episode 581: reward = -162.00, steps = 162
    00:02:46 [INFO] train episode 582: reward = -115.00, steps = 115
    00:02:46 [INFO] train episode 583: reward = -108.00, steps = 108
    00:02:47 [INFO] train episode 584: reward = -84.00, steps = 84
    00:02:47 [INFO] train episode 585: reward = -114.00, steps = 114
    00:02:47 [INFO] train episode 586: reward = -104.00, steps = 104
    00:02:47 [INFO] train episode 587: reward = -104.00, steps = 104
    00:02:47 [INFO] train episode 588: reward = -83.00, steps = 83
    00:02:47 [INFO] train episode 589: reward = -101.00, steps = 101
    00:02:47 [INFO] train episode 590: reward = -98.00, steps = 98
    00:02:47 [INFO] train episode 591: reward = -147.00, steps = 147
    00:02:47 [INFO] train episode 592: reward = -104.00, steps = 104
    00:02:47 [INFO] train episode 593: reward = -113.00, steps = 113
    00:02:47 [INFO] train episode 594: reward = -134.00, steps = 134
    00:02:47 [INFO] train episode 595: reward = -155.00, steps = 155
    00:02:47 [INFO] train episode 596: reward = -149.00, steps = 149
    00:02:47 [INFO] train episode 597: reward = -151.00, steps = 151
    00:02:48 [INFO] train episode 598: reward = -83.00, steps = 83
    00:02:48 [INFO] train episode 599: reward = -170.00, steps = 170
    00:02:48 [INFO] train episode 600: reward = -133.00, steps = 133
    00:02:48 [INFO] train episode 601: reward = -100.00, steps = 100
    00:02:48 [INFO] train episode 602: reward = -141.00, steps = 141
    00:02:48 [INFO] train episode 603: reward = -133.00, steps = 133
    00:02:48 [INFO] train episode 604: reward = -134.00, steps = 134
    00:02:48 [INFO] train episode 605: reward = -108.00, steps = 108
    00:02:48 [INFO] train episode 606: reward = -115.00, steps = 115
    00:02:48 [INFO] train episode 607: reward = -154.00, steps = 154
    00:02:48 [INFO] train episode 608: reward = -145.00, steps = 145
    00:02:48 [INFO] train episode 609: reward = -145.00, steps = 145
    00:02:49 [INFO] train episode 610: reward = -134.00, steps = 134
    00:02:49 [INFO] train episode 611: reward = -148.00, steps = 148
    00:02:49 [INFO] train episode 612: reward = -116.00, steps = 116
    00:02:49 [INFO] train episode 613: reward = -145.00, steps = 145
    00:02:49 [INFO] train episode 614: reward = -104.00, steps = 104
    00:02:49 [INFO] train episode 615: reward = -112.00, steps = 112
    00:02:49 [INFO] train episode 616: reward = -138.00, steps = 138
    00:02:49 [INFO] train episode 617: reward = -104.00, steps = 104
    00:02:49 [INFO] train episode 618: reward = -141.00, steps = 141
    00:02:49 [INFO] train episode 619: reward = -104.00, steps = 104
    00:02:49 [INFO] train episode 620: reward = -163.00, steps = 163
    00:02:49 [INFO] train episode 621: reward = -143.00, steps = 143
    00:02:49 [INFO] train episode 622: reward = -129.00, steps = 129
    00:02:50 [INFO] train episode 623: reward = -104.00, steps = 104
    00:02:50 [INFO] train episode 624: reward = -104.00, steps = 104
    00:02:50 [INFO] train episode 625: reward = -138.00, steps = 138
    00:02:50 [INFO] train episode 626: reward = -138.00, steps = 138
    00:02:50 [INFO] train episode 627: reward = -104.00, steps = 104
    00:02:50 [INFO] train episode 628: reward = -84.00, steps = 84
    00:02:50 [INFO] train episode 629: reward = -154.00, steps = 154
    00:02:50 [INFO] train episode 630: reward = -148.00, steps = 148
    00:02:50 [INFO] train episode 631: reward = -90.00, steps = 90
    00:02:50 [INFO] train episode 632: reward = -104.00, steps = 104
    00:02:50 [INFO] train episode 633: reward = -105.00, steps = 105
    00:02:50 [INFO] train episode 634: reward = -90.00, steps = 90
    00:02:50 [INFO] train episode 635: reward = -86.00, steps = 86
    00:02:50 [INFO] train episode 636: reward = -142.00, steps = 142
    00:02:51 [INFO] train episode 637: reward = -140.00, steps = 140
    00:02:51 [INFO] train episode 638: reward = -141.00, steps = 141
    00:02:51 [INFO] train episode 639: reward = -87.00, steps = 87
    00:02:51 [INFO] train episode 640: reward = -157.00, steps = 157
    00:02:51 [INFO] train episode 641: reward = -104.00, steps = 104
    00:02:51 [INFO] train episode 642: reward = -156.00, steps = 156
    00:02:51 [INFO] train episode 643: reward = -104.00, steps = 104
    00:02:51 [INFO] train episode 644: reward = -146.00, steps = 146
    00:02:51 [INFO] train episode 645: reward = -149.00, steps = 149
    00:02:51 [INFO] train episode 646: reward = -145.00, steps = 145
    00:02:51 [INFO] train episode 647: reward = -154.00, steps = 154
    00:02:51 [INFO] train episode 648: reward = -152.00, steps = 152
    00:02:52 [INFO] train episode 649: reward = -145.00, steps = 145
    00:02:52 [INFO] train episode 650: reward = -147.00, steps = 147
    00:02:52 [INFO] train episode 651: reward = -106.00, steps = 106
    00:02:52 [INFO] train episode 652: reward = -149.00, steps = 149
    00:02:52 [INFO] train episode 653: reward = -144.00, steps = 144
    00:02:52 [INFO] train episode 654: reward = -135.00, steps = 135
    00:02:52 [INFO] train episode 655: reward = -137.00, steps = 137
    00:02:52 [INFO] train episode 656: reward = -168.00, steps = 168
    00:02:52 [INFO] train episode 657: reward = -97.00, steps = 97
    00:02:52 [INFO] train episode 658: reward = -106.00, steps = 106
    00:02:52 [INFO] train episode 659: reward = -106.00, steps = 106
    00:02:52 [INFO] train episode 660: reward = -141.00, steps = 141
    00:02:52 [INFO] train episode 661: reward = -114.00, steps = 114
    00:02:53 [INFO] train episode 662: reward = -106.00, steps = 106
    00:02:53 [INFO] train episode 663: reward = -105.00, steps = 105
    00:02:53 [INFO] train episode 664: reward = -161.00, steps = 161
    00:02:53 [INFO] train episode 665: reward = -134.00, steps = 134
    00:02:53 [INFO] train episode 666: reward = -159.00, steps = 159
    00:02:53 [INFO] train episode 667: reward = -104.00, steps = 104
    00:02:53 [INFO] train episode 668: reward = -141.00, steps = 141
    00:02:53 [INFO] train episode 669: reward = -104.00, steps = 104
    00:02:53 [INFO] train episode 670: reward = -146.00, steps = 146
    00:02:53 [INFO] train episode 671: reward = -153.00, steps = 153
    00:02:53 [INFO] train episode 672: reward = -147.00, steps = 147
    00:02:53 [INFO] train episode 673: reward = -86.00, steps = 86
    00:02:53 [INFO] train episode 674: reward = -144.00, steps = 144
    00:02:54 [INFO] train episode 675: reward = -124.00, steps = 124
    00:02:54 [INFO] train episode 676: reward = -127.00, steps = 127
    00:02:54 [INFO] train episode 677: reward = -106.00, steps = 106
    00:02:54 [INFO] train episode 678: reward = -157.00, steps = 157
    00:02:54 [INFO] train episode 679: reward = -129.00, steps = 129
    00:02:54 [INFO] train episode 680: reward = -104.00, steps = 104
    00:02:54 [INFO] train episode 681: reward = -151.00, steps = 151
    00:02:54 [INFO] train episode 682: reward = -145.00, steps = 145
    00:02:54 [INFO] train episode 683: reward = -87.00, steps = 87
    00:02:54 [INFO] train episode 684: reward = -145.00, steps = 145
    00:02:54 [INFO] train episode 685: reward = -139.00, steps = 139
    00:02:54 [INFO] train episode 686: reward = -153.00, steps = 153
    00:02:55 [INFO] train episode 687: reward = -105.00, steps = 105
    00:02:55 [INFO] train episode 688: reward = -105.00, steps = 105
    00:02:55 [INFO] train episode 689: reward = -104.00, steps = 104
    00:02:55 [INFO] train episode 690: reward = -85.00, steps = 85
    00:02:55 [INFO] train episode 691: reward = -89.00, steps = 89
    00:02:55 [INFO] train episode 692: reward = -105.00, steps = 105
    00:02:55 [INFO] train episode 693: reward = -85.00, steps = 85
    00:02:55 [INFO] train episode 694: reward = -151.00, steps = 151
    00:02:55 [INFO] train episode 695: reward = -104.00, steps = 104
    00:02:55 [INFO] train episode 696: reward = -149.00, steps = 149
    00:02:55 [INFO] train episode 697: reward = -104.00, steps = 104
    00:02:55 [INFO] train episode 698: reward = -200.00, steps = 200
    00:02:55 [INFO] train episode 699: reward = -167.00, steps = 167
    00:02:55 [INFO] train episode 700: reward = -104.00, steps = 104
    00:02:56 [INFO] train episode 701: reward = -89.00, steps = 89
    00:02:56 [INFO] train episode 702: reward = -104.00, steps = 104
    00:02:56 [INFO] train episode 703: reward = -104.00, steps = 104
    00:02:56 [INFO] train episode 704: reward = -105.00, steps = 105
    00:02:56 [INFO] train episode 705: reward = -104.00, steps = 104
    00:02:56 [INFO] train episode 706: reward = -105.00, steps = 105
    00:02:56 [INFO] train episode 707: reward = -105.00, steps = 105
    00:02:56 [INFO] train episode 708: reward = -104.00, steps = 104
    00:02:56 [INFO] train episode 709: reward = -161.00, steps = 161
    00:02:56 [INFO] train episode 710: reward = -104.00, steps = 104
    00:02:56 [INFO] train episode 711: reward = -104.00, steps = 104
    00:02:56 [INFO] train episode 712: reward = -104.00, steps = 104
    00:02:56 [INFO] train episode 713: reward = -104.00, steps = 104
    00:02:56 [INFO] train episode 714: reward = -104.00, steps = 104
    00:02:56 [INFO] train episode 715: reward = -105.00, steps = 105
    00:02:56 [INFO] train episode 716: reward = -90.00, steps = 90
    00:02:57 [INFO] train episode 717: reward = -161.00, steps = 161
    00:02:57 [INFO] train episode 718: reward = -90.00, steps = 90
    00:02:57 [INFO] train episode 719: reward = -160.00, steps = 160
    00:02:57 [INFO] train episode 720: reward = -104.00, steps = 104
    00:02:57 [INFO] train episode 721: reward = -139.00, steps = 139
    00:02:57 [INFO] train episode 722: reward = -146.00, steps = 146
    00:02:57 [INFO] train episode 723: reward = -104.00, steps = 104
    00:02:57 [INFO] train episode 724: reward = -104.00, steps = 104
    00:02:57 [INFO] train episode 725: reward = -136.00, steps = 136
    00:02:57 [INFO] train episode 726: reward = -104.00, steps = 104
    00:02:57 [INFO] train episode 727: reward = -134.00, steps = 134
    00:02:57 [INFO] train episode 728: reward = -101.00, steps = 101
    00:02:57 [INFO] train episode 729: reward = -134.00, steps = 134
    00:02:58 [INFO] train episode 730: reward = -104.00, steps = 104
    00:02:58 [INFO] train episode 731: reward = -104.00, steps = 104
    00:02:58 [INFO] train episode 732: reward = -134.00, steps = 134
    00:02:58 [INFO] train episode 733: reward = -105.00, steps = 105
    00:02:58 [INFO] train episode 734: reward = -134.00, steps = 134
    00:02:58 [INFO] train episode 735: reward = -104.00, steps = 104
    00:02:58 [INFO] train episode 736: reward = -157.00, steps = 157
    00:02:58 [INFO] train episode 737: reward = -104.00, steps = 104
    00:02:58 [INFO] train episode 738: reward = -152.00, steps = 152
    00:02:58 [INFO] train episode 739: reward = -153.00, steps = 153
    00:02:58 [INFO] train episode 740: reward = -104.00, steps = 104
    00:02:58 [INFO] train episode 741: reward = -104.00, steps = 104
    00:02:58 [INFO] train episode 742: reward = -148.00, steps = 148
    00:02:59 [INFO] train episode 743: reward = -105.00, steps = 105
    00:02:59 [INFO] train episode 744: reward = -146.00, steps = 146
    00:02:59 [INFO] train episode 745: reward = -106.00, steps = 106
    00:02:59 [INFO] train episode 746: reward = -146.00, steps = 146
    00:02:59 [INFO] train episode 747: reward = -161.00, steps = 161
    00:02:59 [INFO] train episode 748: reward = -86.00, steps = 86
    00:02:59 [INFO] train episode 749: reward = -85.00, steps = 85
    00:02:59 [INFO] train episode 750: reward = -160.00, steps = 160
    00:02:59 [INFO] train episode 751: reward = -109.00, steps = 109
    00:02:59 [INFO] train episode 752: reward = -153.00, steps = 153
    00:02:59 [INFO] train episode 753: reward = -108.00, steps = 108
    00:02:59 [INFO] train episode 754: reward = -144.00, steps = 144
    00:02:59 [INFO] train episode 755: reward = -93.00, steps = 93
    00:03:00 [INFO] train episode 756: reward = -110.00, steps = 110
    00:03:00 [INFO] train episode 757: reward = -96.00, steps = 96
    00:03:00 [INFO] train episode 758: reward = -88.00, steps = 88
    00:03:00 [INFO] train episode 759: reward = -110.00, steps = 110
    00:03:00 [INFO] train episode 760: reward = -86.00, steps = 86
    00:03:00 [INFO] train episode 761: reward = -169.00, steps = 169
    00:03:00 [INFO] train episode 762: reward = -109.00, steps = 109
    00:03:00 [INFO] train episode 763: reward = -86.00, steps = 86
    00:03:00 [INFO] train episode 764: reward = -85.00, steps = 85
    00:03:00 [INFO] train episode 765: reward = -165.00, steps = 165
    00:03:00 [INFO] train episode 766: reward = -150.00, steps = 150
    00:03:00 [INFO] train episode 767: reward = -168.00, steps = 168
    00:03:00 [INFO] train episode 768: reward = -86.00, steps = 86
    00:03:00 [INFO] train episode 769: reward = -91.00, steps = 91
    00:03:00 [INFO] train episode 770: reward = -85.00, steps = 85
    00:03:01 [INFO] train episode 771: reward = -150.00, steps = 150
    00:03:01 [INFO] train episode 772: reward = -110.00, steps = 110
    00:03:01 [INFO] train episode 773: reward = -110.00, steps = 110
    00:03:01 [INFO] train episode 774: reward = -110.00, steps = 110
    00:03:01 [INFO] train episode 775: reward = -108.00, steps = 108
    00:03:01 [INFO] train episode 776: reward = -107.00, steps = 107
    00:03:01 [INFO] train episode 777: reward = -155.00, steps = 155
    00:03:01 [INFO] train episode 778: reward = -107.00, steps = 107
    00:03:01 [INFO] train episode 779: reward = -153.00, steps = 153
    00:03:01 [INFO] train episode 780: reward = -105.00, steps = 105
    00:03:01 [INFO] train episode 781: reward = -104.00, steps = 104
    00:03:01 [INFO] train episode 782: reward = -106.00, steps = 106
    00:03:01 [INFO] train episode 783: reward = -105.00, steps = 105
    00:03:01 [INFO] train episode 784: reward = -104.00, steps = 104
    00:03:02 [INFO] train episode 785: reward = -156.00, steps = 156
    00:03:02 [INFO] train episode 786: reward = -83.00, steps = 83
    00:03:02 [INFO] train episode 787: reward = -106.00, steps = 106
    00:03:02 [INFO] train episode 788: reward = -105.00, steps = 105
    00:03:02 [INFO] train episode 789: reward = -106.00, steps = 106
    00:03:02 [INFO] train episode 790: reward = -96.00, steps = 96
    00:03:02 [INFO] train episode 791: reward = -85.00, steps = 85
    00:03:02 [INFO] train episode 792: reward = -171.00, steps = 171
    00:03:02 [INFO] train episode 793: reward = -102.00, steps = 102
    00:03:02 [INFO] train episode 794: reward = -103.00, steps = 103
    00:03:02 [INFO] train episode 795: reward = -171.00, steps = 171
    00:03:02 [INFO] train episode 796: reward = -153.00, steps = 153
    00:03:02 [INFO] train episode 797: reward = -104.00, steps = 104
    00:03:02 [INFO] train episode 798: reward = -104.00, steps = 104
    00:03:03 [INFO] train episode 799: reward = -150.00, steps = 150
    00:03:03 [INFO] train episode 800: reward = -104.00, steps = 104
    00:03:03 [INFO] train episode 801: reward = -106.00, steps = 106
    00:03:03 [INFO] train episode 802: reward = -87.00, steps = 87
    00:03:03 [INFO] train episode 803: reward = -147.00, steps = 147
    00:03:03 [INFO] train episode 804: reward = -89.00, steps = 89
    00:03:03 [INFO] train episode 805: reward = -107.00, steps = 107
    00:03:03 [INFO] train episode 806: reward = -103.00, steps = 103
    00:03:03 [INFO] train episode 807: reward = -165.00, steps = 165
    00:03:03 [INFO] train episode 808: reward = -104.00, steps = 104
    00:03:03 [INFO] train episode 809: reward = -98.00, steps = 98
    00:03:03 [INFO] train episode 810: reward = -104.00, steps = 104
    00:03:03 [INFO] train episode 811: reward = -92.00, steps = 92
    00:03:03 [INFO] train episode 812: reward = -103.00, steps = 103
    00:03:03 [INFO] train episode 813: reward = -106.00, steps = 106
    00:03:04 [INFO] train episode 814: reward = -104.00, steps = 104
    00:03:04 [INFO] train episode 815: reward = -157.00, steps = 157
    00:03:04 [INFO] train episode 816: reward = -108.00, steps = 108
    00:03:04 [INFO] train episode 817: reward = -107.00, steps = 107
    00:03:04 [INFO] train episode 818: reward = -107.00, steps = 107
    00:03:04 [INFO] train episode 819: reward = -106.00, steps = 106
    00:03:04 [INFO] train episode 820: reward = -90.00, steps = 90
    00:03:04 [INFO] train episode 821: reward = -87.00, steps = 87
    00:03:04 [INFO] train episode 822: reward = -108.00, steps = 108
    00:03:04 [INFO] train episode 823: reward = -104.00, steps = 104
    00:03:04 [INFO] train episode 824: reward = -99.00, steps = 99
    00:03:04 [INFO] train episode 825: reward = -107.00, steps = 107
    00:03:04 [INFO] train episode 826: reward = -104.00, steps = 104
    00:03:04 [INFO] train episode 827: reward = -90.00, steps = 90
    00:03:04 [INFO] train episode 828: reward = -106.00, steps = 106
    00:03:05 [INFO] train episode 829: reward = -152.00, steps = 152
    00:03:05 [INFO] train episode 830: reward = -105.00, steps = 105
    00:03:05 [INFO] train episode 831: reward = -164.00, steps = 164
    00:03:05 [INFO] train episode 832: reward = -84.00, steps = 84
    00:03:05 [INFO] train episode 833: reward = -148.00, steps = 148
    00:03:05 [INFO] train episode 834: reward = -142.00, steps = 142
    00:03:05 [INFO] train episode 835: reward = -89.00, steps = 89
    00:03:05 [INFO] train episode 836: reward = -106.00, steps = 106
    00:03:05 [INFO] train episode 837: reward = -172.00, steps = 172
    00:03:05 [INFO] train episode 838: reward = -85.00, steps = 85
    00:03:05 [INFO] train episode 839: reward = -150.00, steps = 150
    00:03:05 [INFO] train episode 840: reward = -85.00, steps = 85
    00:03:05 [INFO] train episode 841: reward = -148.00, steps = 148
    00:03:06 [INFO] train episode 842: reward = -154.00, steps = 154
    00:03:06 [INFO] train episode 843: reward = -85.00, steps = 85
    00:03:06 [INFO] train episode 844: reward = -154.00, steps = 154
    00:03:06 [INFO] train episode 845: reward = -147.00, steps = 147
    00:03:06 [INFO] train episode 846: reward = -94.00, steps = 94
    00:03:06 [INFO] train episode 847: reward = -105.00, steps = 105
    00:03:06 [INFO] train episode 848: reward = -150.00, steps = 150
    00:03:06 [INFO] train episode 849: reward = -159.00, steps = 159
    00:03:06 [INFO] train episode 850: reward = -106.00, steps = 106
    00:03:06 [INFO] train episode 851: reward = -94.00, steps = 94
    00:03:06 [INFO] train episode 852: reward = -142.00, steps = 142
    00:03:06 [INFO] train episode 853: reward = -153.00, steps = 153
    00:03:06 [INFO] train episode 854: reward = -142.00, steps = 142
    00:03:07 [INFO] train episode 855: reward = -90.00, steps = 90
    00:03:07 [INFO] train episode 856: reward = -144.00, steps = 144
    00:03:07 [INFO] train episode 857: reward = -140.00, steps = 140
    00:03:07 [INFO] train episode 858: reward = -165.00, steps = 165
    00:03:07 [INFO] train episode 859: reward = -106.00, steps = 106
    00:03:07 [INFO] train episode 860: reward = -108.00, steps = 108
    00:03:07 [INFO] train episode 861: reward = -98.00, steps = 98
    00:03:07 [INFO] train episode 862: reward = -104.00, steps = 104
    00:03:07 [INFO] train episode 863: reward = -111.00, steps = 111
    00:03:07 [INFO] train episode 864: reward = -105.00, steps = 105
    00:03:07 [INFO] train episode 865: reward = -188.00, steps = 188
    00:03:07 [INFO] train episode 866: reward = -105.00, steps = 105
    00:03:07 [INFO] train episode 867: reward = -151.00, steps = 151
    00:03:08 [INFO] train episode 868: reward = -146.00, steps = 146
    00:03:08 [INFO] train episode 869: reward = -158.00, steps = 158
    00:03:08 [INFO] train episode 870: reward = -106.00, steps = 106
    00:03:08 [INFO] train episode 871: reward = -107.00, steps = 107
    00:03:08 [INFO] train episode 872: reward = -109.00, steps = 109
    00:03:08 [INFO] train episode 873: reward = -92.00, steps = 92
    00:03:08 [INFO] train episode 874: reward = -157.00, steps = 157
    00:03:08 [INFO] train episode 875: reward = -104.00, steps = 104
    00:03:08 [INFO] train episode 876: reward = -103.00, steps = 103
    00:03:08 [INFO] train episode 877: reward = -105.00, steps = 105
    00:03:08 [INFO] train episode 878: reward = -145.00, steps = 145
    00:03:08 [INFO] train episode 879: reward = -148.00, steps = 148
    00:03:08 [INFO] train episode 880: reward = -106.00, steps = 106
    00:03:09 [INFO] train episode 881: reward = -105.00, steps = 105
    00:03:09 [INFO] train episode 882: reward = -149.00, steps = 149
    00:03:09 [INFO] train episode 883: reward = -149.00, steps = 149
    00:03:09 [INFO] train episode 884: reward = -148.00, steps = 148
    00:03:09 [INFO] train episode 885: reward = -154.00, steps = 154
    00:03:09 [INFO] train episode 886: reward = -147.00, steps = 147
    00:03:09 [INFO] train episode 887: reward = -154.00, steps = 154
    00:03:09 [INFO] train episode 888: reward = -107.00, steps = 107
    00:03:09 [INFO] train episode 889: reward = -107.00, steps = 107
    00:03:09 [INFO] train episode 890: reward = -106.00, steps = 106
    00:03:09 [INFO] train episode 891: reward = -102.00, steps = 102
    00:03:09 [INFO] train episode 892: reward = -106.00, steps = 106
    00:03:09 [INFO] train episode 893: reward = -159.00, steps = 159
    00:03:10 [INFO] train episode 894: reward = -107.00, steps = 107
    00:03:10 [INFO] train episode 895: reward = -162.00, steps = 162
    00:03:10 [INFO] train episode 896: reward = -112.00, steps = 112
    00:03:10 [INFO] train episode 897: reward = -156.00, steps = 156
    00:03:10 [INFO] train episode 898: reward = -175.00, steps = 175
    00:03:10 [INFO] train episode 899: reward = -107.00, steps = 107
    00:03:10 [INFO] train episode 900: reward = -107.00, steps = 107
    00:03:10 [INFO] train episode 901: reward = -103.00, steps = 103
    00:03:10 [INFO] train episode 902: reward = -106.00, steps = 106
    00:03:10 [INFO] train episode 903: reward = -106.00, steps = 106
    00:03:10 [INFO] train episode 904: reward = -106.00, steps = 106
    00:03:10 [INFO] train episode 905: reward = -88.00, steps = 88
    00:03:10 [INFO] train episode 906: reward = -88.00, steps = 88
    00:03:10 [INFO] train episode 907: reward = -107.00, steps = 107
    00:03:11 [INFO] train episode 908: reward = -105.00, steps = 105
    00:03:11 [INFO] train episode 909: reward = -96.00, steps = 96
    00:03:11 [INFO] train episode 910: reward = -106.00, steps = 106
    00:03:11 [INFO] train episode 911: reward = -107.00, steps = 107
    00:03:11 [INFO] train episode 912: reward = -84.00, steps = 84
    00:03:11 [INFO] train episode 913: reward = -108.00, steps = 108
    00:03:11 [INFO] train episode 914: reward = -92.00, steps = 92
    00:03:11 [INFO] train episode 915: reward = -107.00, steps = 107
    00:03:11 [INFO] train episode 916: reward = -108.00, steps = 108
    00:03:11 [INFO] train episode 917: reward = -107.00, steps = 107
    00:03:11 [INFO] train episode 918: reward = -108.00, steps = 108
    00:03:11 [INFO] train episode 919: reward = -84.00, steps = 84
    00:03:11 [INFO] train episode 920: reward = -108.00, steps = 108
    00:03:11 [INFO] train episode 921: reward = -106.00, steps = 106
    00:03:11 [INFO] train episode 922: reward = -107.00, steps = 107
    00:03:11 [INFO] train episode 923: reward = -83.00, steps = 83
    00:03:12 [INFO] train episode 924: reward = -86.00, steps = 86
    00:03:12 [INFO] train episode 925: reward = -85.00, steps = 85
    00:03:12 [INFO] train episode 926: reward = -107.00, steps = 107
    00:03:12 [INFO] train episode 927: reward = -88.00, steps = 88
    00:03:12 [INFO] train episode 928: reward = -106.00, steps = 106
    00:03:12 [INFO] train episode 929: reward = -106.00, steps = 106
    00:03:12 [INFO] train episode 930: reward = -106.00, steps = 106
    00:03:12 [INFO] train episode 931: reward = -161.00, steps = 161
    00:03:12 [INFO] train episode 932: reward = -86.00, steps = 86
    00:03:12 [INFO] train episode 933: reward = -107.00, steps = 107
    00:03:12 [INFO] train episode 934: reward = -106.00, steps = 106
    00:03:12 [INFO] train episode 935: reward = -106.00, steps = 106
    00:03:12 [INFO] train episode 936: reward = -106.00, steps = 106
    00:03:12 [INFO] train episode 937: reward = -153.00, steps = 153
    00:03:12 [INFO] train episode 938: reward = -90.00, steps = 90
    00:03:12 [INFO] train episode 939: reward = -85.00, steps = 85
    00:03:13 [INFO] train episode 940: reward = -106.00, steps = 106
    00:03:13 [INFO] train episode 941: reward = -155.00, steps = 155
    00:03:13 [INFO] train episode 942: reward = -107.00, steps = 107
    00:03:13 [INFO] train episode 943: reward = -86.00, steps = 86
    00:03:13 [INFO] train episode 944: reward = -147.00, steps = 147
    00:03:13 [INFO] train episode 945: reward = -153.00, steps = 153
    00:03:13 [INFO] train episode 946: reward = -106.00, steps = 106
    00:03:13 [INFO] train episode 947: reward = -107.00, steps = 107
    00:03:13 [INFO] train episode 948: reward = -158.00, steps = 158
    00:03:13 [INFO] train episode 949: reward = -148.00, steps = 148
    00:03:13 [INFO] train episode 950: reward = -106.00, steps = 106
    00:03:13 [INFO] train episode 951: reward = -150.00, steps = 150
    00:03:13 [INFO] train episode 952: reward = -107.00, steps = 107
    00:03:14 [INFO] train episode 953: reward = -106.00, steps = 106
    00:03:14 [INFO] train episode 954: reward = -105.00, steps = 105
    00:03:14 [INFO] train episode 955: reward = -106.00, steps = 106
    00:03:14 [INFO] train episode 956: reward = -104.00, steps = 104
    00:03:14 [INFO] train episode 957: reward = -105.00, steps = 105
    00:03:14 [INFO] train episode 958: reward = -158.00, steps = 158
    00:03:14 [INFO] train episode 959: reward = -148.00, steps = 148
    00:03:14 [INFO] train episode 960: reward = -107.00, steps = 107
    00:03:14 [INFO] train episode 961: reward = -150.00, steps = 150
    00:03:14 [INFO] train episode 962: reward = -105.00, steps = 105
    00:03:14 [INFO] train episode 963: reward = -105.00, steps = 105
    00:03:14 [INFO] train episode 964: reward = -104.00, steps = 104
    00:03:14 [INFO] train episode 965: reward = -107.00, steps = 107
    00:03:14 [INFO] train episode 966: reward = -108.00, steps = 108
    00:03:15 [INFO] train episode 967: reward = -104.00, steps = 104
    00:03:15 [INFO] train episode 968: reward = -92.00, steps = 92
    00:03:15 [INFO] train episode 969: reward = -92.00, steps = 92
    00:03:15 [INFO] train episode 970: reward = -86.00, steps = 86
    00:03:15 [INFO] train episode 971: reward = -107.00, steps = 107
    00:03:15 [INFO] train episode 972: reward = -100.00, steps = 100
    00:03:15 [INFO] train episode 973: reward = -148.00, steps = 148
    00:03:15 [INFO] train episode 974: reward = -158.00, steps = 158
    00:03:15 [INFO] train episode 975: reward = -106.00, steps = 106
    00:03:15 [INFO] train episode 976: reward = -123.00, steps = 123
    00:03:15 [INFO] train episode 977: reward = -148.00, steps = 148
    00:03:15 [INFO] train episode 978: reward = -145.00, steps = 145
    00:03:15 [INFO] train episode 979: reward = -107.00, steps = 107
    00:03:15 [INFO] train episode 980: reward = -148.00, steps = 148
    00:03:16 [INFO] train episode 981: reward = -104.00, steps = 104
    00:03:16 [INFO] train episode 982: reward = -147.00, steps = 147
    00:03:16 [INFO] train episode 983: reward = -147.00, steps = 147
    00:03:16 [INFO] train episode 984: reward = -107.00, steps = 107
    00:03:16 [INFO] train episode 985: reward = -107.00, steps = 107
    00:03:16 [INFO] train episode 986: reward = -106.00, steps = 106
    00:03:16 [INFO] train episode 987: reward = -100.00, steps = 100
    00:03:16 [INFO] train episode 988: reward = -104.00, steps = 104
    00:03:16 [INFO] train episode 989: reward = -104.00, steps = 104
    00:03:16 [INFO] train episode 990: reward = -107.00, steps = 107
    00:03:16 [INFO] train episode 991: reward = -162.00, steps = 162
    00:03:16 [INFO] train episode 992: reward = -148.00, steps = 148
    00:03:16 [INFO] train episode 993: reward = -106.00, steps = 106
    00:03:16 [INFO] train episode 994: reward = -147.00, steps = 147
    00:03:17 [INFO] train episode 995: reward = -107.00, steps = 107
    00:03:17 [INFO] train episode 996: reward = -106.00, steps = 106
    00:03:17 [INFO] train episode 997: reward = -140.00, steps = 140
    00:03:17 [INFO] train episode 998: reward = -143.00, steps = 143
    00:03:17 [INFO] train episode 999: reward = -141.00, steps = 141
    00:03:17 [INFO] train episode 1000: reward = -142.00, steps = 142
    00:03:17 [INFO] train episode 1001: reward = -143.00, steps = 143
    00:03:17 [INFO] train episode 1002: reward = -144.00, steps = 144
    00:03:17 [INFO] train episode 1003: reward = -106.00, steps = 106
    00:03:17 [INFO] train episode 1004: reward = -107.00, steps = 107
    00:03:17 [INFO] train episode 1005: reward = -106.00, steps = 106
    00:03:17 [INFO] train episode 1006: reward = -104.00, steps = 104
    00:03:17 [INFO] train episode 1007: reward = -106.00, steps = 106
    00:03:18 [INFO] train episode 1008: reward = -106.00, steps = 106
    00:03:18 [INFO] train episode 1009: reward = -106.00, steps = 106
    00:03:18 [INFO] train episode 1010: reward = -107.00, steps = 107
    00:03:18 [INFO] train episode 1011: reward = -106.00, steps = 106
    00:03:18 [INFO] train episode 1012: reward = -106.00, steps = 106
    00:03:18 [INFO] train episode 1013: reward = -91.00, steps = 91
    00:03:18 [INFO] train episode 1014: reward = -104.00, steps = 104
    00:03:18 [INFO] train episode 1015: reward = -106.00, steps = 106
    00:03:18 [INFO] train episode 1016: reward = -107.00, steps = 107
    00:03:18 [INFO] train episode 1017: reward = -85.00, steps = 85
    00:03:18 [INFO] train episode 1018: reward = -174.00, steps = 174
    00:03:18 [INFO] train episode 1019: reward = -106.00, steps = 106
    00:03:18 [INFO] train episode 1020: reward = -145.00, steps = 145
    00:03:18 [INFO] train episode 1021: reward = -106.00, steps = 106
    00:03:18 [INFO] train episode 1022: reward = -108.00, steps = 108
    00:03:19 [INFO] train episode 1023: reward = -107.00, steps = 107
    00:03:19 [INFO] train episode 1024: reward = -107.00, steps = 107
    00:03:19 [INFO] train episode 1025: reward = -90.00, steps = 90
    00:03:19 [INFO] train episode 1026: reward = -107.00, steps = 107
    00:03:19 [INFO] train episode 1027: reward = -107.00, steps = 107
    00:03:19 [INFO] train episode 1028: reward = -90.00, steps = 90
    00:03:19 [INFO] train episode 1029: reward = -88.00, steps = 88
    00:03:19 [INFO] train episode 1030: reward = -106.00, steps = 106
    00:03:19 [INFO] train episode 1031: reward = -96.00, steps = 96
    00:03:19 [INFO] train episode 1032: reward = -87.00, steps = 87
    00:03:19 [INFO] train episode 1033: reward = -158.00, steps = 158
    00:03:19 [INFO] train episode 1034: reward = -84.00, steps = 84
    00:03:19 [INFO] train episode 1035: reward = -106.00, steps = 106
    00:03:19 [INFO] train episode 1036: reward = -106.00, steps = 106
    00:03:19 [INFO] train episode 1037: reward = -106.00, steps = 106
    00:03:19 [INFO] train episode 1038: reward = -106.00, steps = 106
    00:03:20 [INFO] train episode 1039: reward = -87.00, steps = 87
    00:03:20 [INFO] train episode 1040: reward = -88.00, steps = 88
    00:03:20 [INFO] train episode 1041: reward = -86.00, steps = 86
    00:03:20 [INFO] train episode 1042: reward = -106.00, steps = 106
    00:03:20 [INFO] train episode 1043: reward = -107.00, steps = 107
    00:03:20 [INFO] train episode 1044: reward = -107.00, steps = 107
    00:03:20 [INFO] train episode 1045: reward = -87.00, steps = 87
    00:03:20 [INFO] train episode 1046: reward = -107.00, steps = 107
    00:03:20 [INFO] train episode 1047: reward = -104.00, steps = 104
    00:03:20 [INFO] train episode 1048: reward = -92.00, steps = 92
    00:03:20 [INFO] train episode 1049: reward = -106.00, steps = 106
    00:03:20 [INFO] train episode 1050: reward = -105.00, steps = 105
    00:03:20 [INFO] train episode 1051: reward = -90.00, steps = 90
    00:03:20 [INFO] train episode 1052: reward = -106.00, steps = 106
    00:03:20 [INFO] train episode 1053: reward = -88.00, steps = 88
    00:03:20 [INFO] train episode 1054: reward = -106.00, steps = 106
    00:03:21 [INFO] train episode 1055: reward = -89.00, steps = 89
    00:03:21 [INFO] train episode 1056: reward = -85.00, steps = 85
    00:03:21 [INFO] train episode 1057: reward = -105.00, steps = 105
    00:03:21 [INFO] train episode 1058: reward = -106.00, steps = 106
    00:03:21 [INFO] train episode 1059: reward = -104.00, steps = 104
    00:03:21 [INFO] train episode 1060: reward = -84.00, steps = 84
    00:03:21 [INFO] train episode 1061: reward = -104.00, steps = 104
    00:03:21 [INFO] train episode 1062: reward = -164.00, steps = 164
    00:03:21 [INFO] train episode 1063: reward = -84.00, steps = 84
    00:03:21 [INFO] train episode 1064: reward = -106.00, steps = 106
    00:03:21 [INFO] train episode 1065: reward = -85.00, steps = 85
    00:03:21 [INFO] train episode 1066: reward = -107.00, steps = 107
    00:03:21 [INFO] train episode 1067: reward = -85.00, steps = 85
    00:03:21 [INFO] train episode 1068: reward = -88.00, steps = 88
    00:03:21 [INFO] train episode 1069: reward = -83.00, steps = 83
    00:03:21 [INFO] train episode 1070: reward = -106.00, steps = 106
    00:03:21 [INFO] train episode 1071: reward = -107.00, steps = 107
    00:03:22 [INFO] train episode 1072: reward = -105.00, steps = 105
    00:03:22 [INFO] train episode 1073: reward = -107.00, steps = 107
    00:03:22 [INFO] train episode 1074: reward = -83.00, steps = 83
    00:03:22 [INFO] ==== test ====
    00:03:22 [INFO] test episode 0: reward = -108.00, steps = 108
    00:03:22 [INFO] test episode 1: reward = -83.00, steps = 83
    00:03:22 [INFO] test episode 2: reward = -88.00, steps = 88
    00:03:22 [INFO] test episode 3: reward = -172.00, steps = 172
    00:03:22 [INFO] test episode 4: reward = -108.00, steps = 108
    00:03:22 [INFO] test episode 5: reward = -108.00, steps = 108
    00:03:22 [INFO] test episode 6: reward = -107.00, steps = 107
    00:03:22 [INFO] test episode 7: reward = -142.00, steps = 142
    00:03:22 [INFO] test episode 8: reward = -100.00, steps = 100
    00:03:22 [INFO] test episode 9: reward = -109.00, steps = 109
    00:03:22 [INFO] test episode 10: reward = -171.00, steps = 171
    00:03:22 [INFO] test episode 11: reward = -107.00, steps = 107
    00:03:22 [INFO] test episode 12: reward = -169.00, steps = 169
    00:03:22 [INFO] test episode 13: reward = -200.00, steps = 200
    00:03:22 [INFO] test episode 14: reward = -107.00, steps = 107
    00:03:22 [INFO] test episode 15: reward = -101.00, steps = 101
    00:03:22 [INFO] test episode 16: reward = -107.00, steps = 107
    00:03:22 [INFO] test episode 17: reward = -86.00, steps = 86
    00:03:22 [INFO] test episode 18: reward = -108.00, steps = 108
    00:03:23 [INFO] test episode 19: reward = -91.00, steps = 91
    00:03:23 [INFO] test episode 20: reward = -85.00, steps = 85
    00:03:23 [INFO] test episode 21: reward = -85.00, steps = 85
    00:03:23 [INFO] test episode 22: reward = -118.00, steps = 118
    00:03:23 [INFO] test episode 23: reward = -101.00, steps = 101
    00:03:23 [INFO] test episode 24: reward = -108.00, steps = 108
    00:03:23 [INFO] test episode 25: reward = -107.00, steps = 107
    00:03:23 [INFO] test episode 26: reward = -107.00, steps = 107
    00:03:23 [INFO] test episode 27: reward = -107.00, steps = 107
    00:03:23 [INFO] test episode 28: reward = -84.00, steps = 84
    00:03:23 [INFO] test episode 29: reward = -98.00, steps = 98
    00:03:23 [INFO] test episode 30: reward = -107.00, steps = 107
    00:03:23 [INFO] test episode 31: reward = -108.00, steps = 108
    00:03:23 [INFO] test episode 32: reward = -91.00, steps = 91
    00:03:23 [INFO] test episode 33: reward = -108.00, steps = 108
    00:03:23 [INFO] test episode 34: reward = -85.00, steps = 85
    00:03:23 [INFO] test episode 35: reward = -107.00, steps = 107
    00:03:23 [INFO] test episode 36: reward = -90.00, steps = 90
    00:03:23 [INFO] test episode 37: reward = -107.00, steps = 107
    00:03:23 [INFO] test episode 38: reward = -108.00, steps = 108
    00:03:23 [INFO] test episode 39: reward = -108.00, steps = 108
    00:03:23 [INFO] test episode 40: reward = -87.00, steps = 87
    00:03:23 [INFO] test episode 41: reward = -92.00, steps = 92
    00:03:23 [INFO] test episode 42: reward = -89.00, steps = 89
    00:03:23 [INFO] test episode 43: reward = -111.00, steps = 111
    00:03:23 [INFO] test episode 44: reward = -107.00, steps = 107
    00:03:23 [INFO] test episode 45: reward = -94.00, steps = 94
    00:03:23 [INFO] test episode 46: reward = -94.00, steps = 94
    00:03:23 [INFO] test episode 47: reward = -115.00, steps = 115
    00:03:24 [INFO] test episode 48: reward = -91.00, steps = 91
    00:03:24 [INFO] test episode 49: reward = -92.00, steps = 92
    00:03:24 [INFO] test episode 50: reward = -85.00, steps = 85
    00:03:24 [INFO] test episode 51: reward = -108.00, steps = 108
    00:03:24 [INFO] test episode 52: reward = -169.00, steps = 169
    00:03:24 [INFO] test episode 53: reward = -109.00, steps = 109
    00:03:24 [INFO] test episode 54: reward = -87.00, steps = 87
    00:03:24 [INFO] test episode 55: reward = -101.00, steps = 101
    00:03:24 [INFO] test episode 56: reward = -101.00, steps = 101
    00:03:24 [INFO] test episode 57: reward = -108.00, steps = 108
    00:03:24 [INFO] test episode 58: reward = -107.00, steps = 107
    00:03:24 [INFO] test episode 59: reward = -108.00, steps = 108
    00:03:24 [INFO] test episode 60: reward = -107.00, steps = 107
    00:03:24 [INFO] test episode 61: reward = -89.00, steps = 89
    00:03:24 [INFO] test episode 62: reward = -87.00, steps = 87
    00:03:24 [INFO] test episode 63: reward = -84.00, steps = 84
    00:03:24 [INFO] test episode 64: reward = -114.00, steps = 114
    00:03:24 [INFO] test episode 65: reward = -90.00, steps = 90
    00:03:24 [INFO] test episode 66: reward = -107.00, steps = 107
    00:03:24 [INFO] test episode 67: reward = -85.00, steps = 85
    00:03:24 [INFO] test episode 68: reward = -84.00, steps = 84
    00:03:24 [INFO] test episode 69: reward = -107.00, steps = 107
    00:03:24 [INFO] test episode 70: reward = -108.00, steps = 108
    00:03:24 [INFO] test episode 71: reward = -88.00, steps = 88
    00:03:24 [INFO] test episode 72: reward = -107.00, steps = 107
    00:03:24 [INFO] test episode 73: reward = -87.00, steps = 87
    00:03:24 [INFO] test episode 74: reward = -105.00, steps = 105
    00:03:24 [INFO] test episode 75: reward = -108.00, steps = 108
    00:03:25 [INFO] test episode 76: reward = -90.00, steps = 90
    00:03:25 [INFO] test episode 77: reward = -88.00, steps = 88
    00:03:25 [INFO] test episode 78: reward = -108.00, steps = 108
    00:03:25 [INFO] test episode 79: reward = -86.00, steps = 86
    00:03:25 [INFO] test episode 80: reward = -90.00, steps = 90
    00:03:25 [INFO] test episode 81: reward = -91.00, steps = 91
    00:03:25 [INFO] test episode 82: reward = -107.00, steps = 107
    00:03:25 [INFO] test episode 83: reward = -110.00, steps = 110
    00:03:25 [INFO] test episode 84: reward = -107.00, steps = 107
    00:03:25 [INFO] test episode 85: reward = -88.00, steps = 88
    00:03:25 [INFO] test episode 86: reward = -88.00, steps = 88
    00:03:25 [INFO] test episode 87: reward = -84.00, steps = 84
    00:03:25 [INFO] test episode 88: reward = -89.00, steps = 89
    00:03:25 [INFO] test episode 89: reward = -108.00, steps = 108
    00:03:25 [INFO] test episode 90: reward = -108.00, steps = 108
    00:03:25 [INFO] test episode 91: reward = -108.00, steps = 108
    00:03:25 [INFO] test episode 92: reward = -88.00, steps = 88
    00:03:25 [INFO] test episode 93: reward = -107.00, steps = 107
    00:03:25 [INFO] test episode 94: reward = -91.00, steps = 91
    00:03:25 [INFO] test episode 95: reward = -93.00, steps = 93
    00:03:25 [INFO] test episode 96: reward = -108.00, steps = 108
    00:03:25 [INFO] test episode 97: reward = -108.00, steps = 108
    00:03:25 [INFO] test episode 98: reward = -84.00, steps = 84
    00:03:25 [INFO] test episode 99: reward = -109.00, steps = 109
    00:03:25 [INFO] average episode reward = -103.41 ± 19.96

#### Visualize¶

In [7]:

    poses = np.linspace(env.min_position, env.max_position, 128)
    vels = np.linspace(-env.max_speed, env.max_speed, 128)
    positions, velocities = np.meshgrid(poses, vels)

Plot action value

In [8]:

    @np.vectorize
    def get_q(position, velocity, action):
        return agent.get_q((position, velocity), action)

    q_values = np.empty((len(poses), len(vels), 3))
    for action in range(3):
        q_values[:, :, action] = get_q(positions, velocities, action)

    fig, axes = plt.subplots(1, 3, figsize=(15, 4))
    for action, ax in enumerate(axes):
        c = ax.pcolormesh(positions, velocities, q_values[:, :, action],
                shading='auto')
        ax.set_xlabel('position')
        ax.set_ylabel('velocity')
        fig.colorbar(c, ax=ax)
        ax.set_title('action = {}'.format(action))

Plot state value

In [9]:

    v_values = q_values.max(axis=-1)

    fig, ax = plt.subplots(1, 1)
    c = ax.pcolormesh(positions, velocities, v_values, shading='auto')
    ax.set_xlabel('position')
    ax.set_ylabel('velocity')
    fig.colorbar(c, ax=ax);

Plot policy

In [10]:

    @np.vectorize
    def decide(position, velocity):
        return agent.step((position, velocity), _, _)

    q_values = np.empty((len(poses), len(vels), 3))
    action_values = decide(positions, velocities)

    fig, ax = plt.subplots()
    c = ax.pcolormesh(positions, velocities, action_values, shading='auto')
    ax.set_xlabel('position')
    ax.set_ylabel('velocity')
    fig.colorbar(c, ax=ax, boundaries=[-.5, .5, 1.5, 2.5], ticks=[0, 1, 2]);

In [11]:

    env.close()