# Use Soft Actor-Critic with Auto $\alpha$ Tuning to Play LunarLanderContinuous-v2¶

PyTorch version

In [1]:

    %matplotlib inline

    import sys
    import logging
    import itertools
    import copy

    import numpy as np
    np.random.seed(0)
    import pandas as pd
    import gym
    import matplotlib.pyplot as plt
    import torch
    torch.manual_seed(0)
    import torch.nn as nn
    import torch.optim as optim
    import torch.distributions as distributions

    logging.basicConfig(level=logging.INFO,
            format='%(asctime)s [%(levelname)s] %(message)s',
            stream=sys.stdout, datefmt='%H:%M:%S')

In [2]:

    env = gym.make('LunarLanderContinuous-v2')
    for key in vars(env):
        logging.info('%s: %s', key, vars(env)[key])
    for key in vars(env.spec):
        logging.info('%s: %s', key, vars(env.spec)[key])

    08:18:11 [INFO] env: <LunarLanderContinuous<LunarLanderContinuous-v2>>
    08:18:11 [INFO] action_space: Box(-1.0, 1.0, (2,), float32)
    08:18:11 [INFO] observation_space: Box(-inf, inf, (8,), float32)
    08:18:11 [INFO] reward_range: (-inf, inf)
    08:18:11 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 50}
    08:18:11 [INFO] _max_episode_steps: 1000
    08:18:11 [INFO] _elapsed_steps: None
    08:18:11 [INFO] id: LunarLanderContinuous-v2
    08:18:11 [INFO] entry_point: gym.envs.box2d:LunarLanderContinuous
    08:18:11 [INFO] reward_threshold: 200
    08:18:11 [INFO] nondeterministic: False
    08:18:11 [INFO] max_episode_steps: 1000
    08:18:11 [INFO] _kwargs: {}
    08:18:11 [INFO] _env_name: LunarLanderContinuous

In [3]:

    class DQNReplayer:
        def __init__(self, capacity):
            self.memory = pd.DataFrame(index=range(capacity),
                    columns=['state', 'action', 'reward', 'next_state', 'terminated'])
            self.i = 0
            self.count = 0
            self.capacity = capacity

        def store(self, *args):
            self.memory.loc[self.i] = np.asarray(args, dtype=object)
            self.i = (self.i + 1) % self.capacity
            self.count = min(self.count + 1, self.capacity)

        def sample(self, size):
            indices = np.random.choice(self.count, size=size)
            return (np.stack(self.memory.loc[indices, field]) for field in
                    self.memory.columns)

In [4]:

    class SACAgent:
        def __init__(self, env):
            state_dim = env.observation_space.shape[0]
            self.action_dim = env.action_space.shape[0]
            self.action_low = env.action_space.low
            self.action_high = env.action_space.high
            self.gamma = 0.99

            self.replayer = DQNReplayer(100000)

            # create alpha
            self.target_entropy = -self.action_dim
            self.ln_alpha_tensor = torch.zeros(1, requires_grad=True)
            self.alpha_optimizer = optim.Adam([self.ln_alpha_tensor,], lr=0.0003)

            # create actor
            self.actor_net = self.build_net(input_size=state_dim,
                    hidden_sizes=[256, 256], output_size=self.action_dim*2,
                    output_activator=nn.Tanh())
            self.actor_optimizier = optim.Adam(self.actor_net.parameters(), lr=0.0003)

            # create V critic
            self.v_evaluate_net = self.build_net(input_size=state_dim,
                    hidden_sizes=[256, 256])
            self.v_target_net = copy.deepcopy(self.v_evaluate_net)
            self.v_loss = nn.MSELoss()
            self.v_optimizer = optim.Adam(self.v_evaluate_net.parameters(), lr=0.0003)

            # create Q critic
            self.q0_net = self.build_net(input_size=state_dim+self.action_dim,
                    hidden_sizes=[256, 256])
            self.q1_net = self.build_net(input_size=state_dim+self.action_dim,
                    hidden_sizes=[256, 256])
            self.q0_loss = nn.MSELoss()
            self.q1_loss = nn.MSELoss()
            self.q0_optimizer = optim.Adam(self.q0_net.parameters(), lr=0.0003)
            self.q1_optimizer = optim.Adam(self.q1_net.parameters(), lr=0.0003)

        def build_net(self, input_size, hidden_sizes, output_size=1,
                output_activator=None):
            layers = []
            for input_size, output_size in zip(
                    [input_size,] + hidden_sizes, hidden_sizes + [output_size,]):
                layers.append(nn.Linear(input_size, output_size))
                layers.append(nn.ReLU())
            layers = layers[:-1]
            if output_activator:
                layers.append(output_activator)
            net = nn.Sequential(*layers)
            return net

        def get_action_ln_prob_tensors(self, state_tensor):
            mean_ln_std_tensor = self.actor_net(state_tensor)
            mean_tensor, ln_std_tensor = torch.split(mean_ln_std_tensor,
                    self.action_dim, dim=-1)
            if self.mode == 'train':
                std_tensor = torch.exp(ln_std_tensor)
                normal_dist = distributions.Normal(mean_tensor, std_tensor)
                rsample_tensor = normal_dist.rsample()
                action_tensor = torch.tanh(rsample_tensor)
                ln_prob_tensor = normal_dist.log_prob(rsample_tensor) - \
                        torch.log1p(1e-6 - action_tensor.pow(2))
                ln_prob_tensor = ln_prob_tensor.sum(-1, keepdim=True)
            else:
                action_tensor = torch.tanh(mean_tensor)
                ln_prob_tensor = torch.ones_like(action_tensor)
            return action_tensor, ln_prob_tensor

        def reset(self, mode):
            self.mode = mode
            if self.mode == 'train':
                self.trajectory = []

        def step(self, observation, reward, terminated):
            if self.mode == 'train' and self.replayer.count < 5000:
                action = np.random.uniform(self.action_low, self.action_high)
            else:
                state_tensor = torch.as_tensor(observation, dtype=torch.float
                        ).unsqueeze(0)
                action_tensor, _ = self.get_action_ln_prob_tensors(state_tensor)
                action = action_tensor[0].detach().numpy()
            if self.mode == 'train':
                self.trajectory += [observation, reward, terminated, action]
                if len(self.trajectory) >= 8:
                    state, _, _, act, next_state, reward, terminated, _ = \
                            self.trajectory[-8:]
                    self.replayer.store(state, act, reward, next_state, terminated)
                if self.replayer.count >= 128:
                    self.learn()
            return action

        def close(self):
            pass

        def update_net(self, target_net, evaluate_net, learning_rate=0.005):
            for target_param, evaluate_param in zip(
                    target_net.parameters(), evaluate_net.parameters()):
                target_param.data.copy_(learning_rate * evaluate_param.data
                        + (1 - learning_rate) * target_param.data)

        def learn(self):
            states, actions, rewards, next_states, terminateds = \
                    self.replayer.sample(128)
            state_tensor = torch.as_tensor(states, dtype=torch.float)
            action_tensor = torch.as_tensor(actions, dtype=torch.float)
            reward_tensor = torch.as_tensor(rewards, dtype=torch.float)
            next_state_tensor = torch.as_tensor(next_states, dtype=torch.float)
            terminated_tensor = torch.as_tensor(terminateds, dtype=torch.float)

            # update alpha
            act_tensor, ln_prob_tensor = self.get_action_ln_prob_tensors(state_tensor)
            alpha_loss_tensor = (-self.ln_alpha_tensor * (ln_prob_tensor +
                    self.target_entropy).detach()).mean()

            self.alpha_optimizer.zero_grad()
            alpha_loss_tensor.backward()
            self.alpha_optimizer.step()

            # update Q critic
            states_action_tensor = torch.cat((state_tensor, action_tensor), dim=-1)
            q0_tensor = self.q0_net(states_action_tensor)
            q1_tensor = self.q1_net(states_action_tensor)
            next_v_tensor = self.v_target_net(next_state_tensor)
            q_target = reward_tensor.unsqueeze(1) + \
                    self.gamma * next_v_tensor * (1. - terminated_tensor.unsqueeze(1))
            q0_loss_tensor = self.q0_loss(q0_tensor, q_target.detach())
            q1_loss_tensor = self.q1_loss(q1_tensor, q_target.detach())

            self.q0_optimizer.zero_grad()
            q0_loss_tensor.backward()
            self.q0_optimizer.step()

            self.q1_optimizer.zero_grad()
            q1_loss_tensor.backward()
            self.q1_optimizer.step()

            # update V critic
            state_act_tensor = torch.cat((state_tensor, act_tensor), dim=-1)
            v_pred_tensor = self.v_evaluate_net(state_tensor)
            q0_pred_tensor = self.q0_net(state_act_tensor)
            q1_pred_tensor = self.q1_net(state_act_tensor)
            q_pred_tensor = torch.min(q0_pred_tensor, q1_pred_tensor)
            alpha_tensor = self.ln_alpha_tensor.exp()
            v_target_tensor = q_pred_tensor - alpha_tensor * ln_prob_tensor
            v_loss_tensor = self.v_loss(v_pred_tensor, v_target_tensor.detach())

            self.v_optimizer.zero_grad()
            v_loss_tensor.backward()
            self.v_optimizer.step()

            self.update_net(self.v_target_net, self.v_evaluate_net)

            # update actor
            actor_loss_tensor = (alpha_tensor * ln_prob_tensor
                    - q0_pred_tensor).mean()

            self.actor_optimizier.zero_grad()
            actor_loss_tensor.backward()
            self.actor_optimizier.step()

    agent = SACAgent(env)

In [5]:

    def play_episode(env, agent, seed=None, mode=None, render=False):
        observation, _ = env.reset(seed=seed)
        reward, terminated, truncated = 0., False, False
        agent.reset(mode=mode)
        episode_reward, elapsed_steps = 0., 0
        while True:
            action = agent.step(observation, reward, terminated)
            if render:
                env.render()
            if terminated or truncated:
                break
            observation, reward, terminated, truncated, _ = env.step(action)
            episode_reward += reward
            elapsed_steps += 1
        agent.close()
        return episode_reward, elapsed_steps

    logging.info('==== train ====')
    episode_rewards = []
    for episode in itertools.count():
        episode_reward, elapsed_steps = play_episode(env, agent, seed=episode,
                mode='train')
        episode_rewards.append(episode_reward)
        logging.info('train episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
        if np.mean(episode_rewards[-10:]) > 250:
            break
    plt.plot(episode_rewards)

    logging.info('==== test ====')
    episode_rewards = []
    for episode in range(100):
        episode_reward, elapsed_steps = play_episode(env, agent)
        episode_rewards.append(episode_reward)
        logging.info('test episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
    logging.info('average episode reward = %.2f ± %.2f',
            np.mean(episode_rewards), np.std(episode_rewards))

    08:18:11 [INFO] ==== train ====
    08:18:12 [INFO] train episode 0: reward = -112.08, steps = 140
    08:18:18 [INFO] train episode 1: reward = -281.50, steps = 159
    08:18:23 [INFO] train episode 2: reward = -234.58, steps = 137
    08:18:30 [INFO] train episode 3: reward = -158.37, steps = 187
    08:18:37 [INFO] train episode 4: reward = -298.07, steps = 170
    08:18:40 [INFO] train episode 5: reward = -335.86, steps = 75
    08:18:44 [INFO] train episode 6: reward = -426.55, steps = 108
    08:18:48 [INFO] train episode 7: reward = -149.63, steps = 111
    08:18:51 [INFO] train episode 8: reward = -69.06, steps = 73
    08:18:55 [INFO] train episode 9: reward = -271.07, steps = 96
    08:18:59 [INFO] train episode 10: reward = -408.78, steps = 105
    08:19:04 [INFO] train episode 11: reward = -102.08, steps = 137
    08:19:10 [INFO] train episode 12: reward = -390.09, steps = 145
    08:19:15 [INFO] train episode 13: reward = -408.79, steps = 107
    08:19:17 [INFO] train episode 14: reward = -92.82, steps = 69
    08:19:21 [INFO] train episode 15: reward = -352.78, steps = 87
    08:19:24 [INFO] train episode 16: reward = -234.04, steps = 89
    08:19:29 [INFO] train episode 17: reward = -250.68, steps = 127
    08:19:34 [INFO] train episode 18: reward = -250.59, steps = 119
    08:19:40 [INFO] train episode 19: reward = -320.67, steps = 152
    08:19:43 [INFO] train episode 20: reward = -373.46, steps = 81
    08:19:46 [INFO] train episode 21: reward = -88.18, steps = 83
    08:19:49 [INFO] train episode 22: reward = -36.75, steps = 80
    08:19:53 [INFO] train episode 23: reward = -139.52, steps = 97
    08:19:57 [INFO] train episode 24: reward = -352.28, steps = 106
    08:20:02 [INFO] train episode 25: reward = -111.02, steps = 121
    08:20:06 [INFO] train episode 26: reward = -173.87, steps = 104
    08:20:10 [INFO] train episode 27: reward = -357.54, steps = 101
    08:20:15 [INFO] train episode 28: reward = -187.84, steps = 125
    08:20:23 [INFO] train episode 29: reward = -259.05, steps = 213
    08:20:27 [INFO] train episode 30: reward = -189.09, steps = 113
    08:20:31 [INFO] train episode 31: reward = -233.59, steps = 106
    08:20:35 [INFO] train episode 32: reward = -447.59, steps = 104
    08:20:41 [INFO] train episode 33: reward = -298.22, steps = 157
    08:20:46 [INFO] train episode 34: reward = -312.16, steps = 106
    08:20:50 [INFO] train episode 35: reward = -99.37, steps = 114
    08:20:54 [INFO] train episode 36: reward = -402.26, steps = 115
    08:20:58 [INFO] train episode 37: reward = -280.93, steps = 104
    08:21:02 [INFO] train episode 38: reward = -77.80, steps = 104
    08:21:05 [INFO] train episode 39: reward = -194.40, steps = 82
    08:21:10 [INFO] train episode 40: reward = -260.22, steps = 120
    08:21:15 [INFO] train episode 41: reward = -267.42, steps = 117
    08:21:20 [INFO] train episode 42: reward = -116.98, steps = 134
    08:21:30 [INFO] train episode 43: reward = -338.80, steps = 225
    08:21:40 [INFO] train episode 44: reward = -270.31, steps = 233
    08:21:47 [INFO] train episode 45: reward = -133.80, steps = 155
    08:21:54 [INFO] train episode 46: reward = -63.22, steps = 150
    08:22:01 [INFO] train episode 47: reward = -290.41, steps = 176
    08:22:12 [INFO] train episode 48: reward = -46.56, steps = 252
    08:22:30 [INFO] train episode 49: reward = -115.06, steps = 431
    08:22:46 [INFO] train episode 50: reward = -102.78, steps = 392
    08:23:29 [INFO] train episode 51: reward = -150.02, steps = 1000
    08:24:12 [INFO] train episode 52: reward = -58.18, steps = 1000
    08:24:55 [INFO] train episode 53: reward = -137.97, steps = 1000
    08:25:16 [INFO] train episode 54: reward = 189.33, steps = 514
    08:25:58 [INFO] train episode 55: reward = -79.11, steps = 1000
    08:26:41 [INFO] train episode 56: reward = -62.32, steps = 1000
    08:27:09 [INFO] train episode 57: reward = -146.16, steps = 657
    08:27:18 [INFO] train episode 58: reward = -310.16, steps = 233
    08:27:26 [INFO] train episode 59: reward = -141.95, steps = 207
    08:27:33 [INFO] train episode 60: reward = -24.10, steps = 171
    08:27:39 [INFO] train episode 61: reward = 10.28, steps = 156
    08:27:47 [INFO] train episode 62: reward = -72.96, steps = 177
    08:28:09 [INFO] train episode 63: reward = 189.82, steps = 562
    08:28:19 [INFO] train episode 64: reward = -45.08, steps = 249
    08:28:36 [INFO] train episode 65: reward = 202.26, steps = 395
    08:29:18 [INFO] train episode 66: reward = -1.59, steps = 1000
    08:29:49 [INFO] train episode 67: reward = 85.66, steps = 765
    08:30:31 [INFO] train episode 68: reward = 49.17, steps = 1000
    08:31:14 [INFO] train episode 69: reward = -9.73, steps = 1000
    08:31:59 [INFO] train episode 70: reward = -43.56, steps = 1000
    08:32:32 [INFO] train episode 71: reward = 92.58, steps = 775
    08:33:17 [INFO] train episode 72: reward = 104.25, steps = 1000
    08:34:00 [INFO] train episode 73: reward = -2.90, steps = 1000
    08:34:44 [INFO] train episode 74: reward = -16.13, steps = 1000
    08:35:27 [INFO] train episode 75: reward = -81.60, steps = 1000
    08:36:12 [INFO] train episode 76: reward = -51.90, steps = 1000
    08:36:56 [INFO] train episode 77: reward = -34.58, steps = 1000
    08:37:39 [INFO] train episode 78: reward = -8.94, steps = 1000
    08:38:21 [INFO] train episode 79: reward = -138.96, steps = 992
    08:39:03 [INFO] train episode 80: reward = -37.00, steps = 1000
    08:39:45 [INFO] train episode 81: reward = -26.09, steps = 1000
    08:40:29 [INFO] train episode 82: reward = -38.16, steps = 1000
    08:41:11 [INFO] train episode 83: reward = -28.83, steps = 1000
    08:41:54 [INFO] train episode 84: reward = -32.24, steps = 1000
    08:42:31 [INFO] train episode 85: reward = 1.92, steps = 1000
    08:43:09 [INFO] train episode 86: reward = -41.07, steps = 1000
    08:43:49 [INFO] train episode 87: reward = -77.63, steps = 1000
    08:44:24 [INFO] train episode 88: reward = -144.59, steps = 926
    08:45:02 [INFO] train episode 89: reward = -51.94, steps = 1000
    08:45:40 [INFO] train episode 90: reward = -64.39, steps = 1000
    08:46:19 [INFO] train episode 91: reward = -30.99, steps = 1000
    08:46:56 [INFO] train episode 92: reward = -35.70, steps = 1000
    08:47:34 [INFO] train episode 93: reward = 40.63, steps = 1000
    08:48:11 [INFO] train episode 94: reward = -48.49, steps = 1000
    08:48:51 [INFO] train episode 95: reward = 14.42, steps = 1000
    08:49:30 [INFO] train episode 96: reward = -56.95, steps = 1000
    08:50:07 [INFO] train episode 97: reward = -1.87, steps = 1000
    08:50:44 [INFO] train episode 98: reward = -29.84, steps = 1000
    08:51:23 [INFO] train episode 99: reward = -29.44, steps = 1000
    08:52:00 [INFO] train episode 100: reward = -3.35, steps = 1000
    08:52:40 [INFO] train episode 101: reward = 4.47, steps = 1000
    08:53:17 [INFO] train episode 102: reward = -28.23, steps = 1000
    08:53:59 [INFO] train episode 103: reward = -36.16, steps = 1000
    08:54:18 [INFO] train episode 104: reward = -80.85, steps = 499
    08:54:55 [INFO] train episode 105: reward = -1.22, steps = 1000
    08:55:24 [INFO] train episode 106: reward = -147.22, steps = 761
    08:55:45 [INFO] train episode 107: reward = -113.34, steps = 574
    08:56:21 [INFO] train episode 108: reward = -236.57, steps = 919
    08:56:53 [INFO] train episode 109: reward = -208.71, steps = 845
    08:57:31 [INFO] train episode 110: reward = -76.35, steps = 1000
    08:58:10 [INFO] train episode 111: reward = -48.00, steps = 1000
    08:58:51 [INFO] train episode 112: reward = -35.55, steps = 1000
    08:59:32 [INFO] train episode 113: reward = -36.37, steps = 1000
    09:00:12 [INFO] train episode 114: reward = -24.10, steps = 1000
    09:00:53 [INFO] train episode 115: reward = -8.93, steps = 1000
    09:01:34 [INFO] train episode 116: reward = -32.26, steps = 1000
    09:02:16 [INFO] train episode 117: reward = -20.86, steps = 1000
    09:02:55 [INFO] train episode 118: reward = -23.36, steps = 1000
    09:03:35 [INFO] train episode 119: reward = -19.84, steps = 1000
    09:04:16 [INFO] train episode 120: reward = 4.26, steps = 1000
    09:04:57 [INFO] train episode 121: reward = 13.64, steps = 1000
    09:05:37 [INFO] train episode 122: reward = -49.42, steps = 1000
    09:06:20 [INFO] train episode 123: reward = 12.37, steps = 1000
    09:07:04 [INFO] train episode 124: reward = -3.07, steps = 1000
    09:07:39 [INFO] train episode 125: reward = 192.29, steps = 816
    09:08:20 [INFO] train episode 126: reward = -24.69, steps = 1000
    09:09:05 [INFO] train episode 127: reward = -73.62, steps = 1000
    09:09:46 [INFO] train episode 128: reward = 69.85, steps = 1000
    09:10:27 [INFO] train episode 129: reward = -47.97, steps = 1000
    09:11:09 [INFO] train episode 130: reward = -33.53, steps = 1000
    09:11:53 [INFO] train episode 131: reward = -11.22, steps = 1000
    09:12:24 [INFO] train episode 132: reward = -98.74, steps = 753
    09:13:07 [INFO] train episode 133: reward = -31.19, steps = 1000
    09:13:51 [INFO] train episode 134: reward = -25.83, steps = 1000
    09:14:34 [INFO] train episode 135: reward = -33.07, steps = 1000
    09:15:15 [INFO] train episode 136: reward = -47.70, steps = 1000
    09:15:59 [INFO] train episode 137: reward = 73.33, steps = 1000
    09:16:44 [INFO] train episode 138: reward = -27.60, steps = 1000
    09:17:29 [INFO] train episode 139: reward = -2.45, steps = 1000
    09:18:11 [INFO] train episode 140: reward = 30.67, steps = 1000
    09:18:53 [INFO] train episode 141: reward = 3.14, steps = 1000
    09:19:36 [INFO] train episode 142: reward = -11.03, steps = 1000
    09:20:21 [INFO] train episode 143: reward = -8.69, steps = 1000
    09:21:05 [INFO] train episode 144: reward = -23.17, steps = 1000
    09:21:53 [INFO] train episode 145: reward = -19.40, steps = 1000
    09:22:37 [INFO] train episode 146: reward = 63.90, steps = 1000
    09:23:20 [INFO] train episode 147: reward = -27.68, steps = 1000
    09:24:02 [INFO] train episode 148: reward = -14.27, steps = 1000
    09:24:45 [INFO] train episode 149: reward = -24.26, steps = 1000
    09:25:28 [INFO] train episode 150: reward = -26.83, steps = 1000
    09:26:13 [INFO] train episode 151: reward = -27.14, steps = 1000
    09:26:56 [INFO] train episode 152: reward = -15.51, steps = 1000
    09:27:39 [INFO] train episode 153: reward = -34.58, steps = 1000
    09:28:21 [INFO] train episode 154: reward = 37.20, steps = 1000
    09:29:03 [INFO] train episode 155: reward = 34.22, steps = 1000
    09:29:48 [INFO] train episode 156: reward = -5.36, steps = 1000
    09:30:31 [INFO] train episode 157: reward = -50.48, steps = 1000
    09:31:13 [INFO] train episode 158: reward = 77.25, steps = 1000
    09:31:28 [INFO] train episode 159: reward = -64.55, steps = 361
    09:32:12 [INFO] train episode 160: reward = -50.94, steps = 1000
    09:32:54 [INFO] train episode 161: reward = -24.62, steps = 1000
    09:33:38 [INFO] train episode 162: reward = -39.87, steps = 1000
    09:34:22 [INFO] train episode 163: reward = 113.43, steps = 1000
    09:35:05 [INFO] train episode 164: reward = -18.25, steps = 1000
    09:35:52 [INFO] train episode 165: reward = -32.68, steps = 1000
    09:36:19 [INFO] train episode 166: reward = 251.94, steps = 639
    09:37:03 [INFO] train episode 167: reward = 132.67, steps = 996
    09:37:48 [INFO] train episode 168: reward = 26.10, steps = 1000
    09:38:31 [INFO] train episode 169: reward = -23.84, steps = 1000
    09:39:16 [INFO] train episode 170: reward = -34.61, steps = 1000
    09:39:59 [INFO] train episode 171: reward = 16.13, steps = 1000
    09:40:10 [INFO] train episode 172: reward = -28.65, steps = 267
    09:40:55 [INFO] train episode 173: reward = -6.24, steps = 1000
    09:41:41 [INFO] train episode 174: reward = -37.24, steps = 1000
    09:42:17 [INFO] train episode 175: reward = 153.42, steps = 838
    09:42:48 [INFO] train episode 176: reward = -106.54, steps = 704
    09:43:32 [INFO] train episode 177: reward = -15.26, steps = 1000
    09:44:15 [INFO] train episode 178: reward = -33.40, steps = 1000
    09:45:01 [INFO] train episode 179: reward = -33.85, steps = 1000
    09:45:47 [INFO] train episode 180: reward = -15.62, steps = 1000
    09:46:34 [INFO] train episode 181: reward = -26.81, steps = 1000
    09:47:19 [INFO] train episode 182: reward = 0.83, steps = 1000
    09:48:04 [INFO] train episode 183: reward = 128.82, steps = 981
    09:48:49 [INFO] train episode 184: reward = -46.66, steps = 1000
    09:49:35 [INFO] train episode 185: reward = -45.17, steps = 1000
    09:50:22 [INFO] train episode 186: reward = -39.43, steps = 1000
    09:51:06 [INFO] train episode 187: reward = 79.79, steps = 1000
    09:51:53 [INFO] train episode 188: reward = -23.49, steps = 1000
    09:52:38 [INFO] train episode 189: reward = -37.61, steps = 1000
    09:53:22 [INFO] train episode 190: reward = -12.40, steps = 1000
    09:54:06 [INFO] train episode 191: reward = -10.87, steps = 1000
    09:54:48 [INFO] train episode 192: reward = -18.93, steps = 1000
    09:55:35 [INFO] train episode 193: reward = -30.56, steps = 1000
    09:56:21 [INFO] train episode 194: reward = -40.22, steps = 1000
    09:57:09 [INFO] train episode 195: reward = -10.30, steps = 1000
    09:57:55 [INFO] train episode 196: reward = -1.69, steps = 1000
    09:58:42 [INFO] train episode 197: reward = 10.05, steps = 1000
    09:59:23 [INFO] train episode 198: reward = 167.86, steps = 914
    10:00:10 [INFO] train episode 199: reward = 15.12, steps = 1000
    10:00:54 [INFO] train episode 200: reward = -11.66, steps = 1000
    10:01:40 [INFO] train episode 201: reward = -19.04, steps = 1000
    10:02:06 [INFO] train episode 202: reward = 245.18, steps = 589
    10:02:54 [INFO] train episode 203: reward = 32.02, steps = 1000
    10:03:42 [INFO] train episode 204: reward = 56.35, steps = 1000
    10:04:20 [INFO] train episode 205: reward = 186.69, steps = 836
    10:05:08 [INFO] train episode 206: reward = -14.67, steps = 1000
    10:05:41 [INFO] train episode 207: reward = 203.12, steps = 736
    10:05:57 [INFO] train episode 208: reward = 262.30, steps = 366
    10:06:29 [INFO] train episode 209: reward = 265.60, steps = 726
    10:07:11 [INFO] train episode 210: reward = 180.46, steps = 914
    10:07:36 [INFO] train episode 211: reward = 188.73, steps = 580
    10:08:05 [INFO] train episode 212: reward = 170.45, steps = 661
    10:08:24 [INFO] train episode 213: reward = 278.07, steps = 428
    10:08:48 [INFO] train episode 214: reward = 276.87, steps = 559
    10:09:33 [INFO] train episode 215: reward = -13.98, steps = 1000
    10:09:57 [INFO] train episode 216: reward = 215.50, steps = 537
    10:10:43 [INFO] train episode 217: reward = 1.84, steps = 1000
    10:11:11 [INFO] train episode 218: reward = 217.29, steps = 636
    10:11:34 [INFO] train episode 219: reward = 255.23, steps = 514
    10:12:01 [INFO] train episode 220: reward = 186.56, steps = 597
    10:12:48 [INFO] train episode 221: reward = 81.03, steps = 1000
    10:13:25 [INFO] train episode 222: reward = 153.45, steps = 857
    10:13:31 [INFO] train episode 223: reward = 18.37, steps = 123
    10:14:03 [INFO] train episode 224: reward = 275.56, steps = 744
    10:14:34 [INFO] train episode 225: reward = 240.97, steps = 700
    10:15:01 [INFO] train episode 226: reward = 274.98, steps = 623
    10:15:07 [INFO] train episode 227: reward = 20.80, steps = 144
    10:15:14 [INFO] train episode 228: reward = -1.63, steps = 166
    10:15:31 [INFO] train episode 229: reward = 233.82, steps = 387
    10:15:47 [INFO] train episode 230: reward = 259.04, steps = 363
    10:16:33 [INFO] train episode 231: reward = 107.03, steps = 1000
    10:16:54 [INFO] train episode 232: reward = 224.22, steps = 488
    10:17:20 [INFO] train episode 233: reward = 205.25, steps = 563
    10:18:07 [INFO] train episode 234: reward = 4.50, steps = 1000
    10:18:55 [INFO] train episode 235: reward = 119.17, steps = 993
    10:19:41 [INFO] train episode 236: reward = -31.98, steps = 1000
    10:20:28 [INFO] train episode 237: reward = -49.84, steps = 1000
    10:21:13 [INFO] train episode 238: reward = -43.70, steps = 1000
    10:21:54 [INFO] train episode 239: reward = -112.50, steps = 860
    10:22:38 [INFO] train episode 240: reward = 158.07, steps = 1000
    10:23:22 [INFO] train episode 241: reward = -66.71, steps = 1000
    10:23:44 [INFO] train episode 242: reward = 222.56, steps = 490
    10:24:32 [INFO] train episode 243: reward = -27.62, steps = 1000
    10:24:44 [INFO] train episode 244: reward = 273.56, steps = 298
    10:25:32 [INFO] train episode 245: reward = -53.45, steps = 1000
    10:26:17 [INFO] train episode 246: reward = -3.71, steps = 1000
    10:26:42 [INFO] train episode 247: reward = 204.70, steps = 562
    10:27:12 [INFO] train episode 248: reward = 210.68, steps = 668
    10:27:57 [INFO] train episode 249: reward = -24.15, steps = 1000
    10:28:13 [INFO] train episode 250: reward = 236.60, steps = 371
    10:28:26 [INFO] train episode 251: reward = 275.84, steps = 307
    10:29:00 [INFO] train episode 252: reward = 214.70, steps = 764
    10:29:29 [INFO] train episode 253: reward = 173.57, steps = 651
    10:29:48 [INFO] train episode 254: reward = 265.24, steps = 453
    10:30:32 [INFO] train episode 255: reward = 132.43, steps = 1000
    10:31:19 [INFO] train episode 256: reward = -21.95, steps = 1000
    10:32:04 [INFO] train episode 257: reward = -26.56, steps = 1000
    10:32:30 [INFO] train episode 258: reward = 188.48, steps = 577
    10:33:00 [INFO] train episode 259: reward = -106.38, steps = 660
    10:33:46 [INFO] train episode 260: reward = -34.32, steps = 1000
    10:34:18 [INFO] train episode 261: reward = 203.41, steps = 725
    10:34:36 [INFO] train episode 262: reward = 255.83, steps = 408
    10:35:22 [INFO] train episode 263: reward = -9.11, steps = 1000
    10:36:10 [INFO] train episode 264: reward = 39.38, steps = 1000
    10:36:46 [INFO] train episode 265: reward = 211.19, steps = 808
    10:37:04 [INFO] train episode 266: reward = 261.64, steps = 413
    10:37:48 [INFO] train episode 267: reward = 136.15, steps = 1000
    10:38:33 [INFO] train episode 268: reward = -15.21, steps = 1000
    10:39:05 [INFO] train episode 269: reward = 232.94, steps = 726
    10:39:22 [INFO] train episode 270: reward = 236.03, steps = 375
    10:39:50 [INFO] train episode 271: reward = 237.10, steps = 636
    10:40:06 [INFO] train episode 272: reward = 248.89, steps = 359
    10:40:22 [INFO] train episode 273: reward = 266.12, steps = 378
    10:41:07 [INFO] train episode 274: reward = 1.84, steps = 1000
    10:41:45 [INFO] train episode 275: reward = 157.79, steps = 825
    10:42:17 [INFO] train episode 276: reward = 189.86, steps = 719
    10:42:24 [INFO] train episode 277: reward = -35.46, steps = 155
    10:42:33 [INFO] train episode 278: reward = 3.49, steps = 211
    10:43:01 [INFO] train episode 279: reward = 238.85, steps = 641
    10:43:33 [INFO] train episode 280: reward = 180.11, steps = 713
    10:43:54 [INFO] train episode 281: reward = 232.95, steps = 470
    10:44:11 [INFO] train episode 282: reward = 257.30, steps = 383
    10:44:35 [INFO] train episode 283: reward = 192.55, steps = 533
    10:45:00 [INFO] train episode 284: reward = 256.59, steps = 578
    10:45:30 [INFO] train episode 285: reward = 253.76, steps = 650
    10:45:57 [INFO] train episode 286: reward = 215.79, steps = 626
    10:46:42 [INFO] train episode 287: reward = 127.63, steps = 1000
    10:47:10 [INFO] train episode 288: reward = 237.74, steps = 639
    10:47:24 [INFO] train episode 289: reward = 257.29, steps = 314
    10:47:44 [INFO] train episode 290: reward = 219.75, steps = 454
    10:48:27 [INFO] train episode 291: reward = -34.80, steps = 1000
    10:48:45 [INFO] train episode 292: reward = 225.38, steps = 415
    10:49:30 [INFO] train episode 293: reward = 132.93, steps = 1000
    10:49:42 [INFO] train episode 294: reward = 239.37, steps = 267
    10:50:08 [INFO] train episode 295: reward = 228.38, steps = 598
    10:50:26 [INFO] train episode 296: reward = 288.36, steps = 422
    10:51:11 [INFO] train episode 297: reward = 29.07, steps = 1000
    10:51:27 [INFO] train episode 298: reward = 233.57, steps = 359
    10:51:48 [INFO] train episode 299: reward = 205.36, steps = 478
    10:52:06 [INFO] train episode 300: reward = 233.38, steps = 414
    10:52:24 [INFO] train episode 301: reward = 214.90, steps = 402
    10:52:42 [INFO] train episode 302: reward = 216.78, steps = 420
    10:52:54 [INFO] train episode 303: reward = 272.85, steps = 274
    10:53:27 [INFO] train episode 304: reward = 273.54, steps = 724
    10:54:00 [INFO] train episode 305: reward = 201.24, steps = 734
    10:54:44 [INFO] train episode 306: reward = 51.44, steps = 1000
    10:54:56 [INFO] train episode 307: reward = 243.04, steps = 278
    10:55:11 [INFO] train episode 308: reward = 259.15, steps = 352
    10:55:25 [INFO] train episode 309: reward = 203.67, steps = 319
    10:56:04 [INFO] train episode 310: reward = 229.39, steps = 901
    10:56:25 [INFO] train episode 311: reward = 268.76, steps = 466
    10:57:09 [INFO] train episode 312: reward = 144.95, steps = 1000
    10:57:53 [INFO] train episode 313: reward = -7.47, steps = 1000
    10:58:37 [INFO] train episode 314: reward = 148.34, steps = 1000
    10:59:20 [INFO] train episode 315: reward = 143.54, steps = 1000
    11:00:07 [INFO] train episode 316: reward = 86.12, steps = 1000
    11:00:40 [INFO] train episode 317: reward = 205.33, steps = 744
    11:01:24 [INFO] train episode 318: reward = 162.79, steps = 1000
    11:01:47 [INFO] train episode 319: reward = 204.85, steps = 510
    11:02:03 [INFO] train episode 320: reward = 242.95, steps = 361
    11:02:15 [INFO] train episode 321: reward = 227.31, steps = 292
    11:03:00 [INFO] train episode 322: reward = 67.16, steps = 1000
    11:03:44 [INFO] train episode 323: reward = 164.35, steps = 1000
    11:04:03 [INFO] train episode 324: reward = 226.79, steps = 440
    11:04:17 [INFO] train episode 325: reward = 253.51, steps = 320
    11:04:51 [INFO] train episode 326: reward = 223.09, steps = 730
    11:05:07 [INFO] train episode 327: reward = 291.06, steps = 369
    11:05:27 [INFO] train episode 328: reward = 222.47, steps = 434
    11:05:52 [INFO] train episode 329: reward = 240.93, steps = 563
    11:06:05 [INFO] train episode 330: reward = 296.07, steps = 308
    11:06:20 [INFO] train episode 331: reward = 250.09, steps = 328
    11:06:37 [INFO] train episode 332: reward = 241.10, steps = 395
    11:06:56 [INFO] train episode 333: reward = 250.09, steps = 431
    11:07:14 [INFO] train episode 334: reward = 247.34, steps = 405
    11:07:15 [INFO] ==== test ====
    11:07:15 [INFO] test episode 0: reward = 271.18, steps = 299
    11:07:15 [INFO] test episode 1: reward = 261.38, steps = 254
    11:07:16 [INFO] test episode 2: reward = 0.23, steps = 271
    11:07:16 [INFO] test episode 3: reward = 248.13, steps = 267
    11:07:17 [INFO] test episode 4: reward = 224.61, steps = 330
    11:07:18 [INFO] test episode 5: reward = 216.57, steps = 342
    11:07:18 [INFO] test episode 6: reward = 248.86, steps = 355
    11:07:19 [INFO] test episode 7: reward = 229.78, steps = 296
    11:07:20 [INFO] test episode 8: reward = 274.99, steps = 322
    11:07:21 [INFO] test episode 9: reward = 181.28, steps = 491
    11:07:21 [INFO] test episode 10: reward = 247.33, steps = 311
    11:07:22 [INFO] test episode 11: reward = 235.35, steps = 499
    11:07:23 [INFO] test episode 12: reward = 247.33, steps = 311
    11:07:24 [INFO] test episode 13: reward = 278.01, steps = 264
    11:07:24 [INFO] test episode 14: reward = 239.55, steps = 322
    11:07:25 [INFO] test episode 15: reward = 254.86, steps = 298
    11:07:25 [INFO] test episode 16: reward = 246.03, steps = 364
    11:07:26 [INFO] test episode 17: reward = 250.75, steps = 338
    11:07:27 [INFO] test episode 18: reward = 255.87, steps = 402
    11:07:28 [INFO] test episode 19: reward = 248.91, steps = 298
    11:07:28 [INFO] test episode 20: reward = 287.79, steps = 324
    11:07:29 [INFO] test episode 21: reward = 275.28, steps = 355
    11:07:29 [INFO] test episode 22: reward = 236.50, steps = 367
    11:07:31 [INFO] test episode 23: reward = 192.14, steps = 454
    11:07:33 [INFO] test episode 24: reward = 169.51, steps = 788
    11:07:34 [INFO] test episode 25: reward = 261.63, steps = 444
    11:07:35 [INFO] test episode 26: reward = 228.02, steps = 292
    11:07:36 [INFO] test episode 27: reward = 239.05, steps = 375
    11:07:36 [INFO] test episode 28: reward = 240.58, steps = 326
    11:07:39 [INFO] test episode 29: reward = 161.61, steps = 1000
    11:07:39 [INFO] test episode 30: reward = 252.69, steps = 290
    11:07:40 [INFO] test episode 31: reward = 258.46, steps = 288
    11:07:40 [INFO] test episode 32: reward = 273.27, steps = 326
    11:07:41 [INFO] test episode 33: reward = 256.26, steps = 282
    11:07:41 [INFO] test episode 34: reward = 267.48, steps = 331
    11:07:42 [INFO] test episode 35: reward = 244.72, steps = 344
    11:07:43 [INFO] test episode 36: reward = 247.76, steps = 336
    11:07:44 [INFO] test episode 37: reward = 217.31, steps = 372
    11:07:44 [INFO] test episode 38: reward = 247.06, steps = 342
    11:07:45 [INFO] test episode 39: reward = 211.10, steps = 340
    11:07:45 [INFO] test episode 40: reward = 264.69, steps = 312
    11:07:46 [INFO] test episode 41: reward = 232.47, steps = 303
    11:07:47 [INFO] test episode 42: reward = 228.02, steps = 382
    11:07:48 [INFO] test episode 43: reward = 251.82, steps = 327
    11:07:48 [INFO] test episode 44: reward = 252.08, steps = 304
    11:07:49 [INFO] test episode 45: reward = 220.72, steps = 321
    11:07:49 [INFO] test episode 46: reward = 238.09, steps = 409
    11:07:50 [INFO] test episode 47: reward = 230.84, steps = 334
    11:07:52 [INFO] test episode 48: reward = 215.48, steps = 612
    11:07:52 [INFO] test episode 49: reward = 223.35, steps = 372
    11:07:55 [INFO] test episode 50: reward = -17.11, steps = 1000
    11:07:55 [INFO] test episode 51: reward = 228.41, steps = 294
    11:07:56 [INFO] test episode 52: reward = 255.20, steps = 339
    11:07:57 [INFO] test episode 53: reward = 220.52, steps = 498
    11:07:58 [INFO] test episode 54: reward = 215.52, steps = 291
    11:07:59 [INFO] test episode 55: reward = 243.97, steps = 491
    11:08:00 [INFO] test episode 56: reward = 211.23, steps = 440
    11:08:01 [INFO] test episode 57: reward = 213.63, steps = 306
    11:08:01 [INFO] test episode 58: reward = 252.84, steps = 302
    11:08:02 [INFO] test episode 59: reward = 284.78, steps = 265
    11:08:02 [INFO] test episode 60: reward = 209.75, steps = 399
    11:08:03 [INFO] test episode 61: reward = 254.63, steps = 268
    11:08:07 [INFO] test episode 62: reward = -49.69, steps = 1000
    11:08:07 [INFO] test episode 63: reward = 278.15, steps = 308
    11:08:08 [INFO] test episode 64: reward = 243.76, steps = 468
    11:08:09 [INFO] test episode 65: reward = 203.84, steps = 374
    11:08:10 [INFO] test episode 66: reward = 267.18, steps = 285
    11:08:10 [INFO] test episode 67: reward = 233.67, steps = 420
    11:08:11 [INFO] test episode 68: reward = 251.62, steps = 337
    11:08:12 [INFO] test episode 69: reward = 237.55, steps = 330
    11:08:12 [INFO] test episode 70: reward = 282.45, steps = 364
    11:08:13 [INFO] test episode 71: reward = 271.30, steps = 328
    11:08:14 [INFO] test episode 72: reward = 266.89, steps = 313
    11:08:14 [INFO] test episode 73: reward = 224.90, steps = 364
    11:08:15 [INFO] test episode 74: reward = 231.54, steps = 401
    11:08:16 [INFO] test episode 75: reward = 240.63, steps = 249
    11:08:16 [INFO] test episode 76: reward = 238.24, steps = 325
    11:08:17 [INFO] test episode 77: reward = 267.44, steps = 272
    11:08:18 [INFO] test episode 78: reward = 262.40, steps = 370
    11:08:18 [INFO] test episode 79: reward = 222.95, steps = 283
    11:08:19 [INFO] test episode 80: reward = 231.19, steps = 319
    11:08:20 [INFO] test episode 81: reward = 209.95, steps = 433
    11:08:20 [INFO] test episode 82: reward = 224.54, steps = 323
    11:08:21 [INFO] test episode 83: reward = 198.86, steps = 316
    11:08:21 [INFO] test episode 84: reward = 233.75, steps = 260
    11:08:22 [INFO] test episode 85: reward = 217.96, steps = 346
    11:08:23 [INFO] test episode 86: reward = 259.67, steps = 307
    11:08:23 [INFO] test episode 87: reward = 209.53, steps = 406
    11:08:24 [INFO] test episode 88: reward = 255.52, steps = 316
    11:08:24 [INFO] test episode 89: reward = 268.38, steps = 247
    11:08:25 [INFO] test episode 90: reward = 249.62, steps = 336
    11:08:26 [INFO] test episode 91: reward = 224.22, steps = 364
    11:08:26 [INFO] test episode 92: reward = 284.50, steps = 305
    11:08:27 [INFO] test episode 93: reward = 232.68, steps = 283
    11:08:28 [INFO] test episode 94: reward = 227.78, steps = 326
    11:08:28 [INFO] test episode 95: reward = 252.30, steps = 280
    11:08:29 [INFO] test episode 96: reward = 214.03, steps = 347
    11:08:31 [INFO] test episode 97: reward = 165.31, steps = 765
    11:08:31 [INFO] test episode 98: reward = 254.61, steps = 356
    11:08:32 [INFO] test episode 99: reward = 281.94, steps = 407
    11:08:32 [INFO] average episode reward = 232.31 ± 51.68