# Use Soft Actor-Critic with Auto $\alpha$ Tuning to Play LunarLander-v2¶

TensorFlow version

In [1]:

    %matplotlib inline

    import sys
    import logging
    import itertools

    import numpy as np
    np.random.seed(0)
    import scipy.special
    import pandas as pd
    import gym
    import matplotlib.pyplot as plt
    import tensorflow.compat.v2 as tf
    tf.random.set_seed(0)
    from tensorflow import nn
    from tensorflow import losses
    from tensorflow import optimizers
    from tensorflow import keras
    from tensorflow.keras import layers
    from tensorflow.keras import models

    logging.basicConfig(level=logging.INFO,
            format='%(asctime)s [%(levelname)s] %(message)s',
            stream=sys.stdout, datefmt='%H:%M:%S')

In [2]:

    env = gym.make('LunarLander-v2')
    for key in vars(env):
        logging.info('%s: %s', key, vars(env)[key])
    for key in vars(env.spec):
        logging.info('%s: %s', key, vars(env.spec)[key])

    08:02:30 [INFO] env: <LunarLander<LunarLander-v2>>
    08:02:30 [INFO] action_space: Discrete(4)
    08:02:30 [INFO] observation_space: Box(-inf, inf, (8,), float32)
    08:02:30 [INFO] reward_range: (-inf, inf)
    08:02:30 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 50}
    08:02:30 [INFO] _max_episode_steps: 1000
    08:02:30 [INFO] _elapsed_steps: None
    08:02:30 [INFO] id: LunarLander-v2
    08:02:30 [INFO] entry_point: gym.envs.box2d:LunarLander
    08:02:30 [INFO] reward_threshold: 200
    08:02:30 [INFO] nondeterministic: False
    08:02:30 [INFO] max_episode_steps: 1000
    08:02:30 [INFO] _kwargs: {}
    08:02:30 [INFO] _env_name: LunarLander

In [3]:

    class DQNReplayer:
        def __init__(self, capacity):
            self.memory = pd.DataFrame(index=range(capacity),
                    columns=['state', 'action', 'reward', 'next_state', 'terminated'])
            self.i = 0
            self.count = 0
            self.capacity = capacity

        def store(self, *args):
            self.memory.loc[self.i] = np.asarray(args, dtype=object)
            self.i = (self.i + 1) % self.capacity
            self.count = min(self.count + 1, self.capacity)

        def sample(self, size):
            indices = np.random.choice(self.count, size=size)
            return (np.stack(self.memory.loc[indices, field]) for field in
                    self.memory.columns)

In [4]:

    class SACAgent:
        def __init__(self, env):
            state_dim = env.observation_space.shape[0]
            self.action_n = env.action_space.n
            self.gamma = 0.99

            self.replayer = DQNReplayer(100000)

            # create alpha
            self.target_entropy = np.log(self.action_n) / 4.
            self.ln_alpha_tensor = tf.Variable(0., dtype=tf.float32)
            self.alpha_optimizer = optimizers.Adam(0.0003)

            # create actor
            self.actor_net = self.build_net(hidden_sizes=[256, 256],
                    output_size=self.action_n, output_activation=nn.softmax)

            # create Q critic
            self.q0_net = self.build_net(hidden_sizes=[256, 256],
                    output_size=self.action_n)
            self.q1_net = self.build_net(hidden_sizes=[256, 256],
                    output_size=self.action_n)

            # create V critic
            self.v_evaluate_net = self.build_net(input_size=state_dim,
                    hidden_sizes=[256, 256])
            self.v_target_net = models.clone_model(self.v_evaluate_net)

        def build_net(self, hidden_sizes, output_size=1,
                    activation=nn.relu, output_activation=None, input_size=None,
                    loss=losses.mse, learning_rate=0.0003):
            model = keras.Sequential()
            for layer_idx, hidden_size in enumerate(hidden_sizes):
                kwargs = {'input_shape': (input_size,)} if \
                        layer_idx == 0 and input_size is not None else {}
                model.add(layers.Dense(units=hidden_size,
                        activation=activation, **kwargs))
            model.add(layers.Dense(units=output_size,
                    activation=output_activation))
            optimizer = optimizers.Adam(learning_rate)
            model.compile(optimizer=optimizer, loss=loss)
            return model

        def reset(self, mode=None):
            self.mode = mode
            if self.mode == 'train':
                self.trajectory = []

        def step(self, observation, reward, terminated):
            probs = self.actor_net.predict(observation[np.newaxis], verbose=0)[0]
            action = np.random.choice(self.action_n, p=probs)
            if self.mode == 'train':
                self.trajectory += [observation, reward, terminated, action]
                if len(self.trajectory) >= 8:
                    state, _, _, action, next_state, reward, terminated, _ = \
                            self.trajectory[-8:]
                    self.replayer.store(state, action, reward, next_state, terminated)
                if self.replayer.count >= 500:
                    self.learn()
            return action

        def close(self):
            pass

        def update_net(self, target_net, evaluate_net, learning_rate=0.005):
            average_weights = [(1. - learning_rate) * t + learning_rate * e for t, e
                    in zip(target_net.get_weights(), evaluate_net.get_weights())]
            target_net.set_weights(average_weights)

        def learn(self):
            states, actions, rewards, next_states, terminateds = \
                    self.replayer.sample(128)

            # update alpha
            all_probs = self.actor_net.predict(states, verbose=0)
            probs = np.take_along_axis(all_probs, actions[np.newaxis, :], axis=-1)
            ln_probs = np.log(probs.clip(1e-6, 1.))
            mean_ln_prob = ln_probs.mean()
            with tf.GradientTape() as tape:
                alpha_loss_tensor = -self.ln_alpha_tensor * (mean_ln_prob +
                        self.target_entropy)
            grads = tape.gradient(alpha_loss_tensor, [self.ln_alpha_tensor,])
            self.alpha_optimizer.apply_gradients(zip(grads, [self.ln_alpha_tensor,]))

            # update V critic
            q0s = self.q0_net.predict(states, verbose=0)
            q1s = self.q1_net.predict(states, verbose=0)
            q01s = np.minimum(q0s, q1s)
            pis = self.actor_net.predict(states, verbose=0)
            alpha = tf.exp(self.ln_alpha_tensor).numpy()
            entropic_q01s = pis * q01s - alpha * scipy.special.xlogy(pis, pis)
            v_targets = entropic_q01s.sum(axis=-1)
            self.v_evaluate_net.fit(states, v_targets, verbose=0)
            self.update_net(self.v_target_net, self.v_evaluate_net)

            # update Q critic
            next_vs = self.v_target_net.predict(next_states, verbose=0)
            q_targets = rewards[:, np.newaxis] + \
                    self.gamma * (1. - terminateds[:, np.newaxis]) * next_vs
            np.put_along_axis(q0s, actions.reshape(-1, 1), q_targets, -1)
            np.put_along_axis(q1s, actions.reshape(-1, 1), q_targets, -1)
            self.q0_net.fit(states, q0s, verbose=0)
            self.q1_net.fit(states, q1s, verbose=0)

            # update actor
            state_tensor = tf.convert_to_tensor(states, dtype=tf.float32)
            q0s_tensor = self.q0_net(state_tensor)
            with tf.GradientTape() as tape:
                probs_tensor = self.actor_net(state_tensor)
                alpha_tensor = tf.exp(self.ln_alpha_tensor)
                losses_tensor = alpha_tensor * tf.math.xlogy(probs_tensor,
                        probs_tensor) - probs_tensor * q0s_tensor
                actor_loss_tensor = tf.reduce_sum(losses_tensor, axis=-1)
            grads = tape.gradient(actor_loss_tensor,
                    self.actor_net.trainable_variables)
            self.actor_net.optimizer.apply_gradients(zip(grads,
                    self.actor_net.trainable_variables))

    agent = SACAgent(env)

In [5]:

    def play_episode(env, agent, seed=None, mode=None, render=False):
        observation, _ = env.reset(seed=seed)
        reward, terminated, truncated = 0., False, False
        agent.reset(mode=mode)
        episode_reward, elapsed_steps = 0., 0
        while True:
            action = agent.step(observation, reward, terminated)
            if render:
                env.render()
            if terminated or truncated:
                break
            observation, reward, terminated, truncated, _ = env.step(action)
            episode_reward += reward
            elapsed_steps += 1
        agent.close()
        return episode_reward, elapsed_steps

    logging.info('==== train ====')
    episode_rewards = []
    for episode in itertools.count():
        episode_reward, elapsed_steps = play_episode(env, agent, seed=episode,
                mode='train')
        episode_rewards.append(episode_reward)
        logging.info('train episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
        if np.mean(episode_rewards[-10:]) > 250:
            break
    plt.plot(episode_rewards)

    logging.info('==== test ====')
    episode_rewards = []
    for episode in range(100):
        episode_reward, elapsed_steps = play_episode(env, agent)
        episode_rewards.append(episode_reward)
        logging.info('test episode %d: reward = %.2f, steps = %d',
                episode, episode_reward, elapsed_steps)
    logging.info('average episode reward = %.2f ± %.2f',
            np.mean(episode_rewards), np.std(episode_rewards))

    08:02:31 [INFO] ==== train ====
    08:02:37 [INFO] train episode 0: reward = -75.95, steps = 112
    08:02:44 [INFO] train episode 1: reward = -236.09, steps = 133
    08:02:48 [INFO] train episode 2: reward = -112.13, steps = 76
    08:02:52 [INFO] train episode 3: reward = -117.56, steps = 83
    08:02:56 [INFO] train episode 4: reward = -197.16, steps = 79
    08:03:51 [INFO] train episode 5: reward = -152.45, steps = 130
    08:04:31 [INFO] train episode 6: reward = -506.29, steps = 87
    08:05:27 [INFO] train episode 7: reward = -259.64, steps = 122
    08:06:09 [INFO] train episode 8: reward = -412.11, steps = 90
    08:06:43 [INFO] train episode 9: reward = -101.70, steps = 70
    08:07:16 [INFO] train episode 10: reward = -102.76, steps = 67
    08:08:00 [INFO] train episode 11: reward = -137.17, steps = 94
    08:08:43 [INFO] train episode 12: reward = -114.29, steps = 94
    08:09:21 [INFO] train episode 13: reward = -283.18, steps = 83
    08:09:53 [INFO] train episode 14: reward = -137.77, steps = 68
    08:10:27 [INFO] train episode 15: reward = -118.11, steps = 74
    08:11:37 [INFO] train episode 16: reward = -203.25, steps = 152
    08:12:14 [INFO] train episode 17: reward = -356.60, steps = 81
    08:12:52 [INFO] train episode 18: reward = -134.26, steps = 82
    08:13:35 [INFO] train episode 19: reward = -130.33, steps = 94
    08:14:12 [INFO] train episode 20: reward = -426.51, steps = 80
    08:14:52 [INFO] train episode 21: reward = -81.44, steps = 86
    08:15:31 [INFO] train episode 22: reward = -98.83, steps = 85
    08:16:09 [INFO] train episode 23: reward = -118.78, steps = 82
    08:17:14 [INFO] train episode 24: reward = -129.82, steps = 141
    08:17:47 [INFO] train episode 25: reward = -30.11, steps = 73
    08:18:43 [INFO] train episode 26: reward = -369.42, steps = 121
    08:19:20 [INFO] train episode 27: reward = -94.13, steps = 80
    08:19:55 [INFO] train episode 28: reward = -90.37, steps = 76
    08:20:30 [INFO] train episode 29: reward = -85.42, steps = 77
    08:21:22 [INFO] train episode 30: reward = -229.63, steps = 109
    08:22:07 [INFO] train episode 31: reward = -155.77, steps = 98
    08:22:58 [INFO] train episode 32: reward = -316.18, steps = 110
    08:23:54 [INFO] train episode 33: reward = -83.58, steps = 122
    08:24:47 [INFO] train episode 34: reward = -385.46, steps = 115
    08:25:35 [INFO] train episode 35: reward = -2.04, steps = 105
    08:26:10 [INFO] train episode 36: reward = -467.38, steps = 76
    08:26:55 [INFO] train episode 37: reward = -347.97, steps = 98
    08:27:29 [INFO] train episode 38: reward = -116.66, steps = 72
    08:28:44 [INFO] train episode 39: reward = 7.40, steps = 165
    08:29:42 [INFO] train episode 40: reward = -327.83, steps = 127
    08:30:39 [INFO] train episode 41: reward = -386.37, steps = 124
    08:31:36 [INFO] train episode 42: reward = -344.11, steps = 126
    08:32:18 [INFO] train episode 43: reward = -233.06, steps = 92
    08:33:39 [INFO] train episode 44: reward = -301.23, steps = 177
    08:34:17 [INFO] train episode 45: reward = -24.55, steps = 83
    08:35:09 [INFO] train episode 46: reward = -356.28, steps = 113
    08:36:00 [INFO] train episode 47: reward = -59.24, steps = 112
    08:36:37 [INFO] train episode 48: reward = -341.94, steps = 80
    08:37:09 [INFO] train episode 49: reward = -650.99, steps = 69
    08:37:59 [INFO] train episode 50: reward = -415.77, steps = 109
    08:38:47 [INFO] train episode 51: reward = -436.33, steps = 106
    08:39:34 [INFO] train episode 52: reward = -190.36, steps = 103
    08:40:21 [INFO] train episode 53: reward = -254.22, steps = 101
    08:41:56 [INFO] train episode 54: reward = -109.64, steps = 208
    08:43:11 [INFO] train episode 55: reward = -261.75, steps = 163
    08:43:45 [INFO] train episode 56: reward = -563.66, steps = 75
    08:45:38 [INFO] train episode 57: reward = -258.85, steps = 246
    08:46:22 [INFO] train episode 58: reward = -365.89, steps = 93
    08:47:44 [INFO] train episode 59: reward = -335.37, steps = 173
    08:49:08 [INFO] train episode 60: reward = -113.95, steps = 184
    08:50:08 [INFO] train episode 61: reward = -94.67, steps = 131
    08:51:00 [INFO] train episode 62: reward = -157.79, steps = 113
    08:51:59 [INFO] train episode 63: reward = -187.94, steps = 127
    08:53:18 [INFO] train episode 64: reward = -134.65, steps = 172
    08:54:26 [INFO] train episode 65: reward = -261.54, steps = 147
    08:56:15 [INFO] train episode 66: reward = -389.67, steps = 239
    08:57:47 [INFO] train episode 67: reward = -232.42, steps = 198
    08:58:54 [INFO] train episode 68: reward = -132.40, steps = 147
    09:00:30 [INFO] train episode 69: reward = -427.47, steps = 209
    09:05:05 [INFO] train episode 70: reward = -400.11, steps = 603
    09:10:57 [INFO] train episode 71: reward = -256.93, steps = 772
    09:16:44 [INFO] train episode 72: reward = -204.26, steps = 763
    09:19:17 [INFO] train episode 73: reward = -264.97, steps = 337
    09:21:22 [INFO] train episode 74: reward = -195.56, steps = 271
    09:23:12 [INFO] train episode 75: reward = -133.38, steps = 243
    09:25:51 [INFO] train episode 76: reward = -140.71, steps = 349
    09:29:52 [INFO] train episode 77: reward = -247.25, steps = 527
    09:31:15 [INFO] train episode 78: reward = -238.57, steps = 182
    09:32:44 [INFO] train episode 79: reward = -241.64, steps = 195
    09:34:58 [INFO] train episode 80: reward = -169.24, steps = 293
    09:37:54 [INFO] train episode 81: reward = -177.31, steps = 388
    09:39:09 [INFO] train episode 82: reward = -225.47, steps = 164
    09:42:34 [INFO] train episode 83: reward = -201.34, steps = 449
    09:45:06 [INFO] train episode 84: reward = -150.42, steps = 333
    09:49:46 [INFO] train episode 85: reward = -203.66, steps = 612
    09:52:50 [INFO] train episode 86: reward = -312.12, steps = 402
    09:56:26 [INFO] train episode 87: reward = -240.03, steps = 472
    10:00:50 [INFO] train episode 88: reward = -191.87, steps = 577
    10:03:48 [INFO] train episode 89: reward = -189.17, steps = 391
    10:08:54 [INFO] train episode 90: reward = -220.18, steps = 670
    10:16:31 [INFO] train episode 91: reward = -104.33, steps = 1000
    10:24:11 [INFO] train episode 92: reward = -145.35, steps = 1000
    10:31:50 [INFO] train episode 93: reward = -112.08, steps = 1000
    10:39:26 [INFO] train episode 94: reward = -121.60, steps = 1000
    10:47:02 [INFO] train episode 95: reward = -56.79, steps = 1000
    10:54:38 [INFO] train episode 96: reward = -34.97, steps = 1000
    11:02:17 [INFO] train episode 97: reward = -78.72, steps = 1000
    11:09:55 [INFO] train episode 98: reward = -62.93, steps = 1000
    11:17:33 [INFO] train episode 99: reward = -58.33, steps = 1000
    11:25:12 [INFO] train episode 100: reward = -98.27, steps = 1000
    11:29:53 [INFO] train episode 101: reward = -106.79, steps = 613
    11:34:57 [INFO] train episode 102: reward = -163.99, steps = 663
    11:41:24 [INFO] train episode 103: reward = -224.04, steps = 847
    11:45:43 [INFO] train episode 104: reward = -172.76, steps = 563
    11:53:21 [INFO] train episode 105: reward = -94.31, steps = 1000
    12:01:01 [INFO] train episode 106: reward = -57.36, steps = 1000
    12:04:52 [INFO] train episode 107: reward = -110.76, steps = 504
    12:09:54 [INFO] train episode 108: reward = -140.11, steps = 660
    12:17:33 [INFO] train episode 109: reward = -226.73, steps = 1000
    12:25:14 [INFO] train episode 110: reward = -83.33, steps = 1000
    12:32:52 [INFO] train episode 111: reward = -88.26, steps = 1000
    12:37:42 [INFO] train episode 112: reward = -196.34, steps = 631
    12:44:34 [INFO] train episode 113: reward = -237.39, steps = 896
    12:45:01 [INFO] train episode 114: reward = -132.74, steps = 58
    12:51:09 [INFO] train episode 115: reward = -203.59, steps = 802
    12:57:36 [INFO] train episode 116: reward = -165.61, steps = 842
    13:04:24 [INFO] train episode 117: reward = -190.53, steps = 890
    13:09:35 [INFO] train episode 118: reward = -364.84, steps = 675
    13:15:22 [INFO] train episode 119: reward = 112.08, steps = 755
    13:22:04 [INFO] train episode 120: reward = -184.56, steps = 873
    13:26:11 [INFO] train episode 121: reward = -109.52, steps = 538
    13:33:51 [INFO] train episode 122: reward = -18.55, steps = 1000
    13:40:37 [INFO] train episode 123: reward = 183.85, steps = 882
    13:47:53 [INFO] train episode 124: reward = 123.11, steps = 945
    13:53:36 [INFO] train episode 125: reward = 175.19, steps = 746
    14:01:19 [INFO] train episode 126: reward = -56.07, steps = 1000
    14:09:00 [INFO] train episode 127: reward = -68.65, steps = 1000
    14:15:44 [INFO] train episode 128: reward = 177.29, steps = 877
    14:18:19 [INFO] train episode 129: reward = -375.45, steps = 337
    14:26:03 [INFO] train episode 130: reward = -45.90, steps = 1000
    14:33:44 [INFO] train episode 131: reward = -106.52, steps = 1000
    14:41:23 [INFO] train episode 132: reward = 3.29, steps = 1000
    14:49:05 [INFO] train episode 133: reward = -88.52, steps = 1000
    14:54:38 [INFO] train episode 134: reward = 157.73, steps = 724
    14:59:04 [INFO] train episode 135: reward = 201.34, steps = 574
    15:03:12 [INFO] train episode 136: reward = 234.49, steps = 539
    15:08:15 [INFO] train episode 137: reward = 154.08, steps = 654
    15:13:42 [INFO] train episode 138: reward = 202.69, steps = 709
    15:18:19 [INFO] train episode 139: reward = -366.40, steps = 598
    15:23:05 [INFO] train episode 140: reward = 200.71, steps = 614
    15:27:50 [INFO] train episode 141: reward = 149.77, steps = 617
    15:32:58 [INFO] train episode 142: reward = 187.48, steps = 670
    15:33:27 [INFO] train episode 143: reward = -94.66, steps = 61
    15:38:16 [INFO] train episode 144: reward = 171.54, steps = 626
    15:42:49 [INFO] train episode 145: reward = 178.14, steps = 590
    15:47:14 [INFO] train episode 146: reward = 163.63, steps = 574
    15:50:47 [INFO] train episode 147: reward = -572.60, steps = 462
    15:54:34 [INFO] train episode 148: reward = 203.35, steps = 492
    15:58:07 [INFO] train episode 149: reward = -32.52, steps = 460
    16:01:43 [INFO] train episode 150: reward = 186.49, steps = 464
    16:06:37 [INFO] train episode 151: reward = 226.66, steps = 633
    16:11:27 [INFO] train episode 152: reward = 208.74, steps = 628
    16:14:15 [INFO] train episode 153: reward = 7.96, steps = 361
    16:19:32 [INFO] train episode 154: reward = 186.07, steps = 687
    16:26:20 [INFO] train episode 155: reward = -188.70, steps = 871
    16:30:22 [INFO] train episode 156: reward = 191.01, steps = 523
    16:35:06 [INFO] train episode 157: reward = 212.77, steps = 614
    16:41:26 [INFO] train episode 158: reward = 154.09, steps = 822
    16:47:57 [INFO] train episode 159: reward = 166.73, steps = 843
    16:51:47 [INFO] train episode 160: reward = 186.16, steps = 498
    16:55:14 [INFO] train episode 161: reward = 267.52, steps = 445
    17:00:43 [INFO] train episode 162: reward = 170.62, steps = 710
    17:04:25 [INFO] train episode 163: reward = 232.51, steps = 480
    17:08:20 [INFO] train episode 164: reward = 161.55, steps = 509
    17:12:56 [INFO] train episode 165: reward = 236.34, steps = 595
    17:18:07 [INFO] train episode 166: reward = -133.76, steps = 672
    17:22:20 [INFO] train episode 167: reward = 176.75, steps = 540
    17:30:01 [INFO] train episode 168: reward = -29.10, steps = 1000
    17:35:39 [INFO] train episode 169: reward = -170.41, steps = 732
    17:39:24 [INFO] train episode 170: reward = 264.84, steps = 486
    17:44:58 [INFO] train episode 171: reward = 182.80, steps = 719
    17:52:40 [INFO] train episode 172: reward = -82.54, steps = 1000
    17:56:45 [INFO] train episode 173: reward = 193.18, steps = 528
    18:02:16 [INFO] train episode 174: reward = 92.57, steps = 715
    18:07:06 [INFO] train episode 175: reward = 171.21, steps = 627
    18:11:01 [INFO] train episode 176: reward = 202.79, steps = 508
    18:14:59 [INFO] train episode 177: reward = 196.55, steps = 512
    18:18:53 [INFO] train episode 178: reward = 191.61, steps = 507
    18:22:50 [INFO] train episode 179: reward = 205.00, steps = 508
    18:27:16 [INFO] train episode 180: reward = 193.25, steps = 577
    18:31:35 [INFO] train episode 181: reward = -36.13, steps = 560
    18:37:09 [INFO] train episode 182: reward = -118.54, steps = 721
    18:42:32 [INFO] train episode 183: reward = 223.79, steps = 697
    18:45:55 [INFO] train episode 184: reward = 223.98, steps = 439
    18:49:46 [INFO] train episode 185: reward = 212.47, steps = 498
    18:54:18 [INFO] train episode 186: reward = 190.49, steps = 588
    18:59:05 [INFO] train episode 187: reward = 204.01, steps = 619
    19:05:36 [INFO] train episode 188: reward = -207.79, steps = 833
    19:09:23 [INFO] train episode 189: reward = 179.33, steps = 490
    19:17:06 [INFO] train episode 190: reward = -56.36, steps = 1000
    19:21:30 [INFO] train episode 191: reward = 207.05, steps = 569
    19:29:05 [INFO] train episode 192: reward = 139.30, steps = 984
    19:32:52 [INFO] train episode 193: reward = 199.19, steps = 490
    19:40:34 [INFO] train episode 194: reward = 22.72, steps = 1000
    19:44:55 [INFO] train episode 195: reward = 163.30, steps = 563
    19:48:32 [INFO] train episode 196: reward = 212.92, steps = 467
    19:52:24 [INFO] train episode 197: reward = 175.05, steps = 501
    19:56:37 [INFO] train episode 198: reward = 212.41, steps = 548
    20:01:04 [INFO] train episode 199: reward = 215.40, steps = 573
    20:05:35 [INFO] train episode 200: reward = 167.67, steps = 584
    20:09:17 [INFO] train episode 201: reward = 210.01, steps = 480
    20:14:56 [INFO] train episode 202: reward = 155.22, steps = 730
    20:22:22 [INFO] train episode 203: reward = -150.00, steps = 960
    20:26:30 [INFO] train episode 204: reward = 206.43, steps = 537
    20:30:54 [INFO] train episode 205: reward = 180.73, steps = 567
    20:35:34 [INFO] train episode 206: reward = 224.24, steps = 607
    20:39:35 [INFO] train episode 207: reward = 178.35, steps = 517
    20:43:51 [INFO] train episode 208: reward = 216.35, steps = 552
    20:47:40 [INFO] train episode 209: reward = 205.18, steps = 493
    20:52:34 [INFO] train episode 210: reward = -181.21, steps = 632
    20:57:04 [INFO] train episode 211: reward = 213.54, steps = 577
    21:03:06 [INFO] train episode 212: reward = 225.10, steps = 774
    21:08:22 [INFO] train episode 213: reward = 232.74, steps = 667
    21:13:33 [INFO] train episode 214: reward = -138.47, steps = 642
    21:21:06 [INFO] train episode 215: reward = 194.47, steps = 766
    21:25:55 [INFO] train episode 216: reward = 236.72, steps = 578
    21:29:20 [INFO] train episode 217: reward = -82.69, steps = 392
    21:35:23 [INFO] train episode 218: reward = 230.71, steps = 622
    21:39:31 [INFO] train episode 219: reward = 226.74, steps = 449
    21:48:36 [INFO] train episode 220: reward = 162.23, steps = 960
    21:53:32 [INFO] train episode 221: reward = 248.25, steps = 499
    21:58:10 [INFO] train episode 222: reward = 223.60, steps = 504
    22:03:05 [INFO] train episode 223: reward = 183.60, steps = 546
    22:07:48 [INFO] train episode 224: reward = 165.75, steps = 523
    22:12:43 [INFO] train episode 225: reward = 232.00, steps = 567
    22:17:31 [INFO] train episode 226: reward = 207.23, steps = 584
    22:21:09 [INFO] train episode 227: reward = 211.69, steps = 436
    22:24:36 [INFO] train episode 228: reward = 260.00, steps = 424
    22:28:50 [INFO] train episode 229: reward = 223.62, steps = 527
    22:32:23 [INFO] train episode 230: reward = 220.29, steps = 441
    22:36:07 [INFO] train episode 231: reward = 245.04, steps = 464
    22:40:41 [INFO] train episode 232: reward = 219.37, steps = 515
    22:46:27 [INFO] train episode 233: reward = 114.84, steps = 696
    22:50:17 [INFO] train episode 234: reward = 259.74, steps = 477
    22:56:17 [INFO] train episode 235: reward = 141.86, steps = 749
    23:00:23 [INFO] train episode 236: reward = 231.35, steps = 512
    23:04:53 [INFO] train episode 237: reward = 197.07, steps = 560
    23:10:12 [INFO] train episode 238: reward = 143.84, steps = 665
    23:15:38 [INFO] train episode 239: reward = 206.60, steps = 678
    23:20:03 [INFO] train episode 240: reward = 247.01, steps = 551
    23:24:24 [INFO] train episode 241: reward = 249.17, steps = 539
    23:25:26 [INFO] train episode 242: reward = -82.05, steps = 128
    23:28:38 [INFO] train episode 243: reward = 207.80, steps = 398
    23:31:53 [INFO] train episode 244: reward = 238.15, steps = 406
    23:35:46 [INFO] train episode 245: reward = 239.06, steps = 485
    23:41:07 [INFO] train episode 246: reward = 237.30, steps = 663
    23:42:08 [INFO] train episode 247: reward = -91.87, steps = 127
    23:46:03 [INFO] train episode 248: reward = 188.21, steps = 486
    23:49:53 [INFO] train episode 249: reward = 211.82, steps = 474
    23:56:25 [INFO] train episode 250: reward = 165.45, steps = 812
    00:17:28 [INFO] train episode 251: reward = 243.37, steps = 520
    00:22:51 [INFO] train episode 252: reward = 201.00, steps = 648
    00:30:46 [INFO] train episode 253: reward = -74.19, steps = 1000
    00:35:39 [INFO] train episode 254: reward = 211.46, steps = 618
    00:40:57 [INFO] train episode 255: reward = 100.75, steps = 671
    00:41:50 [INFO] train episode 256: reward = -138.52, steps = 112
    00:46:03 [INFO] train episode 257: reward = 225.28, steps = 540
    00:49:44 [INFO] train episode 258: reward = 224.98, steps = 468
    00:53:48 [INFO] train episode 259: reward = 253.52, steps = 520
    00:58:47 [INFO] train episode 260: reward = 186.02, steps = 637
    01:03:09 [INFO] train episode 261: reward = 224.77, steps = 557
    01:08:05 [INFO] train episode 262: reward = 234.52, steps = 626
    01:12:24 [INFO] train episode 263: reward = 185.70, steps = 553
    01:15:36 [INFO] train episode 264: reward = 274.19, steps = 409
    01:19:30 [INFO] train episode 265: reward = 221.81, steps = 493
    01:23:23 [INFO] train episode 266: reward = 192.32, steps = 492
    01:26:43 [INFO] train episode 267: reward = 268.10, steps = 427
    01:30:35 [INFO] train episode 268: reward = 204.04, steps = 495
    01:33:56 [INFO] train episode 269: reward = 238.46, steps = 437
    01:36:57 [INFO] train episode 270: reward = 235.13, steps = 395
    01:40:29 [INFO] train episode 271: reward = 213.65, steps = 464
    01:43:33 [INFO] train episode 272: reward = 195.15, steps = 400
    01:48:42 [INFO] train episode 273: reward = 155.72, steps = 672
    01:52:48 [INFO] train episode 274: reward = 242.81, steps = 534
    01:58:09 [INFO] train episode 275: reward = 130.96, steps = 700
    02:01:53 [INFO] train episode 276: reward = 226.36, steps = 487
    02:06:24 [INFO] train episode 277: reward = 199.59, steps = 621
    02:09:28 [INFO] train episode 278: reward = 194.98, steps = 433
    02:14:29 [INFO] train episode 279: reward = 199.99, steps = 707
    02:18:55 [INFO] train episode 280: reward = 189.57, steps = 628
    02:20:29 [INFO] train episode 281: reward = -33.29, steps = 222
    02:24:08 [INFO] train episode 282: reward = 226.63, steps = 513
    02:27:52 [INFO] train episode 283: reward = 214.01, steps = 531
    02:30:26 [INFO] train episode 284: reward = 216.55, steps = 363
    02:32:31 [INFO] train episode 285: reward = -22.65, steps = 293
    02:37:54 [INFO] train episode 286: reward = 98.82, steps = 762
    02:40:02 [INFO] train episode 287: reward = -54.31, steps = 300
    02:43:21 [INFO] train episode 288: reward = 223.02, steps = 469
    02:46:52 [INFO] train episode 289: reward = 241.32, steps = 498
    02:50:28 [INFO] train episode 290: reward = 218.97, steps = 508
    02:53:04 [INFO] train episode 291: reward = 240.64, steps = 368
    02:56:02 [INFO] train episode 292: reward = 216.55, steps = 418
    03:01:01 [INFO] train episode 293: reward = 111.76, steps = 702
    03:05:22 [INFO] train episode 294: reward = -18.49, steps = 614
    03:09:16 [INFO] train episode 295: reward = 157.55, steps = 548
    03:11:44 [INFO] train episode 296: reward = 212.53, steps = 347
    03:16:44 [INFO] train episode 297: reward = 90.44, steps = 704
    03:18:51 [INFO] train episode 298: reward = -13.04, steps = 297
    03:21:41 [INFO] train episode 299: reward = 210.03, steps = 399
    03:25:53 [INFO] train episode 300: reward = 222.90, steps = 589
    03:29:15 [INFO] train episode 301: reward = 182.90, steps = 474
    03:34:26 [INFO] train episode 302: reward = 193.92, steps = 731
    03:37:03 [INFO] train episode 303: reward = 202.91, steps = 370
    03:40:11 [INFO] train episode 304: reward = 249.69, steps = 441
    03:42:14 [INFO] train episode 305: reward = 40.16, steps = 287
    03:45:26 [INFO] train episode 306: reward = 231.12, steps = 451
    03:48:30 [INFO] train episode 307: reward = 239.55, steps = 431
    03:51:42 [INFO] train episode 308: reward = 271.71, steps = 454
    03:54:52 [INFO] train episode 309: reward = 239.44, steps = 448
    03:57:57 [INFO] train episode 310: reward = 249.40, steps = 437
    04:01:09 [INFO] train episode 311: reward = 230.17, steps = 452
    04:04:20 [INFO] train episode 312: reward = 198.32, steps = 450
    04:06:52 [INFO] train episode 313: reward = 234.60, steps = 360
    04:10:02 [INFO] train episode 314: reward = 227.26, steps = 445
    04:14:12 [INFO] train episode 315: reward = 219.90, steps = 587
    04:18:31 [INFO] train episode 316: reward = 214.05, steps = 608
    04:21:32 [INFO] train episode 317: reward = 285.45, steps = 421
    04:24:48 [INFO] train episode 318: reward = 203.29, steps = 456
    04:27:28 [INFO] train episode 319: reward = 241.84, steps = 375
    04:30:38 [INFO] train episode 320: reward = -395.77, steps = 445
    04:32:43 [INFO] train episode 321: reward = 53.58, steps = 292
    04:35:26 [INFO] train episode 322: reward = 252.37, steps = 382
    04:38:03 [INFO] train episode 323: reward = 224.64, steps = 368
    04:41:17 [INFO] train episode 324: reward = 235.02, steps = 453
    04:43:06 [INFO] train episode 325: reward = 40.93, steps = 254
    04:46:41 [INFO] train episode 326: reward = 215.24, steps = 504
    04:48:25 [INFO] train episode 327: reward = 214.65, steps = 246
    04:51:18 [INFO] train episode 328: reward = 209.03, steps = 404
    04:53:55 [INFO] train episode 329: reward = 278.41, steps = 368
    04:58:01 [INFO] train episode 330: reward = 238.00, steps = 574
    05:00:31 [INFO] train episode 331: reward = 225.78, steps = 351
    05:02:40 [INFO] train episode 332: reward = 266.91, steps = 301
    05:05:59 [INFO] train episode 333: reward = 269.80, steps = 464
    05:09:00 [INFO] train episode 334: reward = 210.96, steps = 426
    05:11:53 [INFO] train episode 335: reward = 198.76, steps = 405
    05:12:25 [INFO] train episode 336: reward = 39.65, steps = 75
    05:17:39 [INFO] train episode 337: reward = 185.53, steps = 737
    05:18:59 [INFO] train episode 338: reward = 8.63, steps = 189
    05:22:48 [INFO] train episode 339: reward = 214.94, steps = 534
    05:29:50 [INFO] train episode 340: reward = 178.86, steps = 986
    05:32:42 [INFO] train episode 341: reward = 226.91, steps = 403
    05:34:58 [INFO] train episode 342: reward = 11.28, steps = 320
    05:38:07 [INFO] train episode 343: reward = 232.04, steps = 440
    05:40:04 [INFO] train episode 344: reward = 239.20, steps = 273
    05:42:02 [INFO] train episode 345: reward = 233.55, steps = 275
    05:44:52 [INFO] train episode 346: reward = 274.66, steps = 397
    05:47:45 [INFO] train episode 347: reward = 244.40, steps = 403
    05:49:37 [INFO] train episode 348: reward = 261.41, steps = 261
    05:53:18 [INFO] train episode 349: reward = 235.26, steps = 516
    05:55:37 [INFO] train episode 350: reward = 261.96, steps = 325
    05:57:40 [INFO] train episode 351: reward = 263.42, steps = 290
    05:59:51 [INFO] train episode 352: reward = 231.06, steps = 307
    05:01:45 [INFO] train episode 353: reward = 255.21, steps = 267
    05:01:45 [INFO] ==== test ====
    05:02:23 [INFO] test episode 0: reward = 197.07, steps = 813
    05:02:44 [INFO] test episode 1: reward = 231.94, steps = 427
    05:03:01 [INFO] test episode 2: reward = 269.95, steps = 378
    05:03:29 [INFO] test episode 3: reward = 213.18, steps = 578
    05:03:46 [INFO] test episode 4: reward = 273.32, steps = 362
    05:04:08 [INFO] test episode 5: reward = 220.25, steps = 469
    05:04:56 [INFO] test episode 6: reward = 59.40, steps = 1000
    05:05:25 [INFO] test episode 7: reward = 221.96, steps = 594
    05:05:41 [INFO] test episode 8: reward = 267.61, steps = 342
    05:06:08 [INFO] test episode 9: reward = 211.02, steps = 572
    05:06:36 [INFO] test episode 10: reward = 213.36, steps = 590
    05:06:59 [INFO] test episode 11: reward = 199.88, steps = 504
    05:07:22 [INFO] test episode 12: reward = 239.25, steps = 476
    05:07:38 [INFO] test episode 13: reward = 250.92, steps = 346
    05:07:51 [INFO] test episode 14: reward = 269.47, steps = 260
    05:08:09 [INFO] test episode 15: reward = 271.80, steps = 386
    05:08:29 [INFO] test episode 16: reward = 254.40, steps = 437
    05:09:02 [INFO] test episode 17: reward = 215.61, steps = 693
    05:09:17 [INFO] test episode 18: reward = 265.10, steps = 309
    05:09:34 [INFO] test episode 19: reward = 204.88, steps = 365
    05:09:59 [INFO] test episode 20: reward = -2.64, steps = 522
    05:10:23 [INFO] test episode 21: reward = 193.42, steps = 504
    05:10:52 [INFO] test episode 22: reward = 220.02, steps = 617
    05:11:14 [INFO] test episode 23: reward = 283.49, steps = 462
    05:11:27 [INFO] test episode 24: reward = 245.51, steps = 286
    05:11:43 [INFO] test episode 25: reward = 220.60, steps = 331
    05:12:32 [INFO] test episode 26: reward = -19.49, steps = 1000
    05:12:45 [INFO] test episode 27: reward = 264.13, steps = 279
    05:13:16 [INFO] test episode 28: reward = 201.56, steps = 652
    05:13:34 [INFO] test episode 29: reward = 276.81, steps = 378
    05:14:01 [INFO] test episode 30: reward = 209.07, steps = 565
    05:14:15 [INFO] test episode 31: reward = 241.68, steps = 300
    05:14:33 [INFO] test episode 32: reward = 265.74, steps = 380
    05:14:53 [INFO] test episode 33: reward = 272.84, steps = 416
    05:15:26 [INFO] test episode 34: reward = 206.03, steps = 698
    05:15:42 [INFO] test episode 35: reward = 265.84, steps = 339
    05:16:06 [INFO] test episode 36: reward = 212.65, steps = 496
    05:16:18 [INFO] test episode 37: reward = 259.87, steps = 259
    05:16:34 [INFO] test episode 38: reward = 258.43, steps = 333
    05:17:03 [INFO] test episode 39: reward = 204.68, steps = 599
    05:17:37 [INFO] test episode 40: reward = 213.44, steps = 700
    05:17:52 [INFO] test episode 41: reward = 214.49, steps = 321
    05:18:10 [INFO] test episode 42: reward = 258.41, steps = 380
    05:18:31 [INFO] test episode 43: reward = 215.57, steps = 439
    05:18:50 [INFO] test episode 44: reward = 266.78, steps = 393
    05:19:20 [INFO] test episode 45: reward = 214.50, steps = 627
    05:19:54 [INFO] test episode 46: reward = 203.00, steps = 722
    05:20:22 [INFO] test episode 47: reward = 205.55, steps = 601
    05:20:52 [INFO] test episode 48: reward = 203.82, steps = 634
    05:21:06 [INFO] test episode 49: reward = 261.20, steps = 285
    05:21:33 [INFO] test episode 50: reward = 209.59, steps = 553
    05:21:46 [INFO] test episode 51: reward = 231.71, steps = 277
    05:22:01 [INFO] test episode 52: reward = 253.47, steps = 324
    05:22:33 [INFO] test episode 53: reward = 211.76, steps = 676
    05:22:47 [INFO] test episode 54: reward = 267.07, steps = 279
    05:23:16 [INFO] test episode 55: reward = 216.68, steps = 627
    05:23:32 [INFO] test episode 56: reward = 235.41, steps = 323
    05:23:54 [INFO] test episode 57: reward = 267.46, steps = 471
    05:24:14 [INFO] test episode 58: reward = 212.02, steps = 419
    05:24:27 [INFO] test episode 59: reward = 253.15, steps = 283
    05:24:54 [INFO] test episode 60: reward = 205.08, steps = 564
    05:25:25 [INFO] test episode 61: reward = 186.44, steps = 646
    05:25:51 [INFO] test episode 62: reward = 206.79, steps = 563
    05:26:16 [INFO] test episode 63: reward = 222.43, steps = 520
    05:26:44 [INFO] test episode 64: reward = 223.27, steps = 598
    05:27:09 [INFO] test episode 65: reward = 231.11, steps = 531
    05:27:45 [INFO] test episode 66: reward = 173.36, steps = 751
    05:28:09 [INFO] test episode 67: reward = 225.83, steps = 500
    05:28:25 [INFO] test episode 68: reward = 267.21, steps = 349
    05:28:47 [INFO] test episode 69: reward = 223.80, steps = 455
    05:29:02 [INFO] test episode 70: reward = 226.08, steps = 305
    05:29:21 [INFO] test episode 71: reward = 231.24, steps = 404
    05:29:39 [INFO] test episode 72: reward = 237.29, steps = 380
    05:30:03 [INFO] test episode 73: reward = 219.17, steps = 503
    05:30:17 [INFO] test episode 74: reward = 243.15, steps = 287
    05:30:35 [INFO] test episode 75: reward = 259.16, steps = 370
    05:30:53 [INFO] test episode 76: reward = 276.34, steps = 367
    05:31:06 [INFO] test episode 77: reward = 268.55, steps = 291
    05:31:27 [INFO] test episode 78: reward = 232.35, steps = 426
    05:31:50 [INFO] test episode 79: reward = 211.01, steps = 484
    05:32:05 [INFO] test episode 80: reward = 233.55, steps = 309
    05:32:30 [INFO] test episode 81: reward = 198.27, steps = 528
    05:32:58 [INFO] test episode 82: reward = 223.30, steps = 568
    05:33:11 [INFO] test episode 83: reward = 257.53, steps = 275
    05:33:45 [INFO] test episode 84: reward = 192.43, steps = 698
    05:34:05 [INFO] test episode 85: reward = 207.57, steps = 421
    05:34:19 [INFO] test episode 86: reward = 243.29, steps = 288
    05:34:43 [INFO] test episode 87: reward = 217.84, steps = 489
    05:34:57 [INFO] test episode 88: reward = 233.11, steps = 294
    05:35:12 [INFO] test episode 89: reward = 218.75, steps = 313
    05:35:40 [INFO] test episode 90: reward = 233.77, steps = 595
    05:35:58 [INFO] test episode 91: reward = 190.85, steps = 364
    05:36:13 [INFO] test episode 92: reward = 240.17, steps = 314
    05:36:42 [INFO] test episode 93: reward = 188.19, steps = 589
    05:36:55 [INFO] test episode 94: reward = 281.84, steps = 269
    05:37:08 [INFO] test episode 95: reward = 14.88, steps = 275
    05:37:39 [INFO] test episode 96: reward = 220.79, steps = 642
    05:37:59 [INFO] test episode 97: reward = 249.71, steps = 414
    05:38:26 [INFO] test episode 98: reward = 213.70, steps = 567
    05:39:03 [INFO] test episode 99: reward = 196.80, steps = 768
    05:39:03 [INFO] average episode reward = 222.72 ± 50.43

In [6]:

    env.close()