{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Parameter Heatmap\n",
    "==========\n",
    "\n",
    "This tutorial will show how to optimize strategies with multiple parameters and how to examine and reason about optimization results.\n",
    "It is assumed you're already familiar with\n",
    "[basic _backtesting.py_ usage](https://kernc.github.io/backtesting.py/doc/examples/Quick%20Start%20User%20Guide.html).\n",
    "\n",
    "First, let's again import our helper moving average function.\n",
    "In practice, one should use functions from an indicator library, such as\n",
    "[TA-Lib](https://github.com/mrjbq7/ta-lib) or\n",
    "[Tulipy](https://tulipindicators.org)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"1001\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id != null && id in Bokeh.index) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var id = msg.content.text.trim();\n",
       "            if (id in Bokeh.index) {\n",
       "              Bokeh.index[id].model.document.clear();\n",
       "              delete Bokeh.index[id];\n",
       "            }\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "        toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[toinsert.length - 1]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"1001\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
       "        if (callback != null)\n",
       "          callback();\n",
       "      });\n",
       "    } finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.debug(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(css_urls, js_urls, callback) {\n",
       "    if (css_urls == null) css_urls = [];\n",
       "    if (js_urls == null) js_urls = [];\n",
       "\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
       "\n",
       "    function on_load() {\n",
       "      root._bokeh_is_loading--;\n",
       "      if (root._bokeh_is_loading === 0) {\n",
       "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
       "        run_callbacks()\n",
       "      }\n",
       "    }\n",
       "\n",
       "    function on_error() {\n",
       "      console.error(\"failed to load \" + url);\n",
       "    }\n",
       "\n",
       "    for (var i = 0; i < css_urls.length; i++) {\n",
       "      var url = css_urls[i];\n",
       "      const element = document.createElement(\"link\");\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.rel = \"stylesheet\";\n",
       "      element.type = \"text/css\";\n",
       "      element.href = url;\n",
       "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
       "      document.body.appendChild(element);\n",
       "    }\n",
       "\n",
       "    const hashes = {\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js\": \"qkRvDQVAIfzsJo40iRBbxt6sttt0hv4lh74DG7OK4MCHv4C5oohXYoHUM5W11uqS\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.1.min.js\": \"Sb7Mr06a9TNlet/GEBeKaf5xH3eb6AlCzwjtU82wNPyDrnfoiVl26qnvlKjmcAd+\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.1.min.js\": \"HaJ15vgfmcfRtB4c4YBOI4f1MUujukqInOWVqZJZZGK7Q+ivud0OKGSTn/Vm2iso\"};\n",
       "\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var element = document.createElement('script');\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.async = false;\n",
       "      element.src = url;\n",
       "      if (url in hashes) {\n",
       "        element.crossOrigin = \"anonymous\";\n",
       "        element.integrity = \"sha384-\" + hashes[url];\n",
       "      }\n",
       "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.head.appendChild(element);\n",
       "    }\n",
       "  };\n",
       "\n",
       "  function inject_raw_css(css) {\n",
       "    const element = document.createElement(\"style\");\n",
       "    element.appendChild(document.createTextNode(css));\n",
       "    document.body.appendChild(element);\n",
       "  }\n",
       "\n",
       "  \n",
       "  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.1.min.js\"];\n",
       "  var css_urls = [];\n",
       "  \n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    function(Bokeh) {\n",
       "    \n",
       "    \n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if (root.Bokeh !== undefined || force === true) {\n",
       "      \n",
       "    for (var i = 0; i < inline_js.length; i++) {\n",
       "      inline_js[i].call(root, root.Bokeh);\n",
       "    }\n",
       "    if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(css_urls, js_urls, function() {\n",
       "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"1001\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    const hashes = {\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js\": \"qkRvDQVAIfzsJo40iRBbxt6sttt0hv4lh74DG7OK4MCHv4C5oohXYoHUM5W11uqS\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.1.min.js\": \"Sb7Mr06a9TNlet/GEBeKaf5xH3eb6AlCzwjtU82wNPyDrnfoiVl26qnvlKjmcAd+\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.1.min.js\": \"HaJ15vgfmcfRtB4c4YBOI4f1MUujukqInOWVqZJZZGK7Q+ivud0OKGSTn/Vm2iso\"};\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      if (url in hashes) {\n        element.crossOrigin = \"anonymous\";\n        element.integrity = \"sha384-\" + hashes[url];\n      }\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  \n  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.1.min.js\"];\n  var css_urls = [];\n  \n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    function(Bokeh) {\n    \n    \n    }\n  ];\n\n  function run_inline_js() {\n    \n    if (root.Bokeh !== undefined || force === true) {\n      \n    for (var i = 0; i < inline_js.length; i++) {\n      inline_js[i].call(root, root.Bokeh);\n    }\n    if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from backtesting.test import SMA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our strategy will be a similar moving average cross-over strategy to the one in\n",
    "[Quick Start User Guide](https://kernc.github.io/backtesting.py/doc/examples/Quick%20Start%20User%20Guide.html),\n",
    "but we will use four moving averages in total:\n",
    "two moving averages whose relationship determines a general trend\n",
    "(we only trade long when the shorter MA is above the longer one, and vice versa),\n",
    "and two moving averages whose cross-over with daily _close_ prices determine the signal to enter or exit the position."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from backtesting import Strategy\n",
    "from backtesting.lib import crossover\n",
    "\n",
    "\n",
    "class Sma4Cross(Strategy):\n",
    "    n1 = 50\n",
    "    n2 = 100\n",
    "    n_enter = 20\n",
    "    n_exit = 10\n",
    "    \n",
    "    def init(self):\n",
    "        self.sma1 = self.I(SMA, self.data.Close, self.n1)\n",
    "        self.sma2 = self.I(SMA, self.data.Close, self.n2)\n",
    "        self.sma_enter = self.I(SMA, self.data.Close, self.n_enter)\n",
    "        self.sma_exit = self.I(SMA, self.data.Close, self.n_exit)\n",
    "        \n",
    "    def next(self):\n",
    "        \n",
    "        if not self.position:\n",
    "            \n",
    "            # On upwards trend, if price closes above\n",
    "            # \"entry\" MA, go long\n",
    "            \n",
    "            # Here, even though the operands are arrays, this\n",
    "            # works by implicitly comparing the two last values\n",
    "            if self.sma1 > self.sma2:\n",
    "                if crossover(self.data.Close, self.sma_enter):\n",
    "                    self.buy()\n",
    "                    \n",
    "            # On downwards trend, if price closes below\n",
    "            # \"entry\" MA, go short\n",
    "            \n",
    "            else:\n",
    "                if crossover(self.sma_enter, self.data.Close):\n",
    "                    self.sell()\n",
    "        \n",
    "        # But if we already hold a position and the price\n",
    "        # closes back below (above) \"exit\" MA, close the position\n",
    "        \n",
    "        else:\n",
    "            if (self.position.is_long and\n",
    "                crossover(self.sma_exit, self.data.Close)\n",
    "                or\n",
    "                self.position.is_short and\n",
    "                crossover(self.data.Close, self.sma_exit)):\n",
    "                \n",
    "                self.position.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It's not a robust strategy, but we can optimize it.\n",
    "\n",
    "[Grid search](https://en.wikipedia.org/wiki/Hyperparameter_optimization#Grid_search)\n",
    "is an exhaustive search through a set of specified sets of values of hyperparameters. One evaluates the performance for each set of parameters and finally selects the combination that performs best.\n",
    "\n",
    "Let's optimize our strategy on Google stock data using _randomized_ grid search over the parameter space, evaluating at most (approximately) 200 randomly chosen combinations:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=9.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 322 ms, sys: 42 ms, total: 364 ms\n",
      "Wall time: 6.27 s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "\n",
    "from backtesting import Backtest\n",
    "from backtesting.test import GOOG\n",
    "\n",
    "\n",
    "backtest = Backtest(GOOG, Sma4Cross, commission=.002)\n",
    "\n",
    "stats, heatmap = backtest.optimize(\n",
    "    n1=range(10, 110, 10),\n",
    "    n2=range(20, 210, 20),\n",
    "    n_enter=range(15, 35, 5),\n",
    "    n_exit=range(10, 25, 5),\n",
    "    constraint=lambda p: p.n_exit < p.n_enter < p.n1 < p.n2,\n",
    "    maximize='Equity Final [$]',\n",
    "    max_tries=200,\n",
    "    random_state=0,\n",
    "    return_heatmap=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice `return_heatmap=True` parameter passed to\n",
    "[`Backtest.optimize()`](https://kernc.github.io/backtesting.py/doc/backtesting/backtesting.html#backtesting.backtesting.Backtest.optimize).\n",
    "It makes the function return a heatmap series along with the usual stats of the best run.\n",
    "`heatmap` is a pandas Series indexed with a MultiIndex, a cartesian product of all permissible (tried) parameter values.\n",
    "The series values are from the `maximize=` argument we provided."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "n1   n2   n_enter  n_exit\n",
       "30   40   15       10       10611.01\n",
       "          25       20        9006.74\n",
       "     60   15       10       11297.19\n",
       "          20       10       11539.78\n",
       "     80   25       10       11217.25\n",
       "                              ...   \n",
       "100  180  20       10       10711.02\n",
       "     200  25       10        8991.55\n",
       "                   15        9549.19\n",
       "                   20        8241.72\n",
       "          30       15        9701.12\n",
       "Name: Equity Final [$], Length: 205, dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heatmap"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This heatmap contains the results of all the runs,\n",
    "making it very easy to obtain parameter combinations for e.g. three best runs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "n1   n2   n_enter  n_exit\n",
       "40   60   20       15       20019.78\n",
       "100  120  20       15       20545.03\n",
       "40   60   25       15       21014.63\n",
       "Name: Equity Final [$], dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heatmap.sort_values().iloc[-3:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "But we use vision to make judgements on larger data sets much faster.\n",
    "Let's plot the whole heatmap by projecting it on two chosen dimensions.\n",
    "Say we're mostly interested in how parameters `n1` and `n2`, on average, affect the outcome."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>n2</th>\n",
       "      <th>40</th>\n",
       "      <th>60</th>\n",
       "      <th>80</th>\n",
       "      <th>100</th>\n",
       "      <th>120</th>\n",
       "      <th>140</th>\n",
       "      <th>160</th>\n",
       "      <th>180</th>\n",
       "      <th>200</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n1</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>9808.88</td>\n",
       "      <td>11418.49</td>\n",
       "      <td>12183.57</td>\n",
       "      <td>10970.01</td>\n",
       "      <td>9823.28</td>\n",
       "      <td>10634.02</td>\n",
       "      <td>12761.29</td>\n",
       "      <td>11129.58</td>\n",
       "      <td>10818.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>nan</td>\n",
       "      <td>15237.61</td>\n",
       "      <td>11123.54</td>\n",
       "      <td>8036.80</td>\n",
       "      <td>12899.93</td>\n",
       "      <td>12606.21</td>\n",
       "      <td>12108.13</td>\n",
       "      <td>nan</td>\n",
       "      <td>9707.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>nan</td>\n",
       "      <td>6614.39</td>\n",
       "      <td>10249.40</td>\n",
       "      <td>10759.11</td>\n",
       "      <td>10741.92</td>\n",
       "      <td>11369.56</td>\n",
       "      <td>10508.96</td>\n",
       "      <td>11118.57</td>\n",
       "      <td>10282.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>6949.90</td>\n",
       "      <td>11278.54</td>\n",
       "      <td>9855.09</td>\n",
       "      <td>11674.47</td>\n",
       "      <td>12361.64</td>\n",
       "      <td>9277.57</td>\n",
       "      <td>9520.61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>13241.52</td>\n",
       "      <td>8630.84</td>\n",
       "      <td>9447.04</td>\n",
       "      <td>9647.08</td>\n",
       "      <td>10854.71</td>\n",
       "      <td>9298.65</td>\n",
       "      <td>8120.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>8312.86</td>\n",
       "      <td>9165.66</td>\n",
       "      <td>10169.51</td>\n",
       "      <td>11992.31</td>\n",
       "      <td>10748.77</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>11454.72</td>\n",
       "      <td>12570.32</td>\n",
       "      <td>10656.17</td>\n",
       "      <td>12041.58</td>\n",
       "      <td>10927.90</td>\n",
       "      <td>9396.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>12372.87</td>\n",
       "      <td>10093.88</td>\n",
       "      <td>11733.34</td>\n",
       "      <td>10711.02</td>\n",
       "      <td>9120.90</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "n2      40       60       80       100      120      140      160      180  \\\n",
       "n1                                                                           \n",
       "30  9808.88 11418.49 12183.57 10970.01  9823.28 10634.02 12761.29 11129.58   \n",
       "40      nan 15237.61 11123.54  8036.80 12899.93 12606.21 12108.13      nan   \n",
       "50      nan  6614.39 10249.40 10759.11 10741.92 11369.56 10508.96 11118.57   \n",
       "60      nan      nan  6949.90 11278.54  9855.09 11674.47 12361.64  9277.57   \n",
       "70      nan      nan 13241.52  8630.84  9447.04  9647.08 10854.71  9298.65   \n",
       "80      nan      nan      nan  8312.86  9165.66 10169.51 11992.31 10748.77   \n",
       "90      nan      nan      nan 11454.72 12570.32 10656.17 12041.58 10927.90   \n",
       "100     nan      nan      nan      nan 12372.87 10093.88 11733.34 10711.02   \n",
       "\n",
       "n2       200  \n",
       "n1            \n",
       "30  10818.95  \n",
       "40   9707.38  \n",
       "50  10282.21  \n",
       "60   9520.61  \n",
       "70   8120.51  \n",
       "80       nan  \n",
       "90   9396.30  \n",
       "100  9120.90  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hm = heatmap.groupby(['n1', 'n2']).mean().unstack()\n",
    "hm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's plot this table using the excellent [_Seaborn_](https://seaborn.pydata.org) package:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='n2', ylabel='n1'>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 12376 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "sns.heatmap(hm[::-1], cmap='viridis')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We see that, on average, we obtain the highest result using trend-determining parameters `n1=40` and `n2=60`,\n",
    "and it's not like other nearby combinations work similarly well — in our particular strategy, this combination really stands out.\n",
    "\n",
    "Since our strategy contains several parameters, we might be interested in other relationships between their values.\n",
    "We can use\n",
    "[`backtesting.lib.plot_heatmaps()`](https://kernc.github.io/backtesting.py/doc/backtesting/lib.html#backtesting.lib.plot_heatmaps)\n",
    "function to plot interactive heatmaps of all parameter combinations simultaneously."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "  <div class=\"bk-root\" id=\"841efc58-5cec-4744-9269-0cfa6253a687\" data-root-id=\"1252\"></div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"c468d9bb-e7c5-4dcb-873b-1bd0d8397e1e\":{\"roots\":{\"references\":[{\"attributes\":{\"children\":[{\"id\":\"1251\"},{\"id\":\"1249\"}]},\"id\":\"1252\",\"type\":\"Column\"},{\"attributes\":{},\"id\":\"1205\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1206\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1213\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1215\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1216\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"text\":\"\"},\"id\":\"1220\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1221\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1223\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1225\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1226\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"text\":\"\"},\"id\":\"1230\",\"type\":\"Title\"},{\"attributes\":{\"factors\":[\"15\",\"20\",\"25\",\"30\"]},\"id\":\"1159\",\"type\":\"FactorRange\"},{\"attributes\":{\"factors\":[\"10\",\"15\",\"20\"]},\"id\":\"1161\",\"type\":\"FactorRange\"},{\"attributes\":{},\"id\":\"1231\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1163\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1233\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1165\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1235\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1103\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"factors\":[\"30\",\"40\",\"50\",\"60\",\"70\",\"80\",\"90\",\"100\"]},\"id\":\"1035\",\"type\":\"FactorRange\"},{\"attributes\":{\"axis_label\":\"n_enter\",\"axis_line_color\":null,\"formatter\":{\"id\":\"1241\"},\"major_label_standoff\":0,\"major_tick_line_color\":null,\"ticker\":{\"id\":\"1168\"}},\"id\":\"1167\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1236\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"text\":\"\"},\"id\":\"1240\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1168\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"factors\":[\"15\",\"20\",\"25\",\"30\"]},\"id\":\"1037\",\"type\":\"FactorRange\"},{\"attributes\":{\"axis\":{\"id\":\"1167\"},\"grid_line_color\":null,\"ticker\":null},\"id\":\"1169\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1241\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"overlay\":{\"id\":\"1114\"}},\"id\":\"1111\",\"type\":\"BoxZoomTool\"},{\"attributes\":{},\"id\":\"1039\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"axis_label\":\"n_exit\",\"axis_line_color\":null,\"formatter\":{\"id\":\"1243\"},\"major_label_standoff\":0,\"major_tick_line_color\":null,\"ticker\":{\"id\":\"1171\"}},\"id\":\"1170\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1243\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1041\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1171\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1245\",\"type\":\"Selection\"},{\"attributes\":{\"axis_label\":\"n1\",\"axis_line_color\":null,\"formatter\":{\"id\":\"1201\"},\"major_label_standoff\":0,\"major_tick_line_color\":null,\"ticker\":{\"id\":\"1044\"}},\"id\":\"1043\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"axis\":{\"id\":\"1170\"},\"dimension\":1,\"grid_line_color\":null,\"ticker\":null},\"id\":\"1172\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1246\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1044\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"logo\":null,\"toolbars\":[{\"id\":\"1023\"},{\"id\":\"1054\"},{\"id\":\"1085\"},{\"id\":\"1116\"},{\"id\":\"1147\"},{\"id\":\"1178\"}],\"tools\":[{\"id\":\"1018\"},{\"id\":\"1019\"},{\"id\":\"1020\"},{\"id\":\"1022\"},{\"id\":\"1049\"},{\"id\":\"1050\"},{\"id\":\"1051\"},{\"id\":\"1053\"},{\"id\":\"1080\"},{\"id\":\"1081\"},{\"id\":\"1082\"},{\"id\":\"1084\"},{\"id\":\"1111\"},{\"id\":\"1112\"},{\"id\":\"1113\"},{\"id\":\"1115\"},{\"id\":\"1142\"},{\"id\":\"1143\"},{\"id\":\"1144\"},{\"id\":\"1146\"},{\"id\":\"1173\"},{\"id\":\"1174\"},{\"id\":\"1175\"},{\"id\":\"1177\"}]},\"id\":\"1250\",\"type\":\"ProxyToolbar\"},{\"attributes\":{\"data_source\":{\"id\":\"1183\"},\"glyph\":{\"id\":\"1185\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1186\"},\"selection_glyph\":null,\"view\":{\"id\":\"1188\"}},\"id\":\"1187\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"axis\":{\"id\":\"1043\"},\"grid_line_color\":null,\"ticker\":null},\"id\":\"1045\",\"type\":\"Grid\"},{\"attributes\":{\"fill_color\":{\"field\":\"_Value\",\"transform\":{\"id\":\"1002\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"n_enter\"},\"y\":{\"field\":\"n_exit\"}},\"id\":\"1185\",\"type\":\"Rect\"},{\"attributes\":{\"toolbar\":{\"id\":\"1250\"},\"toolbar_location\":\"above\"},\"id\":\"1251\",\"type\":\"ToolbarBox\"},{\"attributes\":{\"overlay\":{\"id\":\"1176\"}},\"id\":\"1173\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"axis_label\":\"n_enter\",\"axis_line_color\":null,\"formatter\":{\"id\":\"1203\"},\"major_label_standoff\":0,\"major_tick_line_color\":null,\"ticker\":{\"id\":\"1047\"}},\"id\":\"1046\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1174\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"1047\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1175\",\"type\":\"SaveTool\"},{\"attributes\":{\"axis\":{\"id\":\"1046\"},\"dimension\":1,\"grid_line_color\":null,\"ticker\":null},\"id\":\"1048\",\"type\":\"Grid\"},{\"attributes\":{\"data\":{\"_Value\":{\"__ndarray__\":\"4k+Nlya2xEBHOkRew+PLQN6mP/uxnMJAI2yx21duykDrU6GUm/7KQDoVqTBeX8dAfFi+kII4wkBrg0wyEK7GQPlEWfzvdstAtLQEt5nUwkDcTbfsqCjBQOerViZYkctArnYU5wAWzECSAzX6TGrBQBz1q8Ksnb9AbEeXJKwOxUAe9RCNkGrKQKMzPer9OsFA5fDKI+hcwUAIb4EEAf7LQLbbLjSPVMpAFTeB/KcCvkCWzzdlJe3BQOPjTBNe5cxALk1KQQ8/z0Cxxtl0EDjAQFxZaVJLhsFA9Z2Y9ZJsyUD7823B+srPQHQ486txXsBAkoHK+HdCwUA=\",\"dtype\":\"float64\",\"order\":\"little\",\"shape\":[31]},\"index\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30],\"n1\":[\"30\",\"30\",\"30\",\"40\",\"40\",\"40\",\"40\",\"50\",\"50\",\"50\",\"50\",\"60\",\"60\",\"60\",\"60\",\"70\",\"70\",\"70\",\"70\",\"80\",\"80\",\"80\",\"80\",\"90\",\"90\",\"90\",\"90\",\"100\",\"100\",\"100\",\"100\"],\"n_enter\":[\"15\",\"20\",\"25\",\"15\",\"20\",\"25\",\"30\",\"15\",\"20\",\"25\",\"30\",\"15\",\"20\",\"25\",\"30\",\"15\",\"20\",\"25\",\"30\",\"15\",\"20\",\"25\",\"30\",\"15\",\"20\",\"25\",\"30\",\"15\",\"20\",\"25\",\"30\"]},\"selected\":{\"id\":\"1205\"},\"selection_policy\":{\"id\":\"1206\"}},\"id\":\"1059\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"text\":\"\"},\"id\":\"1190\",\"type\":\"Title\"},{\"attributes\":{\"axis_label\":\"n2\",\"axis_line_color\":null,\"formatter\":{\"id\":\"1221\"},\"major_label_standoff\":0,\"major_tick_line_color\":null,\"ticker\":{\"id\":\"1106\"}},\"id\":\"1105\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"overlay\":{\"id\":\"1052\"}},\"id\":\"1049\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":0.5,\"fill_color\":\"lightgrey\",\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":1.0,\"line_color\":\"black\",\"line_dash\":[4,4],\"line_width\":2,\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"1176\",\"type\":\"BoxAnnotation\"},{\"attributes\":{},\"id\":\"1050\",\"type\":\"ResetTool\"},{\"attributes\":{\"source\":{\"id\":\"1183\"}},\"id\":\"1188\",\"type\":\"CDSView\"},{\"attributes\":{\"below\":[{\"id\":\"1012\"}],\"center\":[{\"id\":\"1014\"},{\"id\":\"1017\"}],\"left\":[{\"id\":\"1015\"}],\"plot_height\":400,\"plot_width\":400,\"renderers\":[{\"id\":\"1032\"}],\"title\":{\"id\":\"1190\"},\"toolbar\":{\"id\":\"1023\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"1004\"},\"x_scale\":{\"id\":\"1008\"},\"y_range\":{\"id\":\"1006\"},\"y_scale\":{\"id\":\"1010\"}},\"id\":\"1003\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1051\",\"type\":\"SaveTool\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"field\":\"_Value\",\"transform\":{\"id\":\"1002\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"n_enter\"},\"y\":{\"field\":\"n_exit\"}},\"id\":\"1186\",\"type\":\"Rect\"},{\"attributes\":{\"data\":{\"_Value\":{\"__ndarray__\":\"1uJTAHAow0Dob0IhPk3GQEDjwoHIy8dA4lgXtwFtxUCTvoI0oy/DQA7yQc8CxcRAdAexM6XsyED/5AKqyrzFQLUfKSJ5IcVAIk31ZM7CzUBD9JGBxbnFQB1sNmnNZL9A6revA/cxyUA9232hGp/IQPtahE8QpsdABskCJrD1wkDEounsZNa5QHUvCpWyBMRAlkx0g44DxUB6vfvj9frEQINTMhPHNMZAjOtRuHqGxECQmfiQSbfFQFDovMYaFcRAhKmfN+Ulu0Bdh9wMRQfGQP7UeOmKP8NA501FKjzNxkAmrfiG0iTIQN/Mr+bIHsJAnjuInU2YwkBip5GWwtzJQJyZmZlr28BAh2Uzh4RzwkBecTjzidfCQEix3NJaM8VAvjkjSlMpwkDav3lxgri/QDCGcqJtPMBAB1slWNTmwUCfGi/dwNzDQOUZNPQnbMdALzMzM2P+xEAwyJblW1/GQECPPtMojchAYygn2hXQxEDjK6cqyoTHQCC+E7PyV8VA8kxC/CZawkBlhx4ebyrIQDKP/MHwtsNAmKjeGqzqxkAOhGQBg+vEQJmiI7ly0MFA\",\"dtype\":\"float64\",\"order\":\"little\",\"shape\":[54]},\"index\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53],\"n1\":[\"30\",\"30\",\"30\",\"30\",\"30\",\"30\",\"30\",\"30\",\"30\",\"40\",\"40\",\"40\",\"40\",\"40\",\"40\",\"40\",\"50\",\"50\",\"50\",\"50\",\"50\",\"50\",\"50\",\"50\",\"60\",\"60\",\"60\",\"60\",\"60\",\"60\",\"60\",\"70\",\"70\",\"70\",\"70\",\"70\",\"70\",\"70\",\"80\",\"80\",\"80\",\"80\",\"80\",\"90\",\"90\",\"90\",\"90\",\"90\",\"90\",\"100\",\"100\",\"100\",\"100\",\"100\"],\"n2\":[\"40\",\"60\",\"80\",\"100\",\"120\",\"140\",\"160\",\"180\",\"200\",\"60\",\"80\",\"100\",\"120\",\"140\",\"160\",\"200\",\"60\",\"80\",\"100\",\"120\",\"140\",\"160\",\"180\",\"200\",\"80\",\"100\",\"120\",\"140\",\"160\",\"180\",\"200\",\"80\",\"100\",\"120\",\"140\",\"160\",\"180\",\"200\",\"100\",\"120\",\"140\",\"160\",\"180\",\"100\",\"120\",\"140\",\"160\",\"180\",\"200\",\"120\",\"140\",\"160\",\"180\",\"200\"]},\"selected\":{\"id\":\"1195\"},\"selection_policy\":{\"id\":\"1196\"}},\"id\":\"1028\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1106\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"factors\":[\"30\",\"40\",\"50\",\"60\",\"70\",\"80\",\"90\",\"100\"]},\"id\":\"1004\",\"type\":\"FactorRange\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"n1\",\"@n1\"],[\"n_enter\",\"@n_enter\"],[\"Value\",\"@_Value{0.[000]}\"]]},\"id\":\"1053\",\"type\":\"HoverTool\"},{\"attributes\":{},\"id\":\"1191\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"factors\":[\"40\",\"60\",\"80\",\"100\",\"120\",\"140\",\"160\",\"180\",\"200\"]},\"id\":\"1006\",\"type\":\"FactorRange\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":0.5,\"fill_color\":\"lightgrey\",\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":1.0,\"line_color\":\"black\",\"line_dash\":[4,4],\"line_width\":2,\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"1052\",\"type\":\"BoxAnnotation\"},{\"attributes\":{},\"id\":\"1008\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"fill_color\":{\"field\":\"_Value\",\"transform\":{\"id\":\"1002\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"n1\"},\"y\":{\"field\":\"n_enter\"}},\"id\":\"1061\",\"type\":\"Rect\"},{\"attributes\":{},\"id\":\"1195\",\"type\":\"Selection\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1049\"},{\"id\":\"1050\"},{\"id\":\"1051\"},{\"id\":\"1053\"}]},\"id\":\"1054\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1010\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1196\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"axis_label\":\"n1\",\"axis_line_color\":null,\"formatter\":{\"id\":\"1191\"},\"major_label_standoff\":0,\"major_tick_line_color\":null,\"ticker\":{\"id\":\"1013\"}},\"id\":\"1012\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1201\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1013\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"field\":\"_Value\",\"transform\":{\"id\":\"1002\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"n1\"},\"y\":{\"field\":\"n_enter\"}},\"id\":\"1062\",\"type\":\"Rect\"},{\"attributes\":{},\"id\":\"1203\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1112\",\"type\":\"ResetTool\"},{\"attributes\":{\"data_source\":{\"id\":\"1059\"},\"glyph\":{\"id\":\"1061\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1062\"},\"selection_glyph\":null,\"view\":{\"id\":\"1064\"}},\"id\":\"1063\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"factors\":[\"10\",\"15\",\"20\"]},\"id\":\"1068\",\"type\":\"FactorRange\"},{\"attributes\":{\"axis\":{\"id\":\"1012\"},\"grid_line_color\":null,\"ticker\":null},\"id\":\"1014\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1113\",\"type\":\"SaveTool\"},{\"attributes\":{},\"id\":\"1070\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"source\":{\"id\":\"1059\"}},\"id\":\"1064\",\"type\":\"CDSView\"},{\"attributes\":{\"axis_label\":\"n2\",\"axis_line_color\":null,\"formatter\":{\"id\":\"1193\"},\"major_label_standoff\":0,\"major_tick_line_color\":null,\"ticker\":{\"id\":\"1016\"}},\"id\":\"1015\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1072\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1211\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1016\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"n2\",\"@n2\"],[\"n_enter\",\"@n_enter\"],[\"Value\",\"@_Value{0.[000]}\"]]},\"id\":\"1115\",\"type\":\"HoverTool\"},{\"attributes\":{\"axis_label\":\"n1\",\"axis_line_color\":null,\"formatter\":{\"id\":\"1211\"},\"major_label_standoff\":0,\"major_tick_line_color\":null,\"ticker\":{\"id\":\"1075\"}},\"id\":\"1074\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"axis\":{\"id\":\"1015\"},\"dimension\":1,\"grid_line_color\":null,\"ticker\":null},\"id\":\"1017\",\"type\":\"Grid\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":0.5,\"fill_color\":\"lightgrey\",\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":1.0,\"line_color\":\"black\",\"line_dash\":[4,4],\"line_width\":2,\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"1114\",\"type\":\"BoxAnnotation\"},{\"attributes\":{},\"id\":\"1075\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1111\"},{\"id\":\"1112\"},{\"id\":\"1113\"},{\"id\":\"1115\"}]},\"id\":\"1116\",\"type\":\"Toolbar\"},{\"attributes\":{\"axis\":{\"id\":\"1074\"},\"grid_line_color\":null,\"ticker\":null},\"id\":\"1076\",\"type\":\"Grid\"},{\"attributes\":{\"source\":{\"id\":\"1028\"}},\"id\":\"1033\",\"type\":\"CDSView\"},{\"attributes\":{\"overlay\":{\"id\":\"1021\"}},\"id\":\"1018\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"axis_label\":\"n_exit\",\"axis_line_color\":null,\"formatter\":{\"id\":\"1213\"},\"major_label_standoff\":0,\"major_tick_line_color\":null,\"ticker\":{\"id\":\"1078\"}},\"id\":\"1077\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"field\":\"_Value\",\"transform\":{\"id\":\"1002\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"n1\"},\"y\":{\"field\":\"n2\"}},\"id\":\"1031\",\"type\":\"Rect\"},{\"attributes\":{\"below\":[{\"id\":\"1136\"}],\"center\":[{\"id\":\"1138\"},{\"id\":\"1141\"}],\"left\":[{\"id\":\"1139\"}],\"plot_height\":400,\"plot_width\":400,\"renderers\":[{\"id\":\"1156\"}],\"title\":{\"id\":\"1230\"},\"toolbar\":{\"id\":\"1147\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"1128\"},\"x_scale\":{\"id\":\"1132\"},\"y_range\":{\"id\":\"1130\"},\"y_scale\":{\"id\":\"1134\"}},\"id\":\"1127\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1019\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"1078\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"data_source\":{\"id\":\"1121\"},\"glyph\":{\"id\":\"1123\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1124\"},\"selection_glyph\":null,\"view\":{\"id\":\"1126\"}},\"id\":\"1125\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"1020\",\"type\":\"SaveTool\"},{\"attributes\":{\"axis\":{\"id\":\"1077\"},\"dimension\":1,\"grid_line_color\":null,\"ticker\":null},\"id\":\"1079\",\"type\":\"Grid\"},{\"attributes\":{\"fill_color\":{\"field\":\"_Value\",\"transform\":{\"id\":\"1002\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"n2\"},\"y\":{\"field\":\"n_enter\"}},\"id\":\"1123\",\"type\":\"Rect\"},{\"attributes\":{\"factors\":[\"40\",\"60\",\"80\",\"100\",\"120\",\"140\",\"160\",\"180\",\"200\"]},\"id\":\"1128\",\"type\":\"FactorRange\"},{\"attributes\":{\"factors\":[\"15\",\"20\",\"25\",\"30\"]},\"id\":\"1099\",\"type\":\"FactorRange\"},{\"attributes\":{\"source\":{\"id\":\"1121\"}},\"id\":\"1126\",\"type\":\"CDSView\"},{\"attributes\":{\"factors\":[\"10\",\"15\",\"20\"]},\"id\":\"1130\",\"type\":\"FactorRange\"},{\"attributes\":{},\"id\":\"1101\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"field\":\"_Value\",\"transform\":{\"id\":\"1002\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"n2\"},\"y\":{\"field\":\"n_enter\"}},\"id\":\"1124\",\"type\":\"Rect\"},{\"attributes\":{\"overlay\":{\"id\":\"1083\"}},\"id\":\"1080\",\"type\":\"BoxZoomTool\"},{\"attributes\":{},\"id\":\"1132\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":0.5,\"fill_color\":\"lightgrey\",\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":1.0,\"line_color\":\"black\",\"line_dash\":[4,4],\"line_width\":2,\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"1021\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1018\"},{\"id\":\"1019\"},{\"id\":\"1020\"},{\"id\":\"1022\"}]},\"id\":\"1023\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1081\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"1134\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"fill_color\":{\"field\":\"_Value\",\"transform\":{\"id\":\"1002\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"n1\"},\"y\":{\"field\":\"n2\"}},\"id\":\"1030\",\"type\":\"Rect\"},{\"attributes\":{},\"id\":\"1082\",\"type\":\"SaveTool\"},{\"attributes\":{\"axis_label\":\"n2\",\"axis_line_color\":null,\"formatter\":{\"id\":\"1231\"},\"major_label_standoff\":0,\"major_tick_line_color\":null,\"ticker\":{\"id\":\"1137\"}},\"id\":\"1136\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1137\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"n1\",\"@n1\"],[\"n_exit\",\"@n_exit\"],[\"Value\",\"@_Value{0.[000]}\"]]},\"id\":\"1084\",\"type\":\"HoverTool\"},{\"attributes\":{\"axis\":{\"id\":\"1136\"},\"grid_line_color\":null,\"ticker\":null},\"id\":\"1138\",\"type\":\"Grid\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":0.5,\"fill_color\":\"lightgrey\",\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":1.0,\"line_color\":\"black\",\"line_dash\":[4,4],\"line_width\":2,\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"1083\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"axis_label\":\"n_exit\",\"axis_line_color\":null,\"formatter\":{\"id\":\"1233\"},\"major_label_standoff\":0,\"major_tick_line_color\":null,\"ticker\":{\"id\":\"1140\"}},\"id\":\"1139\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1080\"},{\"id\":\"1081\"},{\"id\":\"1082\"},{\"id\":\"1084\"}]},\"id\":\"1085\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1140\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"children\":[[{\"id\":\"1003\"},0,0],[{\"id\":\"1034\"},0,1],[{\"id\":\"1065\"},0,2],[{\"id\":\"1096\"},1,0],[{\"id\":\"1127\"},1,1],[{\"id\":\"1158\"},1,2]]},\"id\":\"1249\",\"type\":\"GridBox\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"n1\",\"@n1\"],[\"n2\",\"@n2\"],[\"Value\",\"@_Value{0.[000]}\"]]},\"id\":\"1022\",\"type\":\"HoverTool\"},{\"attributes\":{\"axis\":{\"id\":\"1139\"},\"dimension\":1,\"grid_line_color\":null,\"ticker\":null},\"id\":\"1141\",\"type\":\"Grid\"},{\"attributes\":{\"data_source\":{\"id\":\"1090\"},\"glyph\":{\"id\":\"1092\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1093\"},\"selection_glyph\":null,\"view\":{\"id\":\"1095\"}},\"id\":\"1094\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"data\":{\"_Value\":{\"__ndarray__\":\"2bK1voC5xEDSEvJBX5fBQCWsoHzAB8VAfQqA8UwJ1EA1e9l20jvAQBuJj82JUcZA7qzddjFDx0DeSJANGzrDQNeT5nxKFcRATRjNyt4XwkC4rEai/iHFQP14r1pP5cdALviuw73OwkAUjqTQ+MnGQMlkedn3jsVAGeLjxOVmwkDRoDsiyAzHQFcZNaq3PclAjanhSCVDw0CnkQ0OJ9zDQHBy1Z6eU8dAWOz2WXVPwED3F7v/uFXCQOY00lKFW8VAEn9hd61Mv0A=\",\"dtype\":\"float64\",\"order\":\"little\",\"shape\":[25]},\"index\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24],\"n2\":[\"40\",\"40\",\"60\",\"60\",\"80\",\"80\",\"80\",\"100\",\"100\",\"100\",\"120\",\"120\",\"120\",\"140\",\"140\",\"140\",\"160\",\"160\",\"160\",\"180\",\"180\",\"180\",\"200\",\"200\",\"200\"],\"n_exit\":[\"10\",\"20\",\"10\",\"15\",\"10\",\"15\",\"20\",\"10\",\"15\",\"20\",\"10\",\"15\",\"20\",\"10\",\"15\",\"20\",\"10\",\"15\",\"20\",\"10\",\"15\",\"20\",\"10\",\"15\",\"20\"]},\"selected\":{\"id\":\"1235\"},\"selection_policy\":{\"id\":\"1236\"}},\"id\":\"1152\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1173\"},{\"id\":\"1174\"},{\"id\":\"1175\"},{\"id\":\"1177\"}]},\"id\":\"1178\",\"type\":\"Toolbar\"},{\"attributes\":{\"fill_color\":{\"field\":\"_Value\",\"transform\":{\"id\":\"1002\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"n1\"},\"y\":{\"field\":\"n_exit\"}},\"id\":\"1092\",\"type\":\"Rect\"},{\"attributes\":{\"overlay\":{\"id\":\"1145\"}},\"id\":\"1142\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"below\":[{\"id\":\"1043\"}],\"center\":[{\"id\":\"1045\"},{\"id\":\"1048\"}],\"left\":[{\"id\":\"1046\"}],\"plot_height\":400,\"plot_width\":400,\"renderers\":[{\"id\":\"1063\"}],\"title\":{\"id\":\"1200\"},\"toolbar\":{\"id\":\"1054\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"1035\"},\"x_scale\":{\"id\":\"1039\"},\"y_range\":{\"id\":\"1037\"},\"y_scale\":{\"id\":\"1041\"}},\"id\":\"1034\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"source\":{\"id\":\"1090\"}},\"id\":\"1095\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1143\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"1193\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"field\":\"_Value\",\"transform\":{\"id\":\"1002\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"n1\"},\"y\":{\"field\":\"n_exit\"}},\"id\":\"1093\",\"type\":\"Rect\"},{\"attributes\":{},\"id\":\"1144\",\"type\":\"SaveTool\"},{\"attributes\":{\"text\":\"\"},\"id\":\"1200\",\"type\":\"Title\"},{\"attributes\":{\"data\":{\"_Value\":{\"__ndarray__\":\"PCsrdJBYyUC2e/KwBnjIQBNn/CXdAtBAW0mdgCbFvkAmbgL5kGvDQKGmlq1h68JAN3j1SQvfv0As1DvvGIDCQGfZWt/pNMFA\",\"dtype\":\"float64\",\"order\":\"little\",\"shape\":[9]},\"index\":[0,1,2,3,4,5,6,7,8],\"n_enter\":[\"15\",\"20\",\"20\",\"25\",\"25\",\"25\",\"30\",\"30\",\"30\"],\"n_exit\":[\"10\",\"10\",\"15\",\"10\",\"15\",\"20\",\"10\",\"15\",\"20\"]},\"selected\":{\"id\":\"1245\"},\"selection_policy\":{\"id\":\"1246\"}},\"id\":\"1183\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"n2\",\"@n2\"],[\"n_exit\",\"@n_exit\"],[\"Value\",\"@_Value{0.[000]}\"]]},\"id\":\"1146\",\"type\":\"HoverTool\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":0.5,\"fill_color\":\"lightgrey\",\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":1.0,\"line_color\":\"black\",\"line_dash\":[4,4],\"line_width\":2,\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"1145\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1142\"},{\"id\":\"1143\"},{\"id\":\"1144\"},{\"id\":\"1146\"}]},\"id\":\"1147\",\"type\":\"Toolbar\"},{\"attributes\":{\"factors\":[\"40\",\"60\",\"80\",\"100\",\"120\",\"140\",\"160\",\"180\",\"200\"]},\"id\":\"1097\",\"type\":\"FactorRange\"},{\"attributes\":{\"data_source\":{\"id\":\"1152\"},\"glyph\":{\"id\":\"1154\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1155\"},\"selection_glyph\":null,\"view\":{\"id\":\"1157\"}},\"id\":\"1156\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"fill_color\":{\"field\":\"_Value\",\"transform\":{\"id\":\"1002\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"n2\"},\"y\":{\"field\":\"n_exit\"}},\"id\":\"1154\",\"type\":\"Rect\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"n_enter\",\"@n_enter\"],[\"n_exit\",\"@n_exit\"],[\"Value\",\"@_Value{0.[000]}\"]]},\"id\":\"1177\",\"type\":\"HoverTool\"},{\"attributes\":{\"axis\":{\"id\":\"1105\"},\"grid_line_color\":null,\"ticker\":null},\"id\":\"1107\",\"type\":\"Grid\"},{\"attributes\":{\"source\":{\"id\":\"1152\"}},\"id\":\"1157\",\"type\":\"CDSView\"},{\"attributes\":{\"axis_label\":\"n_enter\",\"axis_line_color\":null,\"formatter\":{\"id\":\"1223\"},\"major_label_standoff\":0,\"major_tick_line_color\":null,\"ticker\":{\"id\":\"1109\"}},\"id\":\"1108\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"field\":\"_Value\",\"transform\":{\"id\":\"1002\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"n2\"},\"y\":{\"field\":\"n_exit\"}},\"id\":\"1155\",\"type\":\"Rect\"},{\"attributes\":{\"below\":[{\"id\":\"1105\"}],\"center\":[{\"id\":\"1107\"},{\"id\":\"1110\"}],\"left\":[{\"id\":\"1108\"}],\"plot_height\":400,\"plot_width\":400,\"renderers\":[{\"id\":\"1125\"}],\"title\":{\"id\":\"1220\"},\"toolbar\":{\"id\":\"1116\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"1097\"},\"x_scale\":{\"id\":\"1101\"},\"y_range\":{\"id\":\"1099\"},\"y_scale\":{\"id\":\"1103\"}},\"id\":\"1096\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1109\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"axis\":{\"id\":\"1108\"},\"dimension\":1,\"grid_line_color\":null,\"ticker\":null},\"id\":\"1110\",\"type\":\"Grid\"},{\"attributes\":{\"below\":[{\"id\":\"1167\"}],\"center\":[{\"id\":\"1169\"},{\"id\":\"1172\"}],\"left\":[{\"id\":\"1170\"}],\"plot_height\":400,\"plot_width\":400,\"renderers\":[{\"id\":\"1187\"}],\"title\":{\"id\":\"1240\"},\"toolbar\":{\"id\":\"1178\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"1159\"},\"x_scale\":{\"id\":\"1163\"},\"y_range\":{\"id\":\"1161\"},\"y_scale\":{\"id\":\"1165\"}},\"id\":\"1158\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"below\":[{\"id\":\"1074\"}],\"center\":[{\"id\":\"1076\"},{\"id\":\"1079\"}],\"left\":[{\"id\":\"1077\"}],\"plot_height\":400,\"plot_width\":400,\"renderers\":[{\"id\":\"1094\"}],\"title\":{\"id\":\"1210\"},\"toolbar\":{\"id\":\"1085\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"1066\"},\"x_scale\":{\"id\":\"1070\"},\"y_range\":{\"id\":\"1068\"},\"y_scale\":{\"id\":\"1072\"}},\"id\":\"1065\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"data_source\":{\"id\":\"1028\"},\"glyph\":{\"id\":\"1030\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1031\"},\"selection_glyph\":null,\"view\":{\"id\":\"1033\"}},\"id\":\"1032\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"data\":{\"_Value\":{\"__ndarray__\":\"2bK1voC5xEDSEvJBX5fBQDxFR3JB1cdAKJ9EcQSOzEAfotEdqIXUQKyGxD3W879AoFT7dKA/y0BunmBACV7HQHbWbrvgDcBArCxDHF0B0UDuGfuSzejHQDXrUP4cTMFApRkA+QRsvkCs2QOtyhHRQEq1T8eAxcxAtD1iOnfswUDVGDf0zA7BQA7gZvHq1slAJumayTePzUCHIBmeBW7AQFlzR//fK8FAMXEwrvPgy0D9pNonaIzQQNsdZdfGwcBAGKz/czD+wkCJmbZ/Bb/GQFMvv9MkzcpA+v91rzWhwUBGUS6NB27CQMUQJrI0KMNA2D15WDwwyUCybpalhubAQBiQ0zlb7MFA\",\"dtype\":\"float64\",\"order\":\"little\",\"shape\":[33]},\"index\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32],\"n2\":[\"40\",\"40\",\"60\",\"60\",\"60\",\"60\",\"80\",\"80\",\"80\",\"100\",\"100\",\"100\",\"100\",\"120\",\"120\",\"120\",\"120\",\"140\",\"140\",\"140\",\"140\",\"160\",\"160\",\"160\",\"160\",\"180\",\"180\",\"180\",\"180\",\"200\",\"200\",\"200\",\"200\"],\"n_enter\":[\"15\",\"25\",\"15\",\"20\",\"25\",\"30\",\"20\",\"25\",\"30\",\"15\",\"20\",\"25\",\"30\",\"15\",\"20\",\"25\",\"30\",\"15\",\"20\",\"25\",\"30\",\"15\",\"20\",\"25\",\"30\",\"15\",\"20\",\"25\",\"30\",\"15\",\"20\",\"25\",\"30\"]},\"selected\":{\"id\":\"1225\"},\"selection_policy\":{\"id\":\"1226\"}},\"id\":\"1121\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"data\":{\"_Value\":{\"__ndarray__\":\"ckpm4k2axECo9QAaNoPIQBy/t+kno8VAwiU9TQj2xEApv3YRwZ/KQEaMnltgGsVATMlaiePfw0D2R9+keffGQEWdO/6kbcNAIUmCcCkHxUALIY8rMW3FQDkZHCX/wMBAylvBSc8hwkBUmpSCnjfFQGbw94sRjsFAdhHfiZq2w0Be+8kYJz3DQNYcIJihkMFAT1O+leXLx0AfG5PxJg7HQK8e5OLRGsFA/KV+3vosw0CGcNCfACbJQClIMxZf08FA\",\"dtype\":\"float64\",\"order\":\"little\",\"shape\":[24]},\"index\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23],\"n1\":[\"30\",\"30\",\"30\",\"40\",\"40\",\"40\",\"50\",\"50\",\"50\",\"60\",\"60\",\"60\",\"70\",\"70\",\"70\",\"80\",\"80\",\"80\",\"90\",\"90\",\"90\",\"100\",\"100\",\"100\"],\"n_exit\":[\"10\",\"15\",\"20\",\"10\",\"15\",\"20\",\"10\",\"15\",\"20\",\"10\",\"15\",\"20\",\"10\",\"15\",\"20\",\"10\",\"15\",\"20\",\"10\",\"15\",\"20\",\"10\",\"15\",\"20\"]},\"selected\":{\"id\":\"1215\"},\"selection_policy\":{\"id\":\"1216\"}},\"id\":\"1090\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"high\":21014.62682,\"low\":6614.394239999998,\"nan_color\":\"white\",\"palette\":[\"#440154\",\"#440255\",\"#440357\",\"#450558\",\"#45065A\",\"#45085B\",\"#46095C\",\"#460B5E\",\"#460C5F\",\"#460E61\",\"#470F62\",\"#471163\",\"#471265\",\"#471466\",\"#471567\",\"#471669\",\"#47186A\",\"#48196B\",\"#481A6C\",\"#481C6E\",\"#481D6F\",\"#481E70\",\"#482071\",\"#482172\",\"#482273\",\"#482374\",\"#472575\",\"#472676\",\"#472777\",\"#472878\",\"#472A79\",\"#472B7A\",\"#472C7B\",\"#462D7C\",\"#462F7C\",\"#46307D\",\"#46317E\",\"#45327F\",\"#45347F\",\"#453580\",\"#453681\",\"#443781\",\"#443982\",\"#433A83\",\"#433B83\",\"#433C84\",\"#423D84\",\"#423E85\",\"#424085\",\"#414186\",\"#414286\",\"#404387\",\"#404487\",\"#3F4587\",\"#3F4788\",\"#3E4888\",\"#3E4989\",\"#3D4A89\",\"#3D4B89\",\"#3D4C89\",\"#3C4D8A\",\"#3C4E8A\",\"#3B508A\",\"#3B518A\",\"#3A528B\",\"#3A538B\",\"#39548B\",\"#39558B\",\"#38568B\",\"#38578C\",\"#37588C\",\"#37598C\",\"#365A8C\",\"#365B8C\",\"#355C8C\",\"#355D8C\",\"#345E8D\",\"#345F8D\",\"#33608D\",\"#33618D\",\"#32628D\",\"#32638D\",\"#31648D\",\"#31658D\",\"#31668D\",\"#30678D\",\"#30688D\",\"#2F698D\",\"#2F6A8D\",\"#2E6B8E\",\"#2E6C8E\",\"#2E6D8E\",\"#2D6E8E\",\"#2D6F8E\",\"#2C708E\",\"#2C718E\",\"#2C728E\",\"#2B738E\",\"#2B748E\",\"#2A758E\",\"#2A768E\",\"#2A778E\",\"#29788E\",\"#29798E\",\"#287A8E\",\"#287A8E\",\"#287B8E\",\"#277C8E\",\"#277D8E\",\"#277E8E\",\"#267F8E\",\"#26808E\",\"#26818E\",\"#25828E\",\"#25838D\",\"#24848D\",\"#24858D\",\"#24868D\",\"#23878D\",\"#23888D\",\"#23898D\",\"#22898D\",\"#228A8D\",\"#228B8D\",\"#218C8D\",\"#218D8C\",\"#218E8C\",\"#208F8C\",\"#20908C\",\"#20918C\",\"#1F928C\",\"#1F938B\",\"#1F948B\",\"#1F958B\",\"#1F968B\",\"#1E978A\",\"#1E988A\",\"#1E998A\",\"#1E998A\",\"#1E9A89\",\"#1E9B89\",\"#1E9C89\",\"#1E9D88\",\"#1E9E88\",\"#1E9F88\",\"#1EA087\",\"#1FA187\",\"#1FA286\",\"#1FA386\",\"#20A485\",\"#20A585\",\"#21A685\",\"#21A784\",\"#22A784\",\"#23A883\",\"#23A982\",\"#24AA82\",\"#25AB81\",\"#26AC81\",\"#27AD80\",\"#28AE7F\",\"#29AF7F\",\"#2AB07E\",\"#2BB17D\",\"#2CB17D\",\"#2EB27C\",\"#2FB37B\",\"#30B47A\",\"#32B57A\",\"#33B679\",\"#35B778\",\"#36B877\",\"#38B976\",\"#39B976\",\"#3BBA75\",\"#3DBB74\",\"#3EBC73\",\"#40BD72\",\"#42BE71\",\"#44BE70\",\"#45BF6F\",\"#47C06E\",\"#49C16D\",\"#4BC26C\",\"#4DC26B\",\"#4FC369\",\"#51C468\",\"#53C567\",\"#55C666\",\"#57C665\",\"#59C764\",\"#5BC862\",\"#5EC961\",\"#60C960\",\"#62CA5F\",\"#64CB5D\",\"#67CC5C\",\"#69CC5B\",\"#6BCD59\",\"#6DCE58\",\"#70CE56\",\"#72CF55\",\"#74D054\",\"#77D052\",\"#79D151\",\"#7CD24F\",\"#7ED24E\",\"#81D34C\",\"#83D34B\",\"#86D449\",\"#88D547\",\"#8BD546\",\"#8DD644\",\"#90D643\",\"#92D741\",\"#95D73F\",\"#97D83E\",\"#9AD83C\",\"#9DD93A\",\"#9FD938\",\"#A2DA37\",\"#A5DA35\",\"#A7DB33\",\"#AADB32\",\"#ADDC30\",\"#AFDC2E\",\"#B2DD2C\",\"#B5DD2B\",\"#B7DD29\",\"#BADE27\",\"#BDDE26\",\"#BFDF24\",\"#C2DF22\",\"#C5DF21\",\"#C7E01F\",\"#CAE01E\",\"#CDE01D\",\"#CFE11C\",\"#D2E11B\",\"#D4E11A\",\"#D7E219\",\"#DAE218\",\"#DCE218\",\"#DFE318\",\"#E1E318\",\"#E4E318\",\"#E7E419\",\"#E9E419\",\"#ECE41A\",\"#EEE51B\",\"#F1E51C\",\"#F3E51E\",\"#F6E61F\",\"#F8E621\",\"#FAE622\",\"#FDE724\"]},\"id\":\"1002\",\"type\":\"LinearColorMapper\"},{\"attributes\":{\"text\":\"\"},\"id\":\"1210\",\"type\":\"Title\"},{\"attributes\":{\"factors\":[\"30\",\"40\",\"50\",\"60\",\"70\",\"80\",\"90\",\"100\"]},\"id\":\"1066\",\"type\":\"FactorRange\"}],\"root_ids\":[\"1252\"]},\"title\":\"Bokeh Application\",\"version\":\"2.2.1\"}};\n",
       "  var render_items = [{\"docid\":\"c468d9bb-e7c5-4dcb-873b-1bd0d8397e1e\",\"root_ids\":[\"1252\"],\"roots\":{\"1252\":\"841efc58-5cec-4744-9269-0cfa6253a687\"}}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        clearInterval(timer);\n",
       "        embed_document(root);\n",
       "      } else {\n",
       "        attempts++;\n",
       "        if (attempts > 100) {\n",
       "          clearInterval(timer);\n",
       "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
       "        }\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "1252"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"display: table;\"><div style=\"display: table-row;\"><div style=\"display: table-cell;\"><b title=\"bokeh.models.layouts.Column\">Column</b>(</div><div style=\"display: table-cell;\">id&nbsp;=&nbsp;'1252', <span id=\"1578\" style=\"cursor: pointer;\">&hellip;)</span></div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">align&nbsp;=&nbsp;'start',</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">aspect_ratio&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">children&nbsp;=&nbsp;[ToolbarBox(id='1251', ...), GridBox(id='1249', ...)],</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">css_classes&nbsp;=&nbsp;[],</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">disabled&nbsp;=&nbsp;False,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">height&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">height_policy&nbsp;=&nbsp;'auto',</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">js_event_callbacks&nbsp;=&nbsp;{},</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">js_property_callbacks&nbsp;=&nbsp;{},</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">margin&nbsp;=&nbsp;(0, 0, 0, 0),</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">max_height&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">max_width&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_height&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_width&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">name&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">rows&nbsp;=&nbsp;'auto',</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">sizing_mode&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">spacing&nbsp;=&nbsp;0,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">subscribed_events&nbsp;=&nbsp;[],</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">tags&nbsp;=&nbsp;[],</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">visible&nbsp;=&nbsp;True,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">width&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">width_policy&nbsp;=&nbsp;'auto')</div></div></div>\n",
       "<script>\n",
       "(function() {\n",
       "  var expanded = false;\n",
       "  var ellipsis = document.getElementById(\"1578\");\n",
       "  ellipsis.addEventListener(\"click\", function() {\n",
       "    var rows = document.getElementsByClassName(\"1577\");\n",
       "    for (var i = 0; i < rows.length; i++) {\n",
       "      var el = rows[i];\n",
       "      el.style.display = expanded ? \"none\" : \"table-row\";\n",
       "    }\n",
       "    ellipsis.innerHTML = expanded ? \"&hellip;)\" : \"&lsaquo;&lsaquo;&lsaquo;\";\n",
       "    expanded = !expanded;\n",
       "  });\n",
       "})();\n",
       "</script>\n"
      ],
      "text/plain": [
       "Column(id='1252', ...)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from backtesting.lib import plot_heatmaps\n",
    "\n",
    "\n",
    "plot_heatmaps(heatmap, agg='mean')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model-based optimization\n",
    "\n",
    "Above, we used _randomized grid search_ optimization method. Any kind of grid search, however, might be computationally expensive for large data sets. In the follwing example, we will use\n",
    "[_scikit-optimize_](https://scikit-optimize.github.io)\n",
    "package to guide our optimization better informed using forests of decision trees.\n",
    "The hyperparameter model is sequentially improved by evaluating the expensive function (the backtest) at the next best point, thereby hopefully converging to a set of optimal parameters with as few evaluations as possible.\n",
    "\n",
    "So, with `method=\"skopt\"`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "\n",
    "! pip install scikit-optimize  # This is a run-time dependency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 24.3 s, sys: 50.4 ms, total: 24.4 s\n",
      "Wall time: 24.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "stats_skopt, heatmap, optimize_result = backtest.optimize(\n",
    "    n1=[10, 100],      # Note: For method=\"skopt\", we\n",
    "    n2=[20, 200],      # only need interval end-points\n",
    "    n_enter=[10, 40],\n",
    "    n_exit=[10, 30],\n",
    "    constraint=lambda p: p.n_exit < p.n_enter < p.n1 < p.n2,\n",
    "    maximize='Equity Final [$]',\n",
    "    method='skopt',\n",
    "    max_tries=200,\n",
    "    random_state=0,\n",
    "    return_heatmap=True,\n",
    "    return_optimization=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "n1  n2   n_enter  n_exit\n",
       "68  96   29       24       28424.02\n",
       "35  98   28       24       28658.80\n",
       "44  134  39       27       30251.81\n",
       "Name: Equity Final [$], dtype: float64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heatmap.sort_values().iloc[-3:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice how the optimization runs somewhat slower even though `max_tries=` is the same. But that's due to the sequential nature of the algorithm and should actually perform rather comparably even in cases of _much larger parameter spaces_ where grid search would effectively blow up, but likely (hopefully) reaching a better local optimum than a randomized search would.\n",
    "A note of warning, again, to take steps to avoid\n",
    "[overfitting](https://en.wikipedia.org/wiki/Overfitting)\n",
    "insofar as possible.\n",
    "\n",
    "Understanding the impact of each parameter on the computed objective function is easy in two dimensions, but as the number of dimensions grows, partial dependency plots are increasingly useful.\n",
    "[Plotting tools from _scikit-optimize_](https://scikit-optimize.github.io/stable/modules/plots.html)\n",
    "take care of many of the more mundane things needed to make good and informative plots of the parameter space:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 85368 characters]\n",
      "text/plain": [
       "<Figure size 576x576 with 16 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from skopt.plots import plot_objective\n",
    "\n",
    "_ = plot_objective(optimize_result, n_points=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 104484 characters]\n",
      "text/plain": [
       "<Figure size 576x576 with 16 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from skopt.plots import plot_evaluations\n",
    "\n",
    "_ = plot_evaluations(optimize_result, bins=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Learn more by exploring further\n",
    "[examples](https://kernc.github.io/backtesting.py/doc/backtesting/index.html#tutorials)\n",
    "or find more framework options in the\n",
    "[full API reference](https://kernc.github.io/backtesting.py/doc/backtesting/index.html#header-submodules)."
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "encoding": "# -*- coding: utf-8 -*-"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
