{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "OCTIS: Optimizing CTM.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "31102c9cf0d64a2a8728d731e0d1a1f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_874c95a78c6b42f7b7f4d11f0b94de7a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ac926ca4616c4c8da576d2be74edb678",
              "IPY_MODEL_0d1783acaceb4d219047bf670e9c230c"
            ]
          }
        },
        "874c95a78c6b42f7b7f4d11f0b94de7a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ac926ca4616c4c8da576d2be74edb678": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5b1b4e3b818b4f1ca3c8024d1cfbd3ed",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 405234788,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 405234788,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7429f1c34b3246e8b962933f13acf606"
          }
        },
        "0d1783acaceb4d219047bf670e9c230c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_598ed64fea754052bee0a27d1095025b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 405M/405M [00:28&lt;00:00, 14.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cdc5d14929cc4c5caecb57045fe4c305"
          }
        },
        "5b1b4e3b818b4f1ca3c8024d1cfbd3ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7429f1c34b3246e8b962933f13acf606": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "598ed64fea754052bee0a27d1095025b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cdc5d14929cc4c5caecb57045fe4c305": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b4c5321deb0546248709f6b602a18c35": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ac745ef19306427685956525b1117198",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_526d4854432e41f79bad4cc54e7f8ac0",
              "IPY_MODEL_3853001a9a7d41658fca2627afd42a06"
            ]
          }
        },
        "ac745ef19306427685956525b1117198": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "526d4854432e41f79bad4cc54e7f8ac0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3b1e7527484d421b860fdacb67d53052",
            "_dom_classes": [],
            "description": "Batches: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 59,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 59,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1714d5ca1b464c64b2025a42c13f3336"
          }
        },
        "3853001a9a7d41658fca2627afd42a06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9a8b61c52d844cb2a698bc22c7001e9c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 59/59 [00:13&lt;00:00,  4.22it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4ce85e23580b4c2ab6de39f12f0e9580"
          }
        },
        "3b1e7527484d421b860fdacb67d53052": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1714d5ca1b464c64b2025a42c13f3336": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9a8b61c52d844cb2a698bc22c7001e9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4ce85e23580b4c2ab6de39f12f0e9580": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dd103201f9fa4845b1ad12eee06d1519": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4b785a4887ff4ccf9fba882cf13ad08b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8abe3e30fb4c4cad86e9f0783eab80b2",
              "IPY_MODEL_36eddb2d44cb4de6a7fed67e4ad517a3"
            ]
          }
        },
        "4b785a4887ff4ccf9fba882cf13ad08b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8abe3e30fb4c4cad86e9f0783eab80b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_380e4ccf270d42eea8f24dba067d4acb",
            "_dom_classes": [],
            "description": "Batches: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 13,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 13,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_355b31da338a4f01a28997a5c2a40fa0"
          }
        },
        "36eddb2d44cb4de6a7fed67e4ad517a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3db916ac8cf347a79ba0a2ff37f94e7d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 13/13 [00:07&lt;00:00,  1.77it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0996f9abc83f47608a2e9a60b3cd3be1"
          }
        },
        "380e4ccf270d42eea8f24dba067d4acb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "355b31da338a4f01a28997a5c2a40fa0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3db916ac8cf347a79ba0a2ff37f94e7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0996f9abc83f47608a2e9a60b3cd3be1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d76028a0594c412b9f4193b5ccc3eedf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2b0ad5f0b7de43d0b5b57e604d698126",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_752eb075b09245b989826728f4fa7700",
              "IPY_MODEL_9f677d6cfb01491ca926d650194c9b57"
            ]
          }
        },
        "2b0ad5f0b7de43d0b5b57e604d698126": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "752eb075b09245b989826728f4fa7700": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3b7704805aea437f9ecaabed614fc7ce",
            "_dom_classes": [],
            "description": "Batches: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 13,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 13,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_47bebaf8d0194fbca2a36e8648c74a67"
          }
        },
        "9f677d6cfb01491ca926d650194c9b57": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bde84bf66a4a4c8ca7209c084ca6446a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 13/13 [00:04&lt;00:00,  3.15it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a1eac445c06f4969809fcf1f62d576b8"
          }
        },
        "3b7704805aea437f9ecaabed614fc7ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "47bebaf8d0194fbca2a36e8648c74a67": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bde84bf66a4a4c8ca7209c084ca6446a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a1eac445c06f4969809fcf1f62d576b8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SypZeh0qCJWy"
      },
      "source": [
        "# Tutorial: How to optimize a Neural Topic Model (Contextualized Topic Models) with OCTIS"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gmoy7q8wCJW1"
      },
      "source": [
        "Welcome! This is a tutorial that allows you to optimize the hyperparameters of a neural topic model using OCTIS (Optimizing and Comparing Topic Models Is Simple). \n",
        "\n",
        "![](https://github.com/MIND-Lab/OCTIS/blob/master/logo.png?raw=true)\n",
        "\n",
        "A topic model allows you to discover the latent topics in your documents in a completely unsupervised way. But to find the best results, you usually need to optimize its hyperparameters. Let's optimize the hyperparameters of the [Zero-shot Contextualized Topic Model](https://github.com/MilaNLProc/contextualized-topic-models), that is already integrated into OCTIS. It's very easy :)\n",
        "\n",
        "## Enabling the GPU\n",
        "\n",
        "First, you'll need to enable GPUs for the notebook:\n",
        "\n",
        "- Navigate to Edit→Notebook Settings\n",
        "- select GPU from the Hardware Accelerator drop-down\n",
        "\n",
        "[Reference](https://colab.research.google.com/notebooks/gpu.ipynb)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sJc4BlT_bKjo"
      },
      "source": [
        "## Installing OCTIS\n",
        "\n",
        "We need to install OCTIS. (We are going to use the library version of OCTIS, but you can also use it through its dashboard. See https://github.com/mind-Lab/octis for more details.)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GAcJGGSfCJW2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "27d40e8c-c780-44a6-9efb-17a152493cce"
      },
      "source": [
        "!pip install octis"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting octis\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/61/5e/f53cef86f2f868ec55ffc6c1e38138e18cc17145380bd2e2e7ecc1d35068/octis-1.1.1-py2.py3-none-any.whl (108kB)\n",
            "\r\u001b[K     |███                             | 10kB 22.6MB/s eta 0:00:01\r\u001b[K     |██████                          | 20kB 18.2MB/s eta 0:00:01\r\u001b[K     |█████████                       | 30kB 14.6MB/s eta 0:00:01\r\u001b[K     |████████████                    | 40kB 13.4MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 51kB 8.2MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 61kB 9.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 71kB 9.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████▏       | 81kB 10.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▏    | 92kB 9.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▏ | 102kB 8.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 112kB 8.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from octis) (1.8.1+cu101)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from octis) (2.23.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from octis) (1.1.5)\n",
            "Collecting libsvm\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4b/11/c7700d0cd3a21eef2d7d996256277fc640ccd4f84717c10228cb6c1567dc/libsvm-3.23.0.4.tar.gz (170kB)\n",
            "\u001b[K     |████████████████████████████████| 174kB 14.2MB/s \n",
            "\u001b[?25hCollecting sentence-transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/35/aa/f672ce489063c4ee7a566ebac1b723c53ac0cea19d9e36599cc241d8ed56/sentence-transformers-1.0.4.tar.gz (74kB)\n",
            "\u001b[K     |████████████████████████████████| 81kB 7.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: flask in /usr/local/lib/python3.7/dist-packages (from octis) (1.1.2)\n",
            "Collecting scikit-optimize>=0.8.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/8b/03/be33e89f55866065a02e515c5b319304a801a9f1027a9b311a9b1d1f8dc7/scikit_optimize-0.8.1-py2.py3-none-any.whl (101kB)\n",
            "\u001b[K     |████████████████████████████████| 102kB 8.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (from octis) (3.2.5)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from octis) (0.22.2.post1)\n",
            "Requirement already satisfied: spacy in /usr/local/lib/python3.7/dist-packages (from octis) (2.2.4)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from octis) (3.2.2)\n",
            "Requirement already satisfied: gensim<=3.8.3 in /usr/local/lib/python3.7/dist-packages (from octis) (3.6.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from octis) (1.19.5)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch->octis) (3.7.4.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->octis) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->octis) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->octis) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->octis) (2.10)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->octis) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->octis) (2018.9)\n",
            "Collecting transformers<5.0.0,>=3.1.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d8/b2/57495b5309f09fa501866e225c84532d1fd89536ea62406b2181933fb418/transformers-4.5.1-py3-none-any.whl (2.1MB)\n",
            "\u001b[K     |████████████████████████████████| 2.1MB 14.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from sentence-transformers->octis) (4.41.1)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from sentence-transformers->octis) (1.4.1)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f5/99/e0808cb947ba10f575839c43e8fafc9cc44e4a7a2c8f79c60db48220a577/sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 49.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: itsdangerous>=0.24 in /usr/local/lib/python3.7/dist-packages (from flask->octis) (1.1.0)\n",
            "Requirement already satisfied: click>=5.1 in /usr/local/lib/python3.7/dist-packages (from flask->octis) (7.1.2)\n",
            "Requirement already satisfied: Werkzeug>=0.15 in /usr/local/lib/python3.7/dist-packages (from flask->octis) (1.0.1)\n",
            "Requirement already satisfied: Jinja2>=2.10.1 in /usr/local/lib/python3.7/dist-packages (from flask->octis) (2.11.3)\n",
            "Collecting pyaml>=16.9\n",
            "  Downloading https://files.pythonhosted.org/packages/15/c4/1310a054d33abc318426a956e7d6df0df76a6ddfa9c66f6310274fb75d42/pyaml-20.4.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-optimize>=0.8.1->octis) (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from nltk->octis) (1.15.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy->octis) (54.2.0)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy->octis) (3.0.5)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy->octis) (2.0.5)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy->octis) (0.8.2)\n",
            "Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy->octis) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy->octis) (1.0.5)\n",
            "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.7/dist-packages (from spacy->octis) (1.1.3)\n",
            "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy->octis) (0.4.1)\n",
            "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.7/dist-packages (from spacy->octis) (1.0.0)\n",
            "Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy->octis) (7.4.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->octis) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->octis) (1.3.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->octis) (2.4.7)\n",
            "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.7/dist-packages (from gensim<=3.8.3->octis) (5.0.0)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 57.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=3.1.0->sentence-transformers->octis) (2019.12.20)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=3.1.0->sentence-transformers->octis) (3.10.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=3.1.0->sentence-transformers->octis) (20.9)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=3.1.0->sentence-transformers->octis) (3.0.12)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 52.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2>=2.10.1->flask->octis) (1.1.1)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from pyaml>=16.9->scikit-optimize>=0.8.1->octis) (3.13)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers<5.0.0,>=3.1.0->sentence-transformers->octis) (3.4.1)\n",
            "Building wheels for collected packages: libsvm, sentence-transformers\n",
            "  Building wheel for libsvm (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for libsvm: filename=libsvm-3.23.0.4-cp37-cp37m-linux_x86_64.whl size=233331 sha256=522fc3a95e9635cabe7fbd6061ee1d62610980c31425a1fa958571b7ecb4c34b\n",
            "  Stored in directory: /root/.cache/pip/wheels/5e/9e/b5/dbb033107407eec2f52b8cd24cf024a4b9ec8b62ea5aee995a\n",
            "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sentence-transformers: filename=sentence_transformers-1.0.4-cp37-none-any.whl size=114307 sha256=af73392513382dae317aba9aafc8a677459a3a86f49392ea400001eabd7e13ca\n",
            "  Stored in directory: /root/.cache/pip/wheels/26/ea/89/d0d2e013d951b6d23270aa9ca4018b82632ab7cd933c331316\n",
            "Successfully built libsvm sentence-transformers\n",
            "Installing collected packages: libsvm, tokenizers, sacremoses, transformers, sentencepiece, sentence-transformers, pyaml, scikit-optimize, octis\n",
            "Successfully installed libsvm-3.23.0.4 octis-1.1.1 pyaml-20.4.0 sacremoses-0.0.45 scikit-optimize-0.8.1 sentence-transformers-1.0.4 sentencepiece-0.1.95 tokenizers-0.10.2 transformers-4.5.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JCA6fM1mCJW4"
      },
      "source": [
        "Let's import what we need. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HyMgyvfKCJW4"
      },
      "source": [
        "from octis.models.CTM import CTM\n",
        "from octis.dataset.dataset import Dataset\n",
        "from octis.optimization.optimizer import Optimizer\n",
        "from skopt.space.space import Real, Categorical, Integer\n",
        "from octis.evaluation_metrics.coherence_metrics import Coherence"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TIYk6bAHCJW4"
      },
      "source": [
        "## Load Data\n",
        "We need some data to run a topic model. OCTIS already provides 4 already-preprocessed datasets. Let's use one of them."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s6VP-srzCJW5"
      },
      "source": [
        "dataset = Dataset()\n",
        "dataset.fetch_dataset(\"M10\")"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i6Sywe4vCJW5"
      },
      "source": [
        "## Load Topic Model\n",
        "The zero-shot version of the [Contextualized Topic Model](https://github.com/MilaNLProc/contextualized-topic-models) is a very recent approach that is based on a Variational Autoencoder and replaces the classical input Bag-of-Words representation with a pre-trained contextualized representation of the document. (See its original paper here: https://arxiv.org/abs/2004.07737)\n",
        "\n",
        "Let's define the topic model and the hyperparameters that we want to fix. Let's set the `number of topics` and the `maximum number of epochs`. The dataset is split in training, validation and testing. The model will stop with an early stopping criterion. We also define the `inference_type` parameter to run the zero-shot CTM (otherwise you can specify `combined` to run the combined CTM version of the model). Finally, we select the language model that we want to use. Here we use `sentence-transformers's RoBERTa`, but you can use whatever language model you like (https://huggingface.co/sentence-transformers)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "podktzHoCJW5"
      },
      "source": [
        "model = CTM(num_topics=10, num_epochs=30, inference_type='zeroshot', bert_model=\"bert-base-nli-mean-tokens\")"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "989VrG4GCJW6"
      },
      "source": [
        "## Define Evaluation Metric\n",
        "\n",
        "We want to find the best hyperparameter configuration of CTM for the topic coherence. So, we initialize the topic coherence. We are going to use the well-know NPMI topic coherence, but we provide a wide range of different measures. See here: https://github.com/MIND-Lab/OCTIS/blob/master/octis/evaluation_metrics/coherence_metrics.py \n",
        "\n",
        "We are going to compute the word co-occurrences of the topic coherence on the dataset itself. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "95Ws4vv5CJW6"
      },
      "source": [
        "npmi = Coherence(texts=dataset.get_corpus())"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3V-eCgvpCJW7"
      },
      "source": [
        "## Define Hyperparameter Search Space \n",
        "\n",
        "Now we are going to define the search space for the optimization algorithm. We need to specify the type of each hyperparameter (`Categorical`, `Integer` or `Real`). If it's Categorical (e.g. the activation function), we need to list the possibile values, otherwise (e.g. the dropout) we specify the lower and upper bounds of the hyperparameter. \n",
        "\n",
        "Note that we choose to define the number of neurons as categorical to reduce the search space. Instead of considering all the possible 200 values between 100 and 300, we reduce the search to possible 3 values. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P1pIZSZOCJW7"
      },
      "source": [
        "search_space = {\"num_layers\": Categorical({1, 2, 3}), \n",
        "                \"num_neurons\": Categorical({100, 200, 300}),\n",
        "                \"activation\": Categorical({'sigmoid', 'relu', 'softplus'}), \n",
        "                \"dropout\": Real(0.0, 0.95)\n",
        "}"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "njjkNjl9CJW8"
      },
      "source": [
        "####Which is the correct the number of optimization iterations?\n",
        "\n",
        "It's a threshold between how much time you want to spend on an experiment and how much accurate you want your experiments to be. **Recall that the more hyperparameters to optimize you have, the more iterations you need to do.** \n",
        "As a rule of thumb, if you have *N* hyperparameters to optimize, then you should make at least *15 times N* iterations. \n",
        "\n",
        "\n",
        "We can also set the number of **model's runs**. Topic models are usually probabilistic and thus produce different results even with the same hyperparameter configuration. So we run the model for times and then take the median of the evaluated metric to get a more reliable result.\n",
        "\n",
        "Since we just want to show the functionalities of OCTIS, we are going to set a low number of iterations and models' runs. Feel free to change the following numbers for more robust results.  "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MYsjlNesCJW8"
      },
      "source": [
        "optimization_runs=30\n",
        "model_runs=1"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CZ3szJThCJW9"
      },
      "source": [
        "We are ready to launch the hyperparameter optimization. Let's define our Optimizer and give it our CTM model, the search space, the metric to evaluate and the other parameters.\n",
        "\n",
        "We are going to save the models' results in the folder \"results/test_ctm\". In this way, you can retrieve the results and use them later to further analysis.\n",
        "\n",
        "*Optional*: You can keep track of other evaluation metrics while you are optimizing for a specified metric. Just define the metrics that you want to evaluate, append them to a list and use them as the parameter `extra_metrics` of the `Optimizer.optimize` method.\n",
        "\n",
        "Now just wait for the experiment to complete.. it will take some time :)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 763,
          "referenced_widgets": [
            "31102c9cf0d64a2a8728d731e0d1a1f0",
            "874c95a78c6b42f7b7f4d11f0b94de7a",
            "ac926ca4616c4c8da576d2be74edb678",
            "0d1783acaceb4d219047bf670e9c230c",
            "5b1b4e3b818b4f1ca3c8024d1cfbd3ed",
            "7429f1c34b3246e8b962933f13acf606",
            "598ed64fea754052bee0a27d1095025b",
            "cdc5d14929cc4c5caecb57045fe4c305",
            "b4c5321deb0546248709f6b602a18c35",
            "ac745ef19306427685956525b1117198",
            "526d4854432e41f79bad4cc54e7f8ac0",
            "3853001a9a7d41658fca2627afd42a06",
            "3b1e7527484d421b860fdacb67d53052",
            "1714d5ca1b464c64b2025a42c13f3336",
            "9a8b61c52d844cb2a698bc22c7001e9c",
            "4ce85e23580b4c2ab6de39f12f0e9580",
            "dd103201f9fa4845b1ad12eee06d1519",
            "4b785a4887ff4ccf9fba882cf13ad08b",
            "8abe3e30fb4c4cad86e9f0783eab80b2",
            "36eddb2d44cb4de6a7fed67e4ad517a3",
            "380e4ccf270d42eea8f24dba067d4acb",
            "355b31da338a4f01a28997a5c2a40fa0",
            "3db916ac8cf347a79ba0a2ff37f94e7d",
            "0996f9abc83f47608a2e9a60b3cd3be1",
            "d76028a0594c412b9f4193b5ccc3eedf",
            "2b0ad5f0b7de43d0b5b57e604d698126",
            "752eb075b09245b989826728f4fa7700",
            "9f677d6cfb01491ca926d650194c9b57",
            "3b7704805aea437f9ecaabed614fc7ce",
            "47bebaf8d0194fbca2a36e8648c74a67",
            "bde84bf66a4a4c8ca7209c084ca6446a",
            "a1eac445c06f4969809fcf1f62d576b8"
          ]
        },
        "id": "Jsevrt33CJW9",
        "outputId": "0ed59794-1077-4def-bf1b-fdc2c1786e00"
      },
      "source": [
        "optimizer=Optimizer()\n",
        "optimization_result = optimizer.optimize(\n",
        "    model, dataset, npmi, search_space, number_of_call=optimization_runs, \n",
        "    model_runs=model_runs, save_models=True, \n",
        "    extra_metrics=None, # to keep track of other metrics\n",
        "    save_path='results/test_ctm//')"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Current call:  0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "31102c9cf0d64a2a8728d731e0d1a1f0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=405234788.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b4c5321deb0546248709f6b602a18c35",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Batches', max=59.0, style=ProgressStyle(description_width…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dd103201f9fa4845b1ad12eee06d1519",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Batches', max=13.0, style=ProgressStyle(description_width…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d76028a0594c412b9f4193b5ccc3eedf",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Batches', max=13.0, style=ProgressStyle(description_width…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Current call:  1\n",
            "Current call:  2\n",
            "Current call:  3\n",
            "Current call:  4\n",
            "Current call:  5\n",
            "Current call:  6\n",
            "Current call:  7\n",
            "Current call:  8\n",
            "Current call:  9\n",
            "Current call:  10\n",
            "Current call:  11\n",
            "Current call:  12\n",
            "Current call:  13\n",
            "Current call:  14\n",
            "Current call:  15\n",
            "Current call:  16\n",
            "Current call:  17\n",
            "Current call:  18\n",
            "Current call:  19\n",
            "Current call:  20\n",
            "Current call:  21\n",
            "Current call:  22\n",
            "Current call:  23\n",
            "Current call:  24\n",
            "Current call:  25\n",
            "Current call:  26\n",
            "Current call:  27\n",
            "Current call:  28\n",
            "Current call:  29\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4c7mZe5NCJW9"
      },
      "source": [
        "The summary of the results are stored in the file \"results.json\" in the results folder. But you can also save the main results of the optimization in a csv file. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aqOZ6b6uCJW9"
      },
      "source": [
        "optimization_result.save_to_csv(\"results_ctm.csv\")"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8GTQnbSK747D"
      },
      "source": [
        "## Analysis of the results\n",
        "\n",
        "Let's inspect the optimization results. Just load the results file. This is the information that is stored. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "er0UeANzLSkF",
        "outputId": "9b3605c7-cca8-4623-bc77-b6282aa15b48"
      },
      "source": [
        "import json\n",
        "res = json.load(open(\"results/test_ctm/result.json\",'r'))\n",
        "res.keys()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['dataset_name', 'dataset_path', 'is_cached', 'kernel', 'acq_func', 'surrogate_model', 'optimization_type', 'model_runs', 'save_models', 'save_step', 'save_name', 'save_path', 'early_stop', 'early_step', 'plot_model', 'plot_best_seen', 'plot_name', 'log_scale_plot', 'search_space', 'model_name', 'model_attributes', 'use_partitioning', 'metric_name', 'extra_metric_names', 'metric_attributes', 'extra_metric_attributes', 'current_call', 'number_of_call', 'random_state', 'x0', 'y0', 'n_random_starts', 'initial_point_generator', 'topk', 'time_eval', 'dict_model_runs', 'f_val', 'x_iters'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i12WwX__eXzE"
      },
      "source": [
        "The key \"x_iters\" is a dictionary containing the values of a given hyperparameter for each iteration. For example, these are the evaluated values of the hyperparameter \"activation\" across all the iterations. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "blPblmCDLyYZ",
        "outputId": "d475a809-f977-46bc-9d27-c8ac06347222"
      },
      "source": [
        "res['x_iters']['activation']"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['sigmoid',\n",
              " 'softplus',\n",
              " 'relu',\n",
              " 'softplus',\n",
              " 'sigmoid',\n",
              " 'sigmoid',\n",
              " 'sigmoid',\n",
              " 'relu',\n",
              " 'sigmoid',\n",
              " 'sigmoid',\n",
              " 'softplus',\n",
              " 'softplus',\n",
              " 'softplus',\n",
              " 'softplus',\n",
              " 'relu',\n",
              " 'relu',\n",
              " 'relu',\n",
              " 'sigmoid',\n",
              " 'relu',\n",
              " 'sigmoid',\n",
              " 'relu',\n",
              " 'sigmoid',\n",
              " 'sigmoid',\n",
              " 'relu',\n",
              " 'sigmoid',\n",
              " 'relu',\n",
              " 'softplus',\n",
              " 'sigmoid',\n",
              " 'softplus',\n",
              " 'relu']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QwylBlHKfJyM"
      },
      "source": [
        "And this is the values of the function across all the iterations. Just use the key \"f_val\". "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6561Rl6ffPdm",
        "outputId": "aeb21412-b6e7-407b-cde3-6b9cc096359b"
      },
      "source": [
        "res[\"f_val\"]"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[-0.0811958384022877,\n",
              " -0.026938754847881226,\n",
              " 0.023606570633877395,\n",
              " 0.026978306655116246,\n",
              " 0.053657605213730954,\n",
              " -0.12804161214591037,\n",
              " -0.014701411443803497,\n",
              " -0.007806547779232728,\n",
              " 0.03206141047206094,\n",
              " -0.025363665885419405,\n",
              " -0.16159967411785928,\n",
              " -0.033399039165565456,\n",
              " -0.022945431819161442,\n",
              " -0.008497785382228052,\n",
              " 0.013488975998346347,\n",
              " 0.01433232434558565,\n",
              " 0.04710382400465463,\n",
              " -0.22789569333803864,\n",
              " 0.007068519274295124,\n",
              " -0.13388267266662082,\n",
              " 0.021942407221298737,\n",
              " 0.016968538353056145,\n",
              " -0.034584000079006944,\n",
              " -0.04315644217904248,\n",
              " 0.00796113614067216,\n",
              " -0.016843512118015512,\n",
              " -0.16150485448077895,\n",
              " -0.06176692695582382,\n",
              " -0.04759654110275695,\n",
              " -0.00499363559485678]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EDIqQLzffV0f"
      },
      "source": [
        "We can plot this to see the progress of the Bayesian optimization.  "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "7N1av_Rkfhog",
        "outputId": "bee06758-cee3-4e51-8822-eb389aec0182"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.plot(res[\"f_val\"])"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f80bd849bd0>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "[encoded data, 22456 characters]\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7zMmhoIZFQRn"
      },
      "source": [
        "## Restart an experiment\n",
        "If you're not satisfied with the results and have a little more time to run your experiments, you can resume the optimization experiment for additional optimization runs. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9EbdwW_amNMD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f5606832-6c0a-4f83-d32d-83c22fa77cab"
      },
      "source": [
        "optimizer=Optimizer()\n",
        "optimization_result = optimizer.resume_optimization('/content/results/test_ctm/result.json',extra_evaluations=5)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Current call:  30\n",
            "Current call:  31\n",
            "Current call:  32\n",
            "Current call:  33\n",
            "Current call:  34\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BxmY1HR9QFeR"
      },
      "source": [
        "## Plot the updated results"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "vSPvojNmgRHR",
        "outputId": "93d2c0e5-923d-4414-987a-0ca5d2f0f4b4"
      },
      "source": [
        "res = json.load(open(\"results/test_ctm/result.json\",'r'))\n",
        "plt.plot(res[\"f_val\"])"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f80b43c28d0>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "[encoded data, 24052 characters]\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BJ1WEszAQH4R"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}