{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyOf+kLu0xMnC5wm0f6YgZ3O",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e6d5ea0ff92f427f8e733ff2bc70322f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_622cf31e058447dd999f64cae77b4d46",
              "IPY_MODEL_ff535c4f41bb40c1b38560ba63fea8c0",
              "IPY_MODEL_540f86d8cae94ba68bb8fb127b75b07d",
              "IPY_MODEL_939416b688f7435c8c88e8b65f70632e"
            ],
            "layout": "IPY_MODEL_665ff727519c4c6d991a512e8881f6f9"
          }
        },
        "c73636a1868f44a1915dbce680036d07": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eea5eb9c8027450d933614a4c582b2d9",
            "placeholder": "​",
            "style": "IPY_MODEL_aa7e687b821543d0b1a6793f9802b3d5",
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
          }
        },
        "06f8d75411d640f39f208ecced297fcc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "PasswordModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "PasswordModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "PasswordView",
            "continuous_update": true,
            "description": "Token:",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_4acfc531cd7441d1917771303d323c3e",
            "placeholder": "​",
            "style": "IPY_MODEL_acb1aeba6d234e538b5b735239522831",
            "value": ""
          }
        },
        "6cac660153f24dc2abb18eec07dbcb65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "CheckboxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Add token as git credential?",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_d47a397605224317a0d9b7e89a186705",
            "style": "IPY_MODEL_ccb37743fb304d34ad8b73252f4ed961",
            "value": true
          }
        },
        "aa7cc359f3934cd585ad5360ab03cb9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ButtonView",
            "button_style": "",
            "description": "Login",
            "disabled": false,
            "icon": "",
            "layout": "IPY_MODEL_02440087692040699be88744213b9853",
            "style": "IPY_MODEL_e805fd8891cc47c98a30e5b4f049e6e4",
            "tooltip": ""
          }
        },
        "04c1555bcd98495ab62d0c4f0ba7d2c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c5a175b447754f66b2441cbf66dff3e5",
            "placeholder": "​",
            "style": "IPY_MODEL_73b510cd99df43eca6b4941506971217",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
          }
        },
        "665ff727519c4c6d991a512e8881f6f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": "center",
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "flex",
            "flex": null,
            "flex_flow": "column",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "eea5eb9c8027450d933614a4c582b2d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "aa7e687b821543d0b1a6793f9802b3d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4acfc531cd7441d1917771303d323c3e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "acb1aeba6d234e538b5b735239522831": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d47a397605224317a0d9b7e89a186705": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ccb37743fb304d34ad8b73252f4ed961": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "02440087692040699be88744213b9853": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e805fd8891cc47c98a30e5b4f049e6e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "button_color": null,
            "font_weight": ""
          }
        },
        "c5a175b447754f66b2441cbf66dff3e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "73b510cd99df43eca6b4941506971217": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "500caec3bea746adb64958a34089d5f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0adfb81229a34d4da7344f22ea4ecbab",
            "placeholder": "​",
            "style": "IPY_MODEL_6dc814aab5e743ea8b4d3cd523108d91",
            "value": "Connecting..."
          }
        },
        "0adfb81229a34d4da7344f22ea4ecbab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6dc814aab5e743ea8b4d3cd523108d91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "622cf31e058447dd999f64cae77b4d46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ca9cd4ea93ee4ceba4e7a276375379ee",
            "placeholder": "​",
            "style": "IPY_MODEL_28cb52e05a8f4d8faef85ae9f0b234df",
            "value": "Token is valid (permission: write)."
          }
        },
        "ff535c4f41bb40c1b38560ba63fea8c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9ba6170111844457b7fa984f54a16bb0",
            "placeholder": "​",
            "style": "IPY_MODEL_1d0461d876be4554a0b515bc8e1850cf",
            "value": "Your token has been saved in your configured git credential helpers (store)."
          }
        },
        "540f86d8cae94ba68bb8fb127b75b07d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b985d68a8720452194082976bf317fa9",
            "placeholder": "​",
            "style": "IPY_MODEL_78f012dd14c64079bc35994342849731",
            "value": "Your token has been saved to /root/.cache/huggingface/token"
          }
        },
        "939416b688f7435c8c88e8b65f70632e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9d8239c7a63147c9839b80d175eebcbd",
            "placeholder": "​",
            "style": "IPY_MODEL_0ff33abe7843439eb27e44fca7005ff5",
            "value": "Login successful"
          }
        },
        "ca9cd4ea93ee4ceba4e7a276375379ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "28cb52e05a8f4d8faef85ae9f0b234df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9ba6170111844457b7fa984f54a16bb0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1d0461d876be4554a0b515bc8e1850cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b985d68a8720452194082976bf317fa9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78f012dd14c64079bc35994342849731": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9d8239c7a63147c9839b80d175eebcbd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0ff33abe7843439eb27e44fca7005ff5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1754df15980b41059f9b79031c78109c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_85bdae5957194998b3297985cef0d36c",
              "IPY_MODEL_71105abe48c44bdf9508e9ab7f26804e",
              "IPY_MODEL_4283d976d8384716a3f32ede781583ad"
            ],
            "layout": "IPY_MODEL_578a3b5a37554f5991c2267d17243b4c"
          }
        },
        "85bdae5957194998b3297985cef0d36c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a96eed2eb0cb40b8ad287b7c78270f12",
            "placeholder": "​",
            "style": "IPY_MODEL_b224fe088b55473cb1c3f04093a63b4d",
            "value": "Map: 100%"
          }
        },
        "71105abe48c44bdf9508e9ab7f26804e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f0640e2c20c446e5b747e2b923b85f1b",
            "max": 2003,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_786a100ecd5e41e78d8bfa40675d2198",
            "value": 2003
          }
        },
        "4283d976d8384716a3f32ede781583ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_33b5ed807aa8457fb6c9b1d7761b170d",
            "placeholder": "​",
            "style": "IPY_MODEL_ca3ca6c789fa4f16bbfd4cbc317d8a54",
            "value": " 2003/2003 [00:00&lt;00:00, 3274.22 examples/s]"
          }
        },
        "578a3b5a37554f5991c2267d17243b4c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a96eed2eb0cb40b8ad287b7c78270f12": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b224fe088b55473cb1c3f04093a63b4d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f0640e2c20c446e5b747e2b923b85f1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "786a100ecd5e41e78d8bfa40675d2198": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "33b5ed807aa8457fb6c9b1d7761b170d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ca3ca6c789fa4f16bbfd4cbc317d8a54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mrm8488/shared_colab_notebooks/blob/master/fine_tune_StarCoder_2_codeAlpaca_20k_qlora.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Fine-tune StarCoder-2 3B on codeAlpaca using QLoRA\n",
        "\n",
        "> Created by [@mrm8488](http://twitter.com/mrm8488)\n"
      ],
      "metadata": {
        "id": "mrFVntsn5CoC"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mZTroIRT40Hp",
        "outputId": "c4ccc142-f41d-4fd1-98c4-2812f2b3954f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Wed Feb 28 18:33:43 2024       \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 535.104.05             Driver Version: 535.104.05   CUDA Version: 12.2     |\n",
            "|-----------------------------------------+----------------------+----------------------+\n",
            "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                                         |                      |               MIG M. |\n",
            "|=========================================+======================+======================|\n",
            "|   0  Tesla T4                       Off | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   66C    P8              11W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
            "|                                         |                      |                  N/A |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "                                                                                         \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| Processes:                                                                            |\n",
            "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
            "|        ID   ID                                                             Usage      |\n",
            "|=======================================================================================|\n",
            "|  No running processes found                                                           |\n",
            "+---------------------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "! nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! rm -rf /content/starcoder-2-3b-ft-codealpaca"
      ],
      "metadata": {
        "id": "o4Z7K-ZU_jz8"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "\n",
        "! pip install git+https://github.com/huggingface/transformers\n",
        "! pip install git+https://github.com/huggingface/trl\n",
        "! pip install peft\n",
        "! pip install wandb\n",
        "! pip install accelerate\n",
        "! pip intall datasets\n",
        "! pip install bitsandbytes"
      ],
      "metadata": {
        "id": "p7FG9j695F8n"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from dataclasses import dataclass, field\n",
        "from typing import Optional\n",
        "import numpy as np\n",
        "\n",
        "import torch\n",
        "from datasets import load_dataset\n",
        "from peft import LoraConfig\n",
        "from transformers import (\n",
        "    AutoModelForCausalLM,\n",
        "    AutoTokenizer,\n",
        "    BitsAndBytesConfig,\n",
        "    TrainingArguments,\n",
        "    Trainer\n",
        ")\n",
        "import wandb\n",
        "from trl import SFTTrainer"
      ],
      "metadata": {
        "id": "F5Z9TQZx59il"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Log in HF to push our model and WandB for tracking"
      ],
      "metadata": {
        "id": "6pr0oH6F6F8R"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from huggingface_hub import notebook_login\n",
        "notebook_login()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "e6d5ea0ff92f427f8e733ff2bc70322f",
            "c73636a1868f44a1915dbce680036d07",
            "06f8d75411d640f39f208ecced297fcc",
            "6cac660153f24dc2abb18eec07dbcb65",
            "aa7cc359f3934cd585ad5360ab03cb9e",
            "04c1555bcd98495ab62d0c4f0ba7d2c4",
            "665ff727519c4c6d991a512e8881f6f9",
            "eea5eb9c8027450d933614a4c582b2d9",
            "aa7e687b821543d0b1a6793f9802b3d5",
            "4acfc531cd7441d1917771303d323c3e",
            "acb1aeba6d234e538b5b735239522831",
            "d47a397605224317a0d9b7e89a186705",
            "ccb37743fb304d34ad8b73252f4ed961",
            "02440087692040699be88744213b9853",
            "e805fd8891cc47c98a30e5b4f049e6e4",
            "c5a175b447754f66b2441cbf66dff3e5",
            "73b510cd99df43eca6b4941506971217",
            "500caec3bea746adb64958a34089d5f8",
            "0adfb81229a34d4da7344f22ea4ecbab",
            "6dc814aab5e743ea8b4d3cd523108d91",
            "622cf31e058447dd999f64cae77b4d46",
            "ff535c4f41bb40c1b38560ba63fea8c0",
            "540f86d8cae94ba68bb8fb127b75b07d",
            "939416b688f7435c8c88e8b65f70632e",
            "ca9cd4ea93ee4ceba4e7a276375379ee",
            "28cb52e05a8f4d8faef85ae9f0b234df",
            "9ba6170111844457b7fa984f54a16bb0",
            "1d0461d876be4554a0b515bc8e1850cf",
            "b985d68a8720452194082976bf317fa9",
            "78f012dd14c64079bc35994342849731",
            "9d8239c7a63147c9839b80d175eebcbd",
            "0ff33abe7843439eb27e44fca7005ff5"
          ]
        },
        "id": "a2vcgn4X5_-H",
        "outputId": "78c7182d-5778-4886-f0b1-3c36a31fc631"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e6d5ea0ff92f427f8e733ff2bc70322f"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Do NOT execute this cell if want to log your experiment to tensorboard\n",
        "wandb.login()\n",
        "\n",
        "wandb_project = \"starcoder-2-3b-ft-codealpaca\"\n",
        "if len(wandb_project) > 0:\n",
        "    os.environ[\"WANDB_PROJECT\"] = wandb_project"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        },
        "id": "yx7ZwX8h6Cz4",
        "outputId": "0ad500df-dca4-42b0-e76a-b52692b701a8"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
            "wandb: Paste an API key from your profile and hit enter, or press ctrl+c to quit:"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " ··········\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "oOVzmCv36QjT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Load and format the dataset"
      ],
      "metadata": {
        "id": "bN83EBep6nxX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "ds_id = \"HuggingFaceH4/CodeAlpaca_20K\"\n",
        "\n",
        "ds = load_dataset(ds_id)"
      ],
      "metadata": {
        "id": "UONY7MawcsC8"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ds"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W0xMGcQlcscW",
        "outputId": "577bd2b7-7bf8-4d44-abc1-74c25cbc2d3b"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['prompt', 'completion'],\n",
              "        num_rows: 18019\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['prompt', 'completion'],\n",
              "        num_rows: 2003\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_ds = ds['train']\n",
        "test_ds = ds['test']"
      ],
      "metadata": {
        "id": "tHC1A9Cw6gvT"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_id = \"bigcode/starcoder2-3b\""
      ],
      "metadata": {
        "id": "V-MKI4FG6k98"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "7Gz_2xFA60qy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(model_id)"
      ],
      "metadata": {
        "id": "swrGpNgVdNAQ"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.special_tokens_map"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jnYHww5iddKY",
        "outputId": "0c1a8a60-4ca1-432d-80a1-a6db25eb05af"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'bos_token': '<|endoftext|>',\n",
              " 'eos_token': '<|endoftext|>',\n",
              " 'unk_token': '<|endoftext|>',\n",
              " 'additional_special_tokens': ['<|endoftext|>',\n",
              "  '<fim_prefix>',\n",
              "  '<fim_middle>',\n",
              "  '<fim_suffix>',\n",
              "  '<fim_pad>',\n",
              "  '<repo_name>',\n",
              "  '<file_sep>',\n",
              "  '<issue_start>',\n",
              "  '<issue_comment>',\n",
              "  '<issue_closed>',\n",
              "  '<jupyter_start>',\n",
              "  '<jupyter_text>',\n",
              "  '<jupyter_code>',\n",
              "  '<jupyter_output>',\n",
              "  '<jupyter_script>',\n",
              "  '<empty_output>',\n",
              "  '<code_to_intermediate>',\n",
              "  '<intermediate_to_code>',\n",
              "  '<pr>',\n",
              "  '<pr_status>',\n",
              "  '<pr_is_merged>',\n",
              "  '<pr_base>',\n",
              "  '<pr_file>',\n",
              "  '<pr_base_code>',\n",
              "  '<pr_diff>',\n",
              "  '<pr_diff_hunk>',\n",
              "  '<pr_comment>',\n",
              "  '<pr_event_id>',\n",
              "  '<pr_review>',\n",
              "  '<pr_review_state>',\n",
              "  '<pr_review_comment>',\n",
              "  '<pr_in_reply_to_review_id>',\n",
              "  '<pr_in_reply_to_comment_id>',\n",
              "  '<pr_diff_hunk_comment_line>',\n",
              "  '<NAME>',\n",
              "  '<EMAIL>',\n",
              "  '<KEY>',\n",
              "  '<PASSWORD>']}"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_ds[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kc9cnj226_fr",
        "outputId": "f1aa8249-1c31-40bd-de33-5dfd315d4043"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'prompt': 'Create a Java class which sorts the given array of numbers.\\n[9, 2, 4, 3, 6, 1]',\n",
              " 'completion': 'class ArraySort { \\n  \\n    void sort(int arr[]) { \\n        int n = arr.length; \\n  \\n        // One by one move boundary of unsorted subarray \\n        for (int i = 0; i < n-1; i++) { \\n            \\n            // Find the minimum element in unsorted array \\n            int min_index = i; \\n            for (int j = i+1; j < n; j++) \\n                if (arr[j] < arr[min_index]) \\n                    min_index = j; \\n  \\n            // Swap the found minimum element with the first element \\n            int temp = arr[min_index]; \\n            arr[min_index] = arr[i]; \\n            arr[i] = temp; \\n        } \\n    } \\n  \\n    // Prints the array \\n    void printArray(int arr[]) { \\n        int n = arr.length; \\n        for (int i=0; i<n; ++i) \\n            System.out.print(arr[i] + \" \"); \\n        System.out.println(); \\n    } \\n  \\n    // Driver code to test above \\n    public static void main(String args[]) \\n    { \\n        ArraySort ob = new ArraySort(); \\n        int arr[] = {9, 2, 4, 3, 6, 1}; \\n        ob.sort(arr); \\n        System.out.println(\"Sorted array\"); \\n        ob.printArray(arr); \\n    } \\n}'}"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def format_ds(ex):\n",
        "  prompt = ex['prompt']\n",
        "  completion = ex['completion']\n",
        "  system = f\"<|endoftext|><|system|>\\nYou are a helpful coding assistant.<|endoftext|>\\n\"\n",
        "  text = f\"{system}<|user|>\\n{prompt}<|endoftext|>\\n<|assistant|>\\n{completion}<|endoftext|>\"\n",
        "  return {'text': text}"
      ],
      "metadata": {
        "id": "d7KMDDu37Ba5"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(format_ds(train_ds[0])['text'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kBpYQttq7N4w",
        "outputId": "449fd13f-e76d-4199-a006-b72f8d313bdd"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<|endoftext|><|system|>\n",
            "You are a helpful coding assistant.<|endoftext|>\n",
            "<|user|>\n",
            "Create a Java class which sorts the given array of numbers.\n",
            "[9, 2, 4, 3, 6, 1]<|endoftext|>\n",
            "<|assistant|>\n",
            "class ArraySort { \n",
            "  \n",
            "    void sort(int arr[]) { \n",
            "        int n = arr.length; \n",
            "  \n",
            "        // One by one move boundary of unsorted subarray \n",
            "        for (int i = 0; i < n-1; i++) { \n",
            "            \n",
            "            // Find the minimum element in unsorted array \n",
            "            int min_index = i; \n",
            "            for (int j = i+1; j < n; j++) \n",
            "                if (arr[j] < arr[min_index]) \n",
            "                    min_index = j; \n",
            "  \n",
            "            // Swap the found minimum element with the first element \n",
            "            int temp = arr[min_index]; \n",
            "            arr[min_index] = arr[i]; \n",
            "            arr[i] = temp; \n",
            "        } \n",
            "    } \n",
            "  \n",
            "    // Prints the array \n",
            "    void printArray(int arr[]) { \n",
            "        int n = arr.length; \n",
            "        for (int i=0; i<n; ++i) \n",
            "            System.out.print(arr[i] + \" \"); \n",
            "        System.out.println(); \n",
            "    } \n",
            "  \n",
            "    // Driver code to test above \n",
            "    public static void main(String args[]) \n",
            "    { \n",
            "        ArraySort ob = new ArraySort(); \n",
            "        int arr[] = {9, 2, 4, 3, 6, 1}; \n",
            "        ob.sort(arr); \n",
            "        System.out.println(\"Sorted array\"); \n",
            "        ob.printArray(arr); \n",
            "    } \n",
            "}<|endoftext|>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "BmYmArI67OT5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "IX__X-YfS9pR"
      },
      "outputs": [],
      "source": [
        "train_ds = train_ds.map(format_ds)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "PkDDl1SpTCdQ"
      },
      "outputs": [],
      "source": [
        "test_ds = test_ds.map(format_ds)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Examine Token Length Distribution"
      ],
      "metadata": {
        "id": "CgBLA08k7W4T"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Assuming token_lengths contains the lengths of tokens from your dataset\n",
        "token_lengths = [len(tokenizer.encode(ex['text'])) for ex in train_ds]\n",
        "\n",
        "# Plotting the histogram\n",
        "plt.figure(figsize=(10, 6))\n",
        "plt.hist(token_lengths, bins=50, color='blue', alpha=0.7)\n",
        "plt.title('Token Length Distribution')\n",
        "plt.xlabel('Token Length')\n",
        "plt.ylabel('Frequency')\n",
        "plt.grid(True)\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 451
        },
        "id": "jkr0tKrD7XGq",
        "outputId": "d1b07338-d73e-4da6-bea4-d52e6e3f0a32"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x600 with 1 Axes>"
            ],
            "image/png": "[encoded data, 27596 characters]\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Set the QLoRA params and Load the model"
      ],
      "metadata": {
        "id": "GP6rOm_L7e_1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "bnb_config = BitsAndBytesConfig(\n",
        "        load_in_4bit=True,\n",
        "        bnb_4bit_quant_type=\"nf4\",\n",
        "        bnb_4bit_compute_dtype=torch.bfloat16\n",
        "        )"
      ],
      "metadata": {
        "id": "M2HgRSYd7Xmq"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lora_config = LoraConfig(\n",
        "  r=8,\n",
        "  target_modules=[\n",
        "      \"q_proj\",\n",
        "      \"o_proj\",\n",
        "      \"k_proj\",\n",
        "      \"v_proj\",\n",
        "      \"gate_proj\",\n",
        "      \"up_proj\",\n",
        "      \"down_proj\",\n",
        "  ],\n",
        "  task_type=\"CAUSAL_LM\",\n",
        "  )"
      ],
      "metadata": {
        "id": "d6apoFvN7pBE"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def print_trainable_parameters(model):\n",
        "    \"\"\"\n",
        "    Prints the number of trainable parameters in the model.\n",
        "    \"\"\"\n",
        "    trainable_params = 0\n",
        "    all_param = 0\n",
        "    for _, param in model.named_parameters():\n",
        "        all_param += param.numel()\n",
        "        if param.requires_grad:\n",
        "            trainable_params += param.numel()\n",
        "    print(\n",
        "        f\"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}\"\n",
        "    )"
      ],
      "metadata": {
        "id": "q9-nJGf_8Ya5"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = AutoModelForCausalLM.from_pretrained(\n",
        "    model_id,\n",
        "    quantization_config=bnb_config,\n",
        "    device_map={\"\": 0},\n",
        "    attention_dropout=0.1\n",
        ")"
      ],
      "metadata": {
        "id": "UJ7VZxtR7wXm"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print_trainable_parameters(model)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8jbP_b_a8bxG",
        "outputId": "8b038c64-7f07-45a5-96e5-9184dfce2473"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "trainable params: 151369728 || all params: 1591200768 || trainable%: 9.5129245186488\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Setup Training Arguments"
      ],
      "metadata": {
        "id": "JAM2mTTa-G99"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "training_args=TrainingArguments(\n",
        "        output_dir=\"starcoder-2-3b-ft-codealpaca\",\n",
        "        push_to_hub=True,\n",
        "        hub_private_repo=True,\n",
        "        hub_model_id=\"<user_org_id>/starcoder-2-3b-ft-codealpaca\",\n",
        "        per_device_train_batch_size=2,\n",
        "        gradient_accumulation_steps=4,\n",
        "        per_device_eval_batch_size=2,\n",
        "        warmup_steps=100,\n",
        "        learning_rate=2e-4,\n",
        "        lr_scheduler_type=\"cosine\",\n",
        "        weight_decay=0.01,\n",
        "        bf16=False,\n",
        "        fp16=True,\n",
        "        logging_strategy=\"steps\",\n",
        "        save_strategy=\"steps\",\n",
        "        save_steps=500,\n",
        "        evaluation_strategy=\"steps\",\n",
        "        eval_steps=1000,\n",
        "        save_total_limit=2,\n",
        "        logging_steps=5,\n",
        "        load_best_model_at_end=True,\n",
        "        optim=\"paged_adamw_8bit\",\n",
        "        seed=661,\n",
        "        report_to=\"tensorboard\", # change to wandb if you wish\n",
        "    )"
      ],
      "metadata": {
        "id": "apH4kopp-QMT"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Setup the Trainer"
      ],
      "metadata": {
        "id": "iBT2NdKi8nih"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "trainer = SFTTrainer(\n",
        "    model=model,\n",
        "    train_dataset=train_ds,\n",
        "    eval_dataset=test_ds,\n",
        "    max_seq_length=512,\n",
        "    args=training_args,\n",
        "    peft_config=lora_config,\n",
        "    dataset_text_field=\"text\",\n",
        "    dataset_kwargs={\n",
        "        \"add_special_tokens\": False,  # We template with special tokens\n",
        "        \"append_concat_token\": False, # No need to add additional separator token\n",
        "    }\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "1754df15980b41059f9b79031c78109c",
            "85bdae5957194998b3297985cef0d36c",
            "71105abe48c44bdf9508e9ab7f26804e",
            "4283d976d8384716a3f32ede781583ad",
            "578a3b5a37554f5991c2267d17243b4c",
            "a96eed2eb0cb40b8ad287b7c78270f12",
            "b224fe088b55473cb1c3f04093a63b4d",
            "f0640e2c20c446e5b747e2b923b85f1b",
            "786a100ecd5e41e78d8bfa40675d2198",
            "33b5ed807aa8457fb6c9b1d7761b170d",
            "ca3ca6c789fa4f16bbfd4cbc317d8a54"
          ]
        },
        "id": "qdbhshCc8r8A",
        "outputId": "9ec9273e-236e-4b56-b34c-3e310c9fadf9"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/2003 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1754df15980b41059f9b79031c78109c"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Launch Tensorboard"
      ],
      "metadata": {
        "id": "tXbJhQ2I-8NL"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "40yO_sxz-8qW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%load_ext tensorboard"
      ],
      "metadata": {
        "id": "YbrZXbIxKI5n"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%tensorboard --logdir /content/starcoder-2-3b-ft-codealpaca/runs"
      ],
      "metadata": {
        "colab": {
          "resources": {
            "https://localhost:6006/?tensorboardColab=true": {
              "data": "[encoded data, 711032 characters]",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "text/html; charset=utf-8"
                ]
              ],
              "status": 200,
              "status_text": ""
            },
            "https://localhost:6006/index.js?_file_hash=62538f61": {
              "data": "[encoded data, 7730416 characters]",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "text/javascript; charset=utf-8"
                ]
              ],
              "status": 200,
              "status_text": ""
            },
            "https://localhost:6006/font-roboto/RxZJdnzeo3R5zSexge8UUZBw1xU1rKptJj_0jans920.woff2": {
              "data": "[encoded data, 14384 characters]",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "font/woff2"
                ]
              ],
              "status": 200,
              "status_text": ""
            },
            "https://localhost:6006/icon_bundle.svg": {
              "data": "[encoded data, 19624 characters]",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "image/svg+xml; charset=utf-8"
                ]
              ],
              "status": 200,
              "status_text": ""
            },
            "https://localhost:6006/data/plugins_listing": {
              "data": "eyJ0aW1lc2VyaWVzIjogeyJkaXNhYmxlX3JlbG9hZCI6IGZhbHNlLCAiZW5hYmxlZCI6IGZhbHNlLCAicmVtb3ZlX2RvbSI6IGZhbHNlLCAidGFiX25hbWUiOiAiVGltZSBTZXJpZXMiLCAibG9hZGluZ19tZWNoYW5pc20iOiB7InR5cGUiOiAiTkdfQ09NUE9ORU5UIn19LCAic2NhbGFycyI6IHsiZGlzYWJsZV9yZWxvYWQiOiBmYWxzZSwgImVuYWJsZWQiOiBmYWxzZSwgInJlbW92ZV9kb20iOiBmYWxzZSwgInRhYl9uYW1lIjogInNjYWxhcnMiLCAibG9hZGluZ19tZWNoYW5pc20iOiB7InR5cGUiOiAiQ1VTVE9NX0VMRU1FTlQiLCAiZWxlbWVudF9uYW1lIjogInRmLXNjYWxhci1kYXNoYm9hcmQifX0sICJjdXN0b21fc2NhbGFycyI6IHsiZGlzYWJsZV9yZWxvYWQiOiBmYWxzZSwgImVuYWJsZWQiOiBmYWxzZSwgInJlbW92ZV9kb20iOiBmYWxzZSwgInRhYl9uYW1lIjogIkN1c3RvbSBTY2FsYXJzIiwgImxvYWRpbmdfbWVjaGFuaXNtIjogeyJ0eXBlIjogIkNVU1RPTV9FTEVNRU5UIiwgImVsZW1lbnRfbmFtZSI6ICJ0Zi1jdXN0b20tc2NhbGFyLWRhc2hib2FyZCJ9fSwgImltYWdlcyI6IHsiZGlzYWJsZV9yZWxvYWQiOiBmYWxzZSwgImVuYWJsZWQiOiBmYWxzZSwgInJlbW92ZV9kb20iOiBmYWxzZSwgInRhYl9uYW1lIjogImltYWdlcyIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJDVVNUT01fRUxFTUVOVCIsICJlbGVtZW50X25hbWUiOiAidGYtaW1hZ2UtZGFzaGJvYXJkIn19LCAiYXVkaW8iOiB7ImRpc2FibGVfcmVsb2FkIjogZmFsc2UsICJlbmFibGVkIjogZmFsc2UsICJyZW1vdmVfZG9tIjogZmFsc2UsICJ0YWJfbmFtZSI6ICJhdWRpbyIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJDVVNUT01fRUxFTUVOVCIsICJlbGVtZW50X25hbWUiOiAidGYtYXVkaW8tZGFzaGJvYXJkIn19LCAiZGVidWdnZXItdjIiOiB7ImRpc2FibGVfcmVsb2FkIjogZmFsc2UsICJlbmFibGVkIjogZmFsc2UsICJyZW1vdmVfZG9tIjogZmFsc2UsICJ0YWJfbmFtZSI6ICJEZWJ1Z2dlciBWMiIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJOR19DT01QT05FTlQifX0sICJncmFwaHMiOiB7ImRpc2FibGVfcmVsb2FkIjogdHJ1ZSwgImVuYWJsZWQiOiBmYWxzZSwgInJlbW92ZV9kb20iOiBmYWxzZSwgInRhYl9uYW1lIjogImdyYXBocyIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJDVVNUT01fRUxFTUVOVCIsICJlbGVtZW50X25hbWUiOiAidGYtZ3JhcGgtZGFzaGJvYXJkIn19LCAiZGlzdHJpYnV0aW9ucyI6IHsiZGlzYWJsZV9yZWxvYWQiOiBmYWxzZSwgImVuYWJsZWQiOiBmYWxzZSwgInJlbW92ZV9kb20iOiBmYWxzZSwgInRhYl9uYW1lIjogImRpc3RyaWJ1dGlvbnMiLCAibG9hZGluZ19tZWNoYW5pc20iOiB7InR5cGUiOiAiQ1VTVE9NX0VMRU1FTlQiLCAiZWxlbWVudF9uYW1lIjogInRmLWRpc3RyaWJ1dGlvbi1kYXNoYm9hcmQifX0sICJoaXN0b2dyYW1zIjogeyJkaXNhYmxlX3JlbG9hZCI6IGZhbHNlLCAiZW5hYmxlZCI6IGZhbHNlLCAicmVtb3ZlX2RvbSI6IGZhbHNlLCAidGFiX25hbWUiOiAiaGlzdG9ncmFtcyIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJDVVNUT01fRUxFTUVOVCIsICJlbGVtZW50X25hbWUiOiAidGYtaGlzdG9ncmFtLWRhc2hib2FyZCJ9fSwgInRleHQiOiB7ImRpc2FibGVfcmVsb2FkIjogZmFsc2UsICJlbmFibGVkIjogZmFsc2UsICJyZW1vdmVfZG9tIjogZmFsc2UsICJ0YWJfbmFtZSI6ICJ0ZXh0IiwgImxvYWRpbmdfbWVjaGFuaXNtIjogeyJ0eXBlIjogIkNVU1RPTV9FTEVNRU5UIiwgImVsZW1lbnRfbmFtZSI6ICJ0Zi10ZXh0LWRhc2hib2FyZCJ9fSwgInByX2N1cnZlcyI6IHsiZGlzYWJsZV9yZWxvYWQiOiBmYWxzZSwgImVuYWJsZWQiOiBmYWxzZSwgInJlbW92ZV9kb20iOiBmYWxzZSwgInRhYl9uYW1lIjogIlBSIEN1cnZlcyIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJDVVNUT01fRUxFTUVOVCIsICJlbGVtZW50X25hbWUiOiAidGYtcHItY3VydmUtZGFzaGJvYXJkIn19LCAicHJvZmlsZV9yZWRpcmVjdCI6IHsiZGlzYWJsZV9yZWxvYWQiOiBmYWxzZSwgImVuYWJsZWQiOiBmYWxzZSwgInJlbW92ZV9kb20iOiBmYWxzZSwgInRhYl9uYW1lIjogIlByb2ZpbGUiLCAibG9hZGluZ19tZWNoYW5pc20iOiB7InR5cGUiOiAiQ1VTVE9NX0VMRU1FTlQiLCAiZWxlbWVudF9uYW1lIjogInRmLXByb2ZpbGUtcmVkaXJlY3QtZGFzaGJvYXJkIn19LCAiaHBhcmFtcyI6IHsiZGlzYWJsZV9yZWxvYWQiOiBmYWxzZSwgImVuYWJsZWQiOiBmYWxzZSwgInJlbW92ZV9kb20iOiBmYWxzZSwgInRhYl9uYW1lIjogImhwYXJhbXMiLCAibG9hZGluZ19tZWNoYW5pc20iOiB7InR5cGUiOiAiQ1VTVE9NX0VMRU1FTlQiLCAiZWxlbWVudF9uYW1lIjogInRmLWhwYXJhbXMtZGFzaGJvYXJkIn19LCAibWVzaCI6IHsiZGlzYWJsZV9yZWxvYWQiOiBmYWxzZSwgImVuYWJsZWQiOiBmYWxzZSwgInJlbW92ZV9kb20iOiBmYWxzZSwgInRhYl9uYW1lIjogIm1lc2giLCAibG9hZGluZ19tZWNoYW5pc20iOiB7InR5cGUiOiAiQ1VTVE9NX0VMRU1FTlQiLCAiZWxlbWVudF9uYW1lIjogIm1lc2gtZGFzaGJvYXJkIn19LCAid2l0X3JlZGlyZWN0IjogeyJkaXNhYmxlX3JlbG9hZCI6IGZhbHNlLCAiZW5hYmxlZCI6IGZhbHNlLCAicmVtb3ZlX2RvbSI6IGZhbHNlLCAidGFiX25hbWUiOiAiV2hhdC1JZiBUb29sIiwgImxvYWRpbmdfbWVjaGFuaXNtIjogeyJ0eXBlIjogIkNVU1RPTV9FTEVNRU5UIiwgImVsZW1lbnRfbmFtZSI6ICJ0Zi13aXQtcmVkaXJlY3QtZGFzaGJvYXJkIn19LCAicHJvamVjdG9yIjogeyJkaXNhYmxlX3JlbG9hZCI6IHRydWUsICJlbmFibGVkIjogZmFsc2UsICJyZW1vdmVfZG9tIjogZmFsc2UsICJ0YWJfbmFtZSI6ICJwcm9qZWN0b3IiLCAibG9hZGluZ19tZWNoYW5pc20iOiB7InR5cGUiOiAiSUZSQU1FIiwgIm1vZHVsZV9wYXRoIjogIi9kYXRhL3BsdWdpbi9wcm9qZWN0b3IvaW5kZXguanMifX19",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/json"
                ]
              ],
              "status": 200,
              "status_text": ""
            },
            "https://localhost:6006/font-roboto/oMMgfZMQthOryQo9n22dcuvvDin1pK8aKteLpeZ5c0A.woff2": {
              "data": "[encoded data, 14332 characters]",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "font/woff2"
                ]
              ],
              "status": 200,
              "status_text": ""
            },
            "https://localhost:6006/data/environment": {
              "data": "eyJ2ZXJzaW9uIjogIjIuMTUuMiIsICJkYXRhX2xvY2F0aW9uIjogIi9jb250ZW50L3N0YXJjb2Rlci0yLTNiLWZ0LWNvZGVhbHBhY2EvcnVucyIsICJ3aW5kb3dfdGl0bGUiOiAiIiwgImV4cGVyaW1lbnRfbmFtZSI6ICIiLCAiZXhwZXJpbWVudF9kZXNjcmlwdGlvbiI6ICIiLCAiY3JlYXRpb25fdGltZSI6IDAuMCwgImRlYnVnIjogeyJkYXRhX3Byb3ZpZGVyIjogIkdycGNEYXRhUHJvdmlkZXIoYWRkcj0nbG9jYWxob3N0OjM1MzI1JykiLCAiZmxhZ3MiOiB7ImxvZ2RpciI6ICIvY29udGVudC9zdGFyY29kZXItMi0zYi1mdC1jb2RlYWxwYWNhL3J1bnMiLCAibG9nZGlyX3NwZWMiOiAiIiwgImhvc3QiOiBudWxsLCAiYmluZF9hbGwiOiBmYWxzZSwgInBvcnQiOiBudWxsLCAicmV1c2VfcG9ydCI6IGZhbHNlLCAibG9hZF9mYXN0IjogImF1dG8iLCAiZXh0cmFfZGF0YV9zZXJ2ZXJfZmxhZ3MiOiAiIiwgImdycGNfY3JlZHNfdHlwZSI6ICJsb2NhbCIsICJncnBjX2RhdGFfcHJvdmlkZXIiOiAiIiwgInB1cmdlX29ycGhhbmVkX2RhdGEiOiB0cnVlLCAiZGIiOiAiIiwgImRiX2ltcG9ydCI6IGZhbHNlLCAiaW5zcGVjdCI6IGZhbHNlLCAidmVyc2lvbl90YiI6IGZhbHNlLCAidGFnIjogIiIsICJldmVudF9maWxlIjogIiIsICJwYXRoX3ByZWZpeCI6ICIiLCAid2luZG93X3RpdGxlIjogIiIsICJtYXhfcmVsb2FkX3RocmVhZHMiOiAxLCAicmVsb2FkX2ludGVydmFsIjogNS4wLCAicmVsb2FkX3Rhc2siOiAiYXV0byIsICJyZWxvYWRfbXVsdGlmaWxlIjogbnVsbCwgInJlbG9hZF9tdWx0aWZpbGVfaW5hY3RpdmVfc2VjcyI6IDg2NDAwLCAiZ2VuZXJpY19kYXRhIjogImF1dG8iLCAic2FtcGxlc19wZXJfcGx1Z2luIjoge30sICJkZXRlY3RfZmlsZV9yZXBsYWNlbWVudCI6IG51bGwsICJfX3RlbnNvcmJvYXJkX3N1YmNvbW1hbmQiOiAic2VydmUifX19",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/json"
                ]
              ],
              "status": 200,
              "status_text": ""
            },
            "https://localhost:6006/experiment/defaultExperimentId/data/runs": {
              "data": "W10=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/json"
                ]
              ],
              "status": 200,
              "status_text": ""
            },
            "https://localhost:6006/data/runs": {
              "data": "W10=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/json"
                ]
              ],
              "status": 200,
              "status_text": ""
            },
            "https://localhost:6006/experiment/defaultExperimentId/data/plugin/hparams/experiment?request=%7B%22experimentName%22:%22defaultExperimentId%22%7D": {
              "data": "ewogICJuYW1lIjogIiIsCiAgImRlc2NyaXB0aW9uIjogIiIsCiAgInVzZXIiOiAiIiwKICAidGltZUNyZWF0ZWRTZWNzIjogMC4wLAogICJocGFyYW1JbmZvcyI6IFtdLAogICJtZXRyaWNJbmZvcyI6IFtdCn0=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/json"
                ]
              ],
              "status": 200,
              "status_text": ""
            },
            "https://localhost:6006/experiment/defaultExperimentId/data/plugin/hparams/session_groups?request=%7B%22experimentName%22:%22defaultExperimentId%22,%22allowedStatuses%22:%5B%22STATUS_FAILURE%22,%22STATUS_RUNNING%22,%22STATUS_SUCCESS%22,%22STATUS_UNKNOWN%22%5D,%22colParams%22:%5B%5D,%22startIndex%22:0,%22sliceSize%22:1000000%7D": {
              "data": "ewogICJzZXNzaW9uR3JvdXBzIjogW10sCiAgInRvdGFsU2l6ZSI6IDAKfQ==",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/json"
                ]
              ],
              "status": 200,
              "status_text": ""
            },
            "https://localhost:6006/font-roboto/d-6IYplOFocCacKzxwXSOJBw1xU1rKptJj_0jans920.woff2": {
              "data": "[encoded data, 14352 characters]",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "font/woff2"
                ]
              ],
              "status": 200,
              "status_text": ""
            },
            "https://localhost:6006/font-roboto/vPcynSL0qHq_6dX7lKVByXYhjbSpvc47ee6xR_80Hnw.woff2": {
              "data": "[encoded data, 16472 characters]",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "font/woff2"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 821
        },
        "id": "v-JPv8sJbmLj",
        "outputId": "28902602-3586-4f37-fbef-48684aff54c6"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "        (async () => {\n",
              "            const url = new URL(await google.colab.kernel.proxyPort(6006, {'cache': true}));\n",
              "            url.searchParams.set('tensorboardColab', 'true');\n",
              "            const iframe = document.createElement('iframe');\n",
              "            iframe.src = url;\n",
              "            iframe.setAttribute('width', '100%');\n",
              "            iframe.setAttribute('height', '800');\n",
              "            iframe.setAttribute('frameborder', 0);\n",
              "            document.body.appendChild(iframe);\n",
              "        })();\n",
              "    "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.train()"
      ],
      "metadata": {
        "id": "6QMaxDmn_DiV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.push_to_hub()"
      ],
      "metadata": {
        "id": "rBk343pa_Fwa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "2w1luGJo_0Ld"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}