{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "machine_shape": "hm", "authorship_tag": "ABX9TyO11tndDoFEdL4+/FDgLad9", "include_colab_link": true }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "widgets": { "application/vnd.jupyter.widget-state+json": { "de24d272f2b842c5b01eedb3f536b810": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_0c5dab2657b2473385a424d90f3d4664", "IPY_MODEL_57efe36e546c473d8be34102f6ba9a58", "IPY_MODEL_871bad1d905d4877a9eaa242cfd54c4e" ], "layout": "IPY_MODEL_8951f6b2edf64464869391197c900f84" } }, "0c5dab2657b2473385a424d90f3d4664": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_69a61ad28d5141dcbaea44060bc5ebf7", "placeholder": "​", "style": "IPY_MODEL_76c2fbf005ae4a5790edfeb499b387b7", "value": "tokenizer.model: 100%" } }, "57efe36e546c473d8be34102f6ba9a58": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_116964f328dc45d991d895d684ac1216", "max": 493443, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_1ecec5ba4424498082a5f64cf3d7faf8", "value": 493443 } }, "871bad1d905d4877a9eaa242cfd54c4e": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_fc4edcef273b4e75894f4b512122de94", "placeholder": "​", "style": "IPY_MODEL_ca2323b142f54998985d30481d5cfabe", "value": " 493k/493k [00:00<00:00, 42.2kB/s]" } }, "8951f6b2edf64464869391197c900f84": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "69a61ad28d5141dcbaea44060bc5ebf7": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "76c2fbf005ae4a5790edfeb499b387b7": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "116964f328dc45d991d895d684ac1216": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "1ecec5ba4424498082a5f64cf3d7faf8": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "fc4edcef273b4e75894f4b512122de94": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ca2323b142f54998985d30481d5cfabe": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "63626ac2d0f546188c07512a04c71417": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_decd91747fd04ce39f3e2b733bc7f477", "IPY_MODEL_7140e4c154424fcab846a71889e99ed2", "IPY_MODEL_2264d8b75251425e94e635558af4e223" ], "layout": "IPY_MODEL_c37478198217457cb30c6649203cf4dc" } }, "decd91747fd04ce39f3e2b733bc7f477": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_4918769e4e984dfda924776e2373154c", "placeholder": "​", "style": "IPY_MODEL_9b48494c94cf49b5835489d97f7a24c5", "value": "model-00001-of-00002.safetensors: 100%" } }, "7140e4c154424fcab846a71889e99ed2": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_6ed844da52fe466eb1c10c814489448c", "max": 9942990000, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_9c60efa02e80423e828628190dd13bc3", "value": 9942990000 } }, "2264d8b75251425e94e635558af4e223": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_0170e8cc57d94041956f7afbf2eef449", "placeholder": "​", "style": "IPY_MODEL_220c2ba5f2524271b24fe049431a474c", "value": " 9.94G/9.94G [04:04<00:00, 36.9MB/s]" } }, "c37478198217457cb30c6649203cf4dc": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "4918769e4e984dfda924776e2373154c": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "9b48494c94cf49b5835489d97f7a24c5": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "6ed844da52fe466eb1c10c814489448c": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "9c60efa02e80423e828628190dd13bc3": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "0170e8cc57d94041956f7afbf2eef449": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "220c2ba5f2524271b24fe049431a474c": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "a6f99dd0662846f9a381d2d507a7b447": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_900b9fcb70a84781bd5b4213df54626d", "IPY_MODEL_0ea83f270e164795b64f23b143efb300", "IPY_MODEL_318dcdeac8fb40f88fa60114f1c6a7c1" ], "layout": "IPY_MODEL_af89cf715e0e4c5e9f59943a255394c1" } }, "900b9fcb70a84781bd5b4213df54626d": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_40e23e35299d45d499432b8f1a9bc924", "placeholder": "​", "style": "IPY_MODEL_126b374e286747768ef7218454534640", "value": "Upload 3 LFS files: 100%" } }, "0ea83f270e164795b64f23b143efb300": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_bdd26e54eed5477f99b135552e5f3450", "max": 3, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_163a6fd878134e1eb5f193d1ebfff1c1", "value": 3 } }, "318dcdeac8fb40f88fa60114f1c6a7c1": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_953d7c014f76413c9805a2ef8c2c9356", "placeholder": "​", "style": "IPY_MODEL_348879bf76d1471f9c79c1ec2dc07c1d", "value": " 3/3 [04:05<00:00, 245.46s/it]" } }, "af89cf715e0e4c5e9f59943a255394c1": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "40e23e35299d45d499432b8f1a9bc924": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "126b374e286747768ef7218454534640": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "bdd26e54eed5477f99b135552e5f3450": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "163a6fd878134e1eb5f193d1ebfff1c1": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "953d7c014f76413c9805a2ef8c2c9356": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "348879bf76d1471f9c79c1ec2dc07c1d": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "8d54ae0d028b40e7b018454187db1a1c": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_562353040be54593b23734390f49927c", "IPY_MODEL_00cbebe6df7d4995913f20e39fc71b15", "IPY_MODEL_aee3c563fdc54f9cb3ebc2630c84a9e6" ], "layout": "IPY_MODEL_b74e307a751844ffab9f7f3df367774b" } }, "562353040be54593b23734390f49927c": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_8e6142e41f714fe9abe6a5bb72c071f9", "placeholder": "​", "style": "IPY_MODEL_49cd1c5663404fb5a307c345e7e970c3", "value": "model-00002-of-00002.safetensors: 100%" } }, "00cbebe6df7d4995913f20e39fc71b15": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_920ef8e509d24ccda930f4c47eff158c", "max": 8030324832, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_c8828d61b26a47ac97a1541e14c00f62", "value": 8030324832 } }, "aee3c563fdc54f9cb3ebc2630c84a9e6": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_a3d7e352222647a99be79935b1ebd86a", "placeholder": "​", "style": "IPY_MODEL_80666ef5f07641c482a23618a767791d", "value": " 8.03G/8.03G [03:13<00:00, 54.0MB/s]" } }, "b74e307a751844ffab9f7f3df367774b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8e6142e41f714fe9abe6a5bb72c071f9": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "49cd1c5663404fb5a307c345e7e970c3": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "920ef8e509d24ccda930f4c47eff158c": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "c8828d61b26a47ac97a1541e14c00f62": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "a3d7e352222647a99be79935b1ebd86a": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "80666ef5f07641c482a23618a767791d": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "view-in-github", "colab_type": "text" }, "source": [ "\"Open" ] }, { "cell_type": "markdown", "source": [ "# Merge Large Language Models with mergekit\n", "> 🗣️ [Large Language Model Course](https://github.com/mlabonne/llm-course)\n", "\n", "❤️ Created by [@maximelabonne](https://twitter.com/maximelabonne).\n", "\n", "Model merging only requires a lot of RAM. With a free Google Colab account, you should be able to run it using a T4 GPU (VRAM offloading).\n", "\n", "Examples of merge configurations:\n", "\n", "### TIES-Merging\n", "\n", "```yaml\n", "models:\n", " - model: mistralai/Mistral-7B-v0.1\n", " # no parameters necessary for base model\n", " - model: OpenPipe/mistral-ft-optimized-1218\n", " parameters:\n", " density: 0.5\n", " weight: 0.5\n", " - model: mlabonne/NeuralHermes-2.5-Mistral-7B\n", " parameters:\n", " density: 0.5\n", " weight: 0.3\n", "merge_method: ties\n", "base_model: mistralai/Mistral-7B-v0.1\n", "parameters:\n", " normalize: true\n", "dtype: float16\n", "```\n", "\n", "You can find the final model on the Hugging Face Hub at [mlabonne/NeuralPipe-7B-ties](https://huggingface.co/mlabonne/NeuralPipe-7B-ties).\n", "\n", "### SLERP\n", "\n", "```yaml\n", "slices:\n", " - sources:\n", " - model: OpenPipe/mistral-ft-optimized-1218\n", " layer_range: [0, 32]\n", " - model: mlabonne/NeuralHermes-2.5-Mistral-7B\n", " layer_range: [0, 32]\n", "merge_method: slerp\n", "base_model: OpenPipe/mistral-ft-optimized-1218\n", "parameters:\n", " t:\n", " - filter: self_attn\n", " value: [0, 0.5, 0.3, 0.7, 1]\n", " - filter: mlp\n", " value: [1, 0.5, 0.7, 0.3, 0]\n", " - value: 0.5\n", "dtype: bfloat16\n", "```\n", "\n", "You can find the final model on the Hugging Face Hub at [mlabonne/NeuralPipe-7B-slerp](https://huggingface.co/mlabonne/NeuralPipe-7B-slerp).\n", "\n", "### Passthrough\n", "\n", "```yaml\n", "slices:\n", " - sources:\n", " - model: OpenPipe/mistral-ft-optimized-1218\n", " layer_range: [0, 32]\n", " - sources:\n", " - model: mlabonne/NeuralHermes-2.5-Mistral-7B\n", " layer_range: [24, 32]\n", "merge_method: passthrough\n", "dtype: bfloat16\n", "```\n", "\n", "You can find the final model on the Hugging Face Hub at [mlabonne/NeuralPipe-9B-merged](https://huggingface.co/mlabonne/NeuralPipe-9B-merged)." ], "metadata": { "id": "o12O0YjJvvLW" } }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "NPNPie5Eo3EZ" }, "outputs": [], "source": [ "!git clone https://github.com/cg123/mergekit.git\n", "!cd mergekit && pip install -q -e ." ] }, { "cell_type": "code", "source": [ "import yaml\n", "\n", "MODEL_NAME = \"Marcoro14-7B-slerp\"\n", "yaml_config = \"\"\"\n", "slices:\n", " - sources:\n", " - model: AIDC-ai-business/Marcoroni-7B-v3\n", " layer_range: [0, 32]\n", " - model: EmbeddedLLM/Mistral-7B-Merge-14-v0.1\n", " layer_range: [0, 32]\n", "merge_method: slerp\n", "base_model: AIDC-ai-business/Marcoroni-7B-v3\n", "parameters:\n", " t:\n", " - filter: self_attn\n", " value: [0, 0.5, 0.3, 0.7, 1]\n", " - filter: mlp\n", " value: [1, 0.5, 0.7, 0.3, 0]\n", " - value: 0.5\n", "dtype: bfloat16\n", "\n", "\"\"\"\n", "\n", "# Save config as yaml file\n", "with open('config.yaml', 'w', encoding=\"utf-8\") as f:\n", " f.write(yaml_config)" ], "metadata": { "id": "LGd7jlfCpNcg" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# Merge models\n", "!mergekit-yaml config.yaml merge --copy-tokenizer --allow-crimes --out-shard-size 1B --lazy-unpickle" ], "metadata": { "id": "d5mYzDo1q96y" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "!pip install -qU huggingface_hub\n", "\n", "from huggingface_hub import ModelCard, ModelCardData\n", "from jinja2 import Template\n", "\n", "username = \"mlabonne\"\n", "\n", "template_text = \"\"\"\n", "---\n", "license: apache-2.0\n", "tags:\n", "- merge\n", "- mergekit\n", "- lazymergekit\n", "{%- for model in models %}\n", "- {{ model }}\n", "{%- endfor %}\n", "---\n", "\n", "# {{ model_name }}\n", "\n", "{{ model_name }} is a merge of the following models using [mergekit](https://github.com/cg123/mergekit):\n", "\n", "{%- for model in models %}\n", "* [{{ model }}](https://huggingface.co/{{ model }})\n", "{%- endfor %}\n", "\n", "## 🧩 Configuration\n", "\n", "```yaml\n", "{{- yaml_config -}}\n", "```\n", "\"\"\"\n", "\n", "# Create a Jinja template object\n", "jinja_template = Template(template_text.strip())\n", "\n", "# Get list of models from config\n", "data = yaml.safe_load(yaml_config)\n", "if \"models\" in data:\n", " models = [data[\"models\"][i][\"model\"] for i in range(len(data[\"models\"])) if \"parameters\" in data[\"models\"][i]]\n", "elif \"parameters\" in data:\n", " models = [data[\"slices\"][0][\"sources\"][i][\"model\"] for i in range(len(data[\"slices\"][0][\"sources\"]))]\n", "elif \"slices\" in data:\n", " models = [data[\"slices\"][i][\"sources\"][0][\"model\"] for i in range(len(data[\"slices\"]))]\n", "else:\n", " raise Exception(\"No models or slices found in yaml config\")\n", "\n", "# Fill the template\n", "content = jinja_template.render(\n", " model_name=MODEL_NAME,\n", " models=models,\n", " yaml_config=yaml_config,\n", " username=username,\n", ")\n", "\n", "# Save the model card\n", "card = ModelCard(content)\n", "card.save('merge/README.md')" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "w-RNKev373lI", "outputId": "fccbbd1d-295f-4def-a398-f226813294bb" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/330.1 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[91m━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m112.6/330.1 kB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m330.1/330.1 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h" ] } ] }, { "cell_type": "code", "source": [ "from google.colab import userdata\n", "from huggingface_hub import HfApi\n", "\n", "username = \"mlabonne\"\n", "\n", "# Defined in the secrets tab in Google Colab\n", "api = HfApi(token=userdata.get(\"HF_TOKEN\"))\n", "\n", "api.create_repo(\n", " repo_id=f\"{username}/{MODEL_NAME}\",\n", " repo_type=\"model\"\n", ")\n", "api.upload_folder(\n", " repo_id=f\"{username}/{MODEL_NAME}\",\n", " folder_path=\"merge\",\n", ")" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 164, "referenced_widgets": [ "de24d272f2b842c5b01eedb3f536b810", "0c5dab2657b2473385a424d90f3d4664", "57efe36e546c473d8be34102f6ba9a58", "871bad1d905d4877a9eaa242cfd54c4e", "8951f6b2edf64464869391197c900f84", "69a61ad28d5141dcbaea44060bc5ebf7", "76c2fbf005ae4a5790edfeb499b387b7", "116964f328dc45d991d895d684ac1216", "1ecec5ba4424498082a5f64cf3d7faf8", "fc4edcef273b4e75894f4b512122de94", "ca2323b142f54998985d30481d5cfabe", "63626ac2d0f546188c07512a04c71417", "decd91747fd04ce39f3e2b733bc7f477", "7140e4c154424fcab846a71889e99ed2", "2264d8b75251425e94e635558af4e223", "c37478198217457cb30c6649203cf4dc", "4918769e4e984dfda924776e2373154c", "9b48494c94cf49b5835489d97f7a24c5", "6ed844da52fe466eb1c10c814489448c", "9c60efa02e80423e828628190dd13bc3", "0170e8cc57d94041956f7afbf2eef449", "220c2ba5f2524271b24fe049431a474c", "a6f99dd0662846f9a381d2d507a7b447", "900b9fcb70a84781bd5b4213df54626d", "0ea83f270e164795b64f23b143efb300", "318dcdeac8fb40f88fa60114f1c6a7c1", "af89cf715e0e4c5e9f59943a255394c1", "40e23e35299d45d499432b8f1a9bc924", "126b374e286747768ef7218454534640", "bdd26e54eed5477f99b135552e5f3450", "163a6fd878134e1eb5f193d1ebfff1c1", "953d7c014f76413c9805a2ef8c2c9356", "348879bf76d1471f9c79c1ec2dc07c1d", "8d54ae0d028b40e7b018454187db1a1c", "562353040be54593b23734390f49927c", "00cbebe6df7d4995913f20e39fc71b15", "aee3c563fdc54f9cb3ebc2630c84a9e6", "b74e307a751844ffab9f7f3df367774b", "8e6142e41f714fe9abe6a5bb72c071f9", "49cd1c5663404fb5a307c345e7e970c3", "920ef8e509d24ccda930f4c47eff158c", "c8828d61b26a47ac97a1541e14c00f62", "a3d7e352222647a99be79935b1ebd86a", "80666ef5f07641c482a23618a767791d" ] }, "id": "ik0V0dF55gfU", "outputId": "9f6c605c-6b51-473d-c1fa-b103e9208785" }, "execution_count": null, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "tokenizer.model: 0%| | 0.00/493k [00:00