{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [] }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "widgets": { "application/vnd.jupyter.widget-state+json": { "42279f0aaf454588a7f5ab4126f03f1f": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_9c04c4d9d6ed406d9ff488050806d15a", "IPY_MODEL_9d61c13f30af4391bf2a3012f9871e9e", "IPY_MODEL_9a7883cf26e1423bb216ae85aa271edf" ], "layout": "IPY_MODEL_f34ed1e1d2bc438795b1c8fd78a9e72a" } }, "9c04c4d9d6ed406d9ff488050806d15a": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_7aeb12b1a3944acdb629b75db8237899", "placeholder": "", "style": "IPY_MODEL_0d4eaf03c87a45b69dd0f9c3aad12d56", "value": "Downloading (…)lve/main/config.json: 100%" } }, "9d61c13f30af4391bf2a3012f9871e9e": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_cec7e4851a104479b6dbce556061d1b3", "max": 385, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_2142fefcd627477bbd2b83ac59930cc5", "value": 385 } }, "9a7883cf26e1423bb216ae85aa271edf": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_229b382acaa540cc9fdacbe8f1d388b0", "placeholder": "", "style": "IPY_MODEL_f8e8b814c568465ebdf95a8536005de5", "value": " 385/385 [00:00<00:00, 5.80kB/s]" } }, "f34ed1e1d2bc438795b1c8fd78a9e72a": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "7aeb12b1a3944acdb629b75db8237899": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "0d4eaf03c87a45b69dd0f9c3aad12d56": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "cec7e4851a104479b6dbce556061d1b3": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "2142fefcd627477bbd2b83ac59930cc5": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "229b382acaa540cc9fdacbe8f1d388b0": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f8e8b814c568465ebdf95a8536005de5": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "526d0e6c57774754a343be6b0525ba19": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_a3f707f48a59480faac65f7314fafd47", "IPY_MODEL_cc17d31a4a97417d8a8773e9f0f20c5a", "IPY_MODEL_d3003bb831074b22a29256cc9b96df67" ], "layout": "IPY_MODEL_74602d4b276e4ce6a595f0956af6a85f" } }, "a3f707f48a59480faac65f7314fafd47": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e0acabc0e90a48f2bc08b4aeaf1a6331", "placeholder": "", "style": "IPY_MODEL_4fd3ebf062454067b7a7f888384ac873", "value": "Downloading pytorch_model.bin: 100%" } }, "cc17d31a4a97417d8a8773e9f0f20c5a": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_2763d0cc0a1d430480540f7eea790c35", "max": 440474434, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_b174573848e54b14a878f8f4ee1637c8", "value": 440474434 } }, "d3003bb831074b22a29256cc9b96df67": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f024455a82364bee980c79b91b31e023", "placeholder": "", "style": "IPY_MODEL_424f3fc993d649e597d099b10f51de20", "value": " 440M/440M [00:05<00:00, 121MB/s]" } }, "74602d4b276e4ce6a595f0956af6a85f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e0acabc0e90a48f2bc08b4aeaf1a6331": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "4fd3ebf062454067b7a7f888384ac873": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "2763d0cc0a1d430480540f7eea790c35": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "b174573848e54b14a878f8f4ee1637c8": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "f024455a82364bee980c79b91b31e023": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "424f3fc993d649e597d099b10f51de20": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "c0c47a877e664f4e81b9238c0623bd69": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_a89c457c6f904dd59f013ef048512841", "IPY_MODEL_b6afb2bed86c41599f934b04b1d45d63", "IPY_MODEL_cf5f5635be774821bdf720af21dc1fc3" ], "layout": "IPY_MODEL_193d1fffdf504cd5a610fd7e36de48b0" } }, "a89c457c6f904dd59f013ef048512841": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_59f2dd49767248d9bc7287c510584386", "placeholder": "", "style": "IPY_MODEL_b9d60cf5b653439e8df60b592fc5059e", "value": "Downloading (…)okenizer_config.json: 100%" } }, "b6afb2bed86c41599f934b04b1d45d63": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_c8d3ac3cb2c74f53bd8f68d82ea60f6c", "max": 28, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_ccb64052caed4ed19cadd79e09d97972", "value": 28 } }, "cf5f5635be774821bdf720af21dc1fc3": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_8e55e35a08b6419d9edcfcb2a81fd99f", "placeholder": "", "style": "IPY_MODEL_0ffe6a6a792b4233b56a71b51726555f", "value": " 28.0/28.0 [00:00<00:00, 182B/s]" } }, "193d1fffdf504cd5a610fd7e36de48b0": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "59f2dd49767248d9bc7287c510584386": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "b9d60cf5b653439e8df60b592fc5059e": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "c8d3ac3cb2c74f53bd8f68d82ea60f6c": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ccb64052caed4ed19cadd79e09d97972": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "8e55e35a08b6419d9edcfcb2a81fd99f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "0ffe6a6a792b4233b56a71b51726555f": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "e497e3fb759d4f49bce357053a6a4457": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_5195a990292848e5bc38c12fb8542553", "IPY_MODEL_cc1f35ae39944004b3915cae663c51cb", "IPY_MODEL_ce5ab983090e41098444eb36c397e027" ], "layout": "IPY_MODEL_0b8321428edc4155a6fd096385609b54" } }, "5195a990292848e5bc38c12fb8542553": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_9c2b0e5a4032410e9d05a360feec5209", "placeholder": "", "style": "IPY_MODEL_73997d55a3a144abb1e4c43dc97e320b", "value": "Downloading (…)solve/main/vocab.txt: 100%" } }, "cc1f35ae39944004b3915cae663c51cb": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_1d3d15f8ccca44cbb64a06ddb60017e4", "max": 225062, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_116d37feb437461ea169556cfc6b621d", "value": 225062 } }, "ce5ab983090e41098444eb36c397e027": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e1fefcd310b6497395058972dfd448b9", "placeholder": "", "style": "IPY_MODEL_e75642fb03134b69ae00adb4febabd6b", "value": " 225k/225k [00:00<00:00, 958kB/s]" } }, "0b8321428edc4155a6fd096385609b54": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "9c2b0e5a4032410e9d05a360feec5209": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "73997d55a3a144abb1e4c43dc97e320b": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "1d3d15f8ccca44cbb64a06ddb60017e4": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "116d37feb437461ea169556cfc6b621d": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "e1fefcd310b6497395058972dfd448b9": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e75642fb03134b69ae00adb4febabd6b": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "cells": [ { "cell_type": "markdown", "source": [ "### Clone BiomedCLIP repo\n", "\n", "Will be merged to the open_clip main later" ], "metadata": { "id": "e4ymt2jdGuPx" } }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "J4N1Au3wz10u", "outputId": "b995f3dd-22d8-47d8-ceca-c203e4c18c59" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Cloning into 'open_clip'...\n", "remote: Enumerating objects: 2331, done.\u001b[K\n", "remote: Counting objects: 100% (8/8), done.\u001b[K\n", "remote: Compressing objects: 100% (8/8), done.\u001b[K\n", "remote: Total 2331 (delta 0), reused 3 (delta 0), pack-reused 2323\u001b[K\n", "Receiving objects: 100% (2331/2331), 7.92 MiB | 16.69 MiB/s, done.\n", "Resolving deltas: 100% (1386/1386), done.\n" ] } ], "source": [ "!git clone -b naotous/biomed_clip_224px https://github.com/usuyama/open_clip.git" ] }, { "cell_type": "markdown", "source": [ "### Install libraries\n", "**Make sure to restart the Colab runtime after installation**\n", "\n", "Colab Menu -> Runtime -> Restart runtime" ], "metadata": { "id": "t8BYG2CFF6wD" } }, { "cell_type": "code", "source": [ "!pip install -e ./open_clip transformers" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "nUAkSVJ90DQs", "outputId": "519b19b5-f712-458f-bb06-ae8556a5241c" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", "Obtaining file:///content/open_clip\n", " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", "Collecting transformers\n", " Downloading transformers-4.27.3-py3-none-any.whl (6.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.22.4)\n", "Collecting huggingface-hub<1.0,>=0.11.0\n", " Downloading huggingface_hub-0.13.3-py3-none-any.whl (199 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.8/199.8 KB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.10.0)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.0)\n", "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.65.0)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (6.0)\n", "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n", " Downloading tokenizers-0.13.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.27.1)\n", "Requirement already satisfied: torch>=1.9.0 in /usr/local/lib/python3.9/dist-packages (from open-clip-torch==2.16.0) (1.13.1+cu116)\n", "Requirement already satisfied: torchvision in /usr/local/lib/python3.9/dist-packages (from open-clip-torch==2.16.0) (0.14.1+cu116)\n", "Collecting ftfy\n", " Downloading ftfy-6.1.1-py3-none-any.whl (53 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.1/53.1 KB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting sentencepiece\n", " Downloading sentencepiece-0.1.97-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m19.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: protobuf<4 in /usr/local/lib/python3.9/dist-packages (from open-clip-torch==2.16.0) (3.19.6)\n", "Collecting timm\n", " Downloading timm-0.6.12-py3-none-any.whl (549 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m549.1/549.1 KB\u001b[0m \u001b[31m14.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n", "Requirement already satisfied: wcwidth>=0.2.5 in /usr/local/lib/python3.9/dist-packages (from ftfy->open-clip-torch==2.16.0) (0.2.6)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (3.4)\n", "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2.0.12)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2022.12.7)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.15)\n", "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.9/dist-packages (from torchvision->open-clip-torch==2.16.0) (8.4.0)\n", "Installing collected packages: tokenizers, sentencepiece, ftfy, huggingface-hub, transformers, timm, open-clip-torch\n", " Running setup.py develop for open-clip-torch\n", "Successfully installed ftfy-6.1.1 huggingface-hub-0.13.3 open-clip-torch-2.16.0 sentencepiece-0.1.97 timm-0.6.12 tokenizers-0.13.2 transformers-4.27.3\n" ] } ] }, { "cell_type": "markdown", "source": [ "### Setup AzCopy for downloading files from our Azure storage" ], "metadata": { "id": "-8l0giuDGUX-" } }, { "cell_type": "code", "source": [ "!wget -O azcopy_linux.tar.gz https://aka.ms/downloadazcopy-v10-linux\n", "!tar -xf azcopy_linux.tar.gz --strip-components=1 --wildcards '*/azcopy'\n", "!./azcopy --version" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "dEB_ONMs0qn7", "outputId": "3ece3f9b-c283-463c-890e-6e810e5af5ac" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "--2023-03-23 20:23:27-- https://aka.ms/downloadazcopy-v10-linux\n", "Resolving aka.ms (aka.ms)... 23.12.134.45\n", "Connecting to aka.ms (aka.ms)|23.12.134.45|:443... connected.\n", "HTTP request sent, awaiting response... 301 Moved Permanently\n", "Location: https://azcopyvnext.azureedge.net/release20230123/azcopy_linux_amd64_10.17.0.tar.gz [following]\n", "--2023-03-23 20:23:28-- https://azcopyvnext.azureedge.net/release20230123/azcopy_linux_amd64_10.17.0.tar.gz\n", "Resolving azcopyvnext.azureedge.net (azcopyvnext.azureedge.net)... 23.213.34.169, 23.213.34.191, 2600:1409:9800:21::17d8:93cb, ...\n", "Connecting to azcopyvnext.azureedge.net (azcopyvnext.azureedge.net)|23.213.34.169|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 13494500 (13M) [application/gzip]\n", "Saving to: ‘azcopy_linux.tar.gz’\n", "\n", "azcopy_linux.tar.gz 100%[===================>] 12.87M --.-KB/s in 0.09s \n", "\n", "2023-03-23 20:23:28 (136 MB/s) - ‘azcopy_linux.tar.gz’ saved [13494500/13494500]\n", "\n", "azcopy version 10.17.0\n" ] } ] }, { "cell_type": "markdown", "source": [ "### Download the model checkpoint and example images" ], "metadata": { "id": "G4FRMeHQHW3c" } }, { "cell_type": "code", "source": [ "!./azcopy copy --recursive \"https://hanoverdev.blob.core.windows.net/biomed-clip-share?sv=2021-10-04&st=2023-03-22T22%3A14%3A19Z&se=2024-03-23T22%3A14%3A00Z&sr=c&sp=rl&sig=lau70KJwG9ddkITH1CGhWv%2FDpRFKTpansQltuVWFhNY%3D\" ." ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "FfuizqV60V1Q", "outputId": "3ccd96dc-8ef1-4399-cf7e-38154c43d395" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "INFO: Scanning...\n", "INFO: Any empty folders will not be processed, because source and/or destination doesn't have full folder support\n", "\n", "Job 437698c5-72d3-6744-6cd2-d08967f0f4ab has started\n", "Log file is located at: /root/.azcopy/437698c5-72d3-6744-6cd2-d08967f0f4ab.log\n", "\n", "100.0 %, 10 Done, 0 Failed, 0 Pending, 0 Skipped, 10 Total, \n", "\n", "\n", "Job 437698c5-72d3-6744-6cd2-d08967f0f4ab summary\n", "Elapsed Time (Minutes): 0.2336\n", "Number of File Transfers: 10\n", "Number of Folder Property Transfers: 0\n", "Total Number of Transfers: 10\n", "Number of File Transfers Completed: 10\n", "Number of Folder Transfers Completed: 0\n", "Number of File Transfers Failed: 0\n", "Number of Folder Transfers Failed: 0\n", "Number of File Transfers Skipped: 0\n", "Number of Folder Transfers Skipped: 0\n", "TotalBytesTransferred: 2353110162\n", "Final Job Status: Completed\n", "\n" ] } ] }, { "cell_type": "code", "source": [ "!ls biomed-clip-share" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "4WOxBdKr0e_m", "outputId": "468cd91f-9f81-464c-b747-f88d56960560" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "example_data models\n" ] } ] }, { "cell_type": "markdown", "source": [ "### Load BiomedCLIP model" ], "metadata": { "id": "ZQ7D3zXKGcaM" } }, { "cell_type": "code", "source": [ "import glob\n", "from collections import OrderedDict\n", "\n", "import torch\n", "from PIL import Image\n", "import open_clip\n", "\n", "# Timm contiual pretraining\n", "model_name = 'PubMedBERT_256-timm-vit_base_patch16_224'\n", "checkpoint = 'biomed-clip-share/models/2022_11_08-07_39_28-model_timm-vit_base_patch16_224-lr_0.0005-b_1024-j_8-p_amp/checkpoints/epoch_32.pt'\n", "\n", "model, transform_train, transform_val = open_clip.create_model_and_transforms(model_name)\n", "checkpoint = torch.load(checkpoint, map_location=\"cpu\")\n", "\n", "tokenizer = open_clip.get_tokenizer(model_name)\n", "context_length = 256\n", "\n", "new_state_dict = OrderedDict()\n", "for k,v in checkpoint['state_dict'].items():\n", " new_k = k.replace('module.', '')\n", " new_state_dict[new_k] = v\n", "model.load_state_dict(new_state_dict, strict=False) # can set this to be true except for timm models" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 236, "referenced_widgets": [ "42279f0aaf454588a7f5ab4126f03f1f", "9c04c4d9d6ed406d9ff488050806d15a", "9d61c13f30af4391bf2a3012f9871e9e", "9a7883cf26e1423bb216ae85aa271edf", "f34ed1e1d2bc438795b1c8fd78a9e72a", "7aeb12b1a3944acdb629b75db8237899", "0d4eaf03c87a45b69dd0f9c3aad12d56", "cec7e4851a104479b6dbce556061d1b3", "2142fefcd627477bbd2b83ac59930cc5", "229b382acaa540cc9fdacbe8f1d388b0", "f8e8b814c568465ebdf95a8536005de5", "526d0e6c57774754a343be6b0525ba19", "a3f707f48a59480faac65f7314fafd47", "cc17d31a4a97417d8a8773e9f0f20c5a", "d3003bb831074b22a29256cc9b96df67", "74602d4b276e4ce6a595f0956af6a85f", "e0acabc0e90a48f2bc08b4aeaf1a6331", "4fd3ebf062454067b7a7f888384ac873", "2763d0cc0a1d430480540f7eea790c35", "b174573848e54b14a878f8f4ee1637c8", "f024455a82364bee980c79b91b31e023", "424f3fc993d649e597d099b10f51de20", "c0c47a877e664f4e81b9238c0623bd69", "a89c457c6f904dd59f013ef048512841", "b6afb2bed86c41599f934b04b1d45d63", "cf5f5635be774821bdf720af21dc1fc3", "193d1fffdf504cd5a610fd7e36de48b0", "59f2dd49767248d9bc7287c510584386", "b9d60cf5b653439e8df60b592fc5059e", "c8d3ac3cb2c74f53bd8f68d82ea60f6c", "ccb64052caed4ed19cadd79e09d97972", "8e55e35a08b6419d9edcfcb2a81fd99f", "0ffe6a6a792b4233b56a71b51726555f", "e497e3fb759d4f49bce357053a6a4457", "5195a990292848e5bc38c12fb8542553", "cc1f35ae39944004b3915cae663c51cb", "ce5ab983090e41098444eb36c397e027", "0b8321428edc4155a6fd096385609b54", "9c2b0e5a4032410e9d05a360feec5209", "73997d55a3a144abb1e4c43dc97e320b", "1d3d15f8ccca44cbb64a06ddb60017e4", "116d37feb437461ea169556cfc6b621d", "e1fefcd310b6497395058972dfd448b9", "e75642fb03134b69ae00adb4febabd6b" ] }, "id": "v88W1wY01oxB", "outputId": "0d163c47-ae2d-4c6e-f523-bea45400123f" }, "execution_count": null, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "Downloading (…)lve/main/config.json: 0%| | 0.00/385 [00:00, ?B/s]" ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "42279f0aaf454588a7f5ab4126f03f1f" } }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": [ "Downloading pytorch_model.bin: 0%| | 0.00/440M [00:00, ?B/s]" ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "526d0e6c57774754a343be6b0525ba19" } }, "metadata": {} }, { "output_type": "stream", "name": "stderr", "text": [ "Some weights of the model checkpoint at microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract were not used when initializing BertModel: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.bias']\n", "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "Downloading (…)okenizer_config.json: 0%| | 0.00/28.0 [00:00, ?B/s]" ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "c0c47a877e664f4e81b9238c0623bd69" } }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": [ "Downloading (…)solve/main/vocab.txt: 0%| | 0.00/225k [00:00, ?B/s]" ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "e497e3fb759d4f49bce357053a6a4457" } }, "metadata": {} }, { "output_type": "execute_result", "data": { "text/plain": [ "_IncompatibleKeys(missing_keys=['text.transformer.pooler.dense.weight', 'text.transformer.pooler.dense.bias'], unexpected_keys=[])" ] }, "metadata": {}, "execution_count": 1 } ] }, { "cell_type": "markdown", "source": [ "### Example: Zero-shot classifications" ], "metadata": { "id": "_11A5zFuGfkG" } }, { "cell_type": "code", "source": [ "dataset_path = 'biomed-clip-share/example_data/biomed_image_classification_example_data'\n", "template = 'this is a photo of '\n", "labels = [\n", " 'adenocarcinoma histopathology',\n", " 'brain MRI',\n", " 'covid line chart',\n", " 'squamous cell carcinoma histopathology',\n", " 'immunohistochemistry histopathology',\n", " 'bone X-ray',\n", " 'chest X-ray',\n", " 'pie chart',\n", " 'hematoxylin and eosin histopathology'\n", "]\n", "\n", "import glob\n", "test_imgs = glob.glob(dataset_path + '/*')\n", "\n", "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n", "model.to(device)\n", "model.eval()\n", "\n", "images = torch.stack([transform_val(Image.open(img)) for img in test_imgs]).to(device)\n", "texts = tokenizer([template + l for l in labels], context_length=context_length).to(device)\n", "with torch.no_grad():\n", " image_features, text_features, logit_scale = model(images, texts)\n", "\n", " logits = (logit_scale * image_features @ text_features.t()).detach().softmax(dim=-1)\n", " sorted_indices = torch.argsort(logits, dim=-1, descending=True)\n", "\n", " logits = logits.cpu().numpy()\n", " sorted_indices = sorted_indices.cpu().numpy()\n", "\n", "top_k = -1\n", "\n", "for i, img in enumerate(test_imgs):\n", " pred = labels[sorted_indices[i][0]]\n", "\n", " top_k = len(labels) if top_k == -1 else top_k\n", " print(img.split('/')[-1] + ':')\n", " for j in range(top_k):\n", " jth_index = sorted_indices[i][j]\n", " print(f'{labels[jth_index]}: {logits[i][jth_index]}')\n", " print('\\n')" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "XSJw7Qpm1w-f", "outputId": "08bbc1cd-c6b2-45aa-a4a2-759e0f0dd844" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "adenocarcinoma_histopathology.jpg:\n", "adenocarcinoma histopathology: 0.7818863987922668\n", "hematoxylin and eosin histopathology: 0.15517690777778625\n", "immunohistochemistry histopathology: 0.06149514392018318\n", "squamous cell carcinoma histopathology: 0.0014182085869833827\n", "chest X-ray: 2.017213228100445e-05\n", "brain MRI: 1.2181524198240368e-06\n", "pie chart: 7.932688959044754e-07\n", "bone X-ray: 7.436410101036017e-07\n", "covid line chart: 4.482610052036762e-07\n", "\n", "\n", "covid_line_chart.png:\n", "covid line chart: 0.9493210315704346\n", "adenocarcinoma histopathology: 0.01898195780813694\n", "squamous cell carcinoma histopathology: 0.0175501499325037\n", "immunohistochemistry histopathology: 0.006791787222027779\n", "hematoxylin and eosin histopathology: 0.003417333820834756\n", "brain MRI: 0.002629919210448861\n", "chest X-ray: 0.0010041205678135157\n", "bone X-ray: 0.00024685842799954116\n", "pie chart: 5.6812208640621975e-05\n", "\n", "\n", "bone_X-ray.jpg:\n", "bone X-ray: 0.9037961959838867\n", "hematoxylin and eosin histopathology: 0.07279316335916519\n", "brain MRI: 0.013534954749047756\n", "chest X-ray: 0.00821212213486433\n", "immunohistochemistry histopathology: 0.001647887285798788\n", "squamous cell carcinoma histopathology: 1.418814281350933e-05\n", "covid line chart: 1.1351590956110158e-06\n", "adenocarcinoma histopathology: 2.3802124360372545e-07\n", "pie chart: 9.433303205241828e-08\n", "\n", "\n", "pie_chart.png:\n", "pie chart: 0.999992847442627\n", "covid line chart: 6.056906840967713e-06\n", "brain MRI: 6.212158041307703e-07\n", "bone X-ray: 1.870277799298492e-07\n", "chest X-ray: 1.4315827456812258e-07\n", "immunohistochemistry histopathology: 7.397970591682679e-08\n", "hematoxylin and eosin histopathology: 1.3329795045535775e-08\n", "adenocarcinoma histopathology: 7.695367898463701e-09\n", "squamous cell carcinoma histopathology: 4.512833662317917e-09\n", "\n", "\n", "H_and_E_histopathology.jpg:\n", "hematoxylin and eosin histopathology: 0.7953251600265503\n", "immunohistochemistry histopathology: 0.19779996573925018\n", "chest X-ray: 0.005973907187581062\n", "bone X-ray: 0.0008049230673350394\n", "adenocarcinoma histopathology: 9.133991261478513e-05\n", "squamous cell carcinoma histopathology: 3.6423973597266013e-06\n", "brain MRI: 6.688684948130685e-07\n", "pie chart: 4.278819574210502e-07\n", "covid line chart: 3.051619401617245e-08\n", "\n", "\n", "brain_MRI.jpg:\n", "brain MRI: 0.9565795660018921\n", "hematoxylin and eosin histopathology: 0.041418157517910004\n", "immunohistochemistry histopathology: 0.0019450499676167965\n", "pie chart: 2.7151252652402036e-05\n", "squamous cell carcinoma histopathology: 1.0223812751064543e-05\n", "bone X-ray: 8.662499567435589e-06\n", "chest X-ray: 7.96773747424595e-06\n", "adenocarcinoma histopathology: 2.7692055937222904e-06\n", "covid line chart: 3.420084908611898e-07\n", "\n", "\n", "chest_X-ray.jpg:\n", "chest X-ray: 0.9998347759246826\n", "hematoxylin and eosin histopathology: 0.0001205605294671841\n", "bone X-ray: 4.112880560569465e-05\n", "immunohistochemistry histopathology: 1.0486423889233265e-06\n", "adenocarcinoma histopathology: 9.66637117016944e-07\n", "covid line chart: 9.508977996119938e-07\n", "brain MRI: 3.232386518448038e-07\n", "squamous cell carcinoma histopathology: 2.53368597213921e-07\n", "pie chart: 3.6984038054299617e-09\n", "\n", "\n", "squamous_cell_carcinoma_histopathology.jpeg:\n", "squamous cell carcinoma histopathology: 0.9469489455223083\n", "adenocarcinoma histopathology: 0.05259034037590027\n", "hematoxylin and eosin histopathology: 0.0003988408425357193\n", "immunohistochemistry histopathology: 6.187965482240543e-05\n", "chest X-ray: 1.4099594380923008e-08\n", "pie chart: 3.522500624519864e-10\n", "bone X-ray: 2.9633814846441453e-10\n", "brain MRI: 1.2720452469139332e-10\n", "covid line chart: 1.8425603924565603e-12\n", "\n", "\n", "IHC_histopathology.jpg:\n", "immunohistochemistry histopathology: 0.9465934634208679\n", "hematoxylin and eosin histopathology: 0.03232448548078537\n", "brain MRI: 0.020657211542129517\n", "adenocarcinoma histopathology: 0.000304735847748816\n", "bone X-ray: 4.5735167077509686e-05\n", "squamous cell carcinoma histopathology: 3.150868360535242e-05\n", "covid line chart: 2.0559578842949122e-05\n", "chest X-ray: 1.2715442608168814e-05\n", "pie chart: 9.55282575887395e-06\n", "\n", "\n" ] } ] }, { "cell_type": "markdown", "source": [ "
\n", "adenocarcinoma_histopathology.jpg:\n", "adenocarcinoma histopathology: 0.7818863987922668\n", "hematoxylin and eosin histopathology: 0.15517690777778625\n", "immunohistochemistry histopathology: 0.06149514392018318\n", "squamous cell carcinoma histopathology: 0.0014182085869833827\n", "chest X-ray: 2.017213228100445e-05\n", "brain MRI: 1.2181524198240368e-06\n", "pie chart: 7.932688959044754e-07\n", "bone X-ray: 7.436410101036017e-07\n", "covid line chart: 4.482610052036762e-07\n", "\n", "\n", "covid_line_chart.png:\n", "covid line chart: 0.9493210315704346\n", "adenocarcinoma histopathology: 0.01898195780813694\n", "squamous cell carcinoma histopathology: 0.0175501499325037\n", "immunohistochemistry histopathology: 0.006791787222027779\n", "hematoxylin and eosin histopathology: 0.003417333820834756\n", "brain MRI: 0.002629919210448861\n", "chest X-ray: 0.0010041205678135157\n", "bone X-ray: 0.00024685842799954116\n", "pie chart: 5.6812208640621975e-05\n", "\n", "\n", "bone_X-ray.jpg:\n", "bone X-ray: 0.9037961959838867\n", "hematoxylin and eosin histopathology: 0.07279316335916519\n", "brain MRI: 0.013534954749047756\n", "chest X-ray: 0.00821212213486433\n", "immunohistochemistry histopathology: 0.001647887285798788\n", "squamous cell carcinoma histopathology: 1.418814281350933e-05\n", "covid line chart: 1.1351590956110158e-06\n", "adenocarcinoma histopathology: 2.3802124360372545e-07\n", "pie chart: 9.433303205241828e-08\n", "\n", "\n", "pie_chart.png:\n", "pie chart: 0.999992847442627\n", "covid line chart: 6.056906840967713e-06\n", "brain MRI: 6.212158041307703e-07\n", "bone X-ray: 1.870277799298492e-07\n", "chest X-ray: 1.4315827456812258e-07\n", "immunohistochemistry histopathology: 7.397970591682679e-08\n", "hematoxylin and eosin histopathology: 1.3329795045535775e-08\n", "adenocarcinoma histopathology: 7.695367898463701e-09\n", "squamous cell carcinoma histopathology: 4.512833662317917e-09\n", "\n", "\n", "H_and_E_histopathology.jpg:\n", "hematoxylin and eosin histopathology: 0.7953251600265503\n", "immunohistochemistry histopathology: 0.19779996573925018\n", "chest X-ray: 0.005973907187581062\n", "bone X-ray: 0.0008049230673350394\n", "adenocarcinoma histopathology: 9.133991261478513e-05\n", "squamous cell carcinoma histopathology: 3.6423973597266013e-06\n", "brain MRI: 6.688684948130685e-07\n", "pie chart: 4.278819574210502e-07\n", "covid line chart: 3.051619401617245e-08\n", "\n", "\n", "brain_MRI.jpg:\n", "brain MRI: 0.9565795660018921\n", "hematoxylin and eosin histopathology: 0.041418157517910004\n", "immunohistochemistry histopathology: 0.0019450499676167965\n", "pie chart: 2.7151252652402036e-05\n", "squamous cell carcinoma histopathology: 1.0223812751064543e-05\n", "bone X-ray: 8.662499567435589e-06\n", "chest X-ray: 7.96773747424595e-06\n", "adenocarcinoma histopathology: 2.7692055937222904e-06\n", "covid line chart: 3.420084908611898e-07\n", "\n", "\n", "chest_X-ray.jpg:\n", "chest X-ray: 0.9998347759246826\n", "hematoxylin and eosin histopathology: 0.0001205605294671841\n", "bone X-ray: 4.112880560569465e-05\n", "immunohistochemistry histopathology: 1.0486423889233265e-06\n", "adenocarcinoma histopathology: 9.66637117016944e-07\n", "covid line chart: 9.508977996119938e-07\n", "brain MRI: 3.232386518448038e-07\n", "squamous cell carcinoma histopathology: 2.53368597213921e-07\n", "pie chart: 3.6984038054299617e-09\n", "\n", "\n", "squamous_cell_carcinoma_histopathology.jpeg:\n", "squamous cell carcinoma histopathology: 0.9469489455223083\n", "adenocarcinoma histopathology: 0.05259034037590027\n", "hematoxylin and eosin histopathology: 0.0003988408425357193\n", "immunohistochemistry histopathology: 6.187965482240543e-05\n", "chest X-ray: 1.4099594380923008e-08\n", "pie chart: 3.522500624519864e-10\n", "bone X-ray: 2.9633814846441453e-10\n", "brain MRI: 1.2720452469139332e-10\n", "covid line chart: 1.8425603924565603e-12\n", "\n", "\n", "IHC_histopathology.jpg:\n", "immunohistochemistry histopathology: 0.9465934634208679\n", "hematoxylin and eosin histopathology: 0.03232448548078537\n", "brain MRI: 0.020657211542129517\n", "adenocarcinoma histopathology: 0.000304735847748816\n", "bone X-ray: 4.5735167077509686e-05\n", "squamous cell carcinoma histopathology: 3.150868360535242e-05\n", "covid line chart: 2.0559578842949122e-05\n", "chest X-ray: 1.2715442608168814e-05\n", "pie chart: 9.55282575887395e-06\n", "\n", "