diff --git "a/biomed_clip_example_20230323.ipynb" "b/biomed_clip_example_20230323.ipynb"
new file mode 100644--- /dev/null
+++ "b/biomed_clip_example_20230323.ipynb"
@@ -0,0 +1,2170 @@
+{
+ "nbformat": 4,
+ "nbformat_minor": 0,
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "name": "python3",
+ "display_name": "Python 3"
+ },
+ "language_info": {
+ "name": "python"
+ },
+ "widgets": {
+ "application/vnd.jupyter.widget-state+json": {
+ "42279f0aaf454588a7f5ab4126f03f1f": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "HBoxModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_9c04c4d9d6ed406d9ff488050806d15a",
+ "IPY_MODEL_9d61c13f30af4391bf2a3012f9871e9e",
+ "IPY_MODEL_9a7883cf26e1423bb216ae85aa271edf"
+ ],
+ "layout": "IPY_MODEL_f34ed1e1d2bc438795b1c8fd78a9e72a"
+ }
+ },
+ "9c04c4d9d6ed406d9ff488050806d15a": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "HTMLModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_7aeb12b1a3944acdb629b75db8237899",
+ "placeholder": "",
+ "style": "IPY_MODEL_0d4eaf03c87a45b69dd0f9c3aad12d56",
+ "value": "Downloading (…)lve/main/config.json: 100%"
+ }
+ },
+ "9d61c13f30af4391bf2a3012f9871e9e": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "FloatProgressModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "ProgressView",
+ "bar_style": "success",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_cec7e4851a104479b6dbce556061d1b3",
+ "max": 385,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_2142fefcd627477bbd2b83ac59930cc5",
+ "value": 385
+ }
+ },
+ "9a7883cf26e1423bb216ae85aa271edf": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "HTMLModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_229b382acaa540cc9fdacbe8f1d388b0",
+ "placeholder": "",
+ "style": "IPY_MODEL_f8e8b814c568465ebdf95a8536005de5",
+ "value": " 385/385 [00:00<00:00, 5.80kB/s]"
+ }
+ },
+ "f34ed1e1d2bc438795b1c8fd78a9e72a": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "model_module_version": "1.2.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "7aeb12b1a3944acdb629b75db8237899": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "model_module_version": "1.2.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "0d4eaf03c87a45b69dd0f9c3aad12d56": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "DescriptionStyleModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "cec7e4851a104479b6dbce556061d1b3": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "model_module_version": "1.2.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "2142fefcd627477bbd2b83ac59930cc5": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "ProgressStyleModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
+ },
+ "229b382acaa540cc9fdacbe8f1d388b0": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "model_module_version": "1.2.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "f8e8b814c568465ebdf95a8536005de5": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "DescriptionStyleModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "526d0e6c57774754a343be6b0525ba19": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "HBoxModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_a3f707f48a59480faac65f7314fafd47",
+ "IPY_MODEL_cc17d31a4a97417d8a8773e9f0f20c5a",
+ "IPY_MODEL_d3003bb831074b22a29256cc9b96df67"
+ ],
+ "layout": "IPY_MODEL_74602d4b276e4ce6a595f0956af6a85f"
+ }
+ },
+ "a3f707f48a59480faac65f7314fafd47": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "HTMLModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_e0acabc0e90a48f2bc08b4aeaf1a6331",
+ "placeholder": "",
+ "style": "IPY_MODEL_4fd3ebf062454067b7a7f888384ac873",
+ "value": "Downloading pytorch_model.bin: 100%"
+ }
+ },
+ "cc17d31a4a97417d8a8773e9f0f20c5a": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "FloatProgressModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "ProgressView",
+ "bar_style": "success",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_2763d0cc0a1d430480540f7eea790c35",
+ "max": 440474434,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_b174573848e54b14a878f8f4ee1637c8",
+ "value": 440474434
+ }
+ },
+ "d3003bb831074b22a29256cc9b96df67": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "HTMLModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_f024455a82364bee980c79b91b31e023",
+ "placeholder": "",
+ "style": "IPY_MODEL_424f3fc993d649e597d099b10f51de20",
+ "value": " 440M/440M [00:05<00:00, 121MB/s]"
+ }
+ },
+ "74602d4b276e4ce6a595f0956af6a85f": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "model_module_version": "1.2.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "e0acabc0e90a48f2bc08b4aeaf1a6331": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "model_module_version": "1.2.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "4fd3ebf062454067b7a7f888384ac873": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "DescriptionStyleModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "2763d0cc0a1d430480540f7eea790c35": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "model_module_version": "1.2.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "b174573848e54b14a878f8f4ee1637c8": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "ProgressStyleModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
+ },
+ "f024455a82364bee980c79b91b31e023": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "model_module_version": "1.2.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "424f3fc993d649e597d099b10f51de20": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "DescriptionStyleModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "c0c47a877e664f4e81b9238c0623bd69": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "HBoxModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_a89c457c6f904dd59f013ef048512841",
+ "IPY_MODEL_b6afb2bed86c41599f934b04b1d45d63",
+ "IPY_MODEL_cf5f5635be774821bdf720af21dc1fc3"
+ ],
+ "layout": "IPY_MODEL_193d1fffdf504cd5a610fd7e36de48b0"
+ }
+ },
+ "a89c457c6f904dd59f013ef048512841": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "HTMLModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_59f2dd49767248d9bc7287c510584386",
+ "placeholder": "",
+ "style": "IPY_MODEL_b9d60cf5b653439e8df60b592fc5059e",
+ "value": "Downloading (…)okenizer_config.json: 100%"
+ }
+ },
+ "b6afb2bed86c41599f934b04b1d45d63": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "FloatProgressModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "ProgressView",
+ "bar_style": "success",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_c8d3ac3cb2c74f53bd8f68d82ea60f6c",
+ "max": 28,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_ccb64052caed4ed19cadd79e09d97972",
+ "value": 28
+ }
+ },
+ "cf5f5635be774821bdf720af21dc1fc3": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "HTMLModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_8e55e35a08b6419d9edcfcb2a81fd99f",
+ "placeholder": "",
+ "style": "IPY_MODEL_0ffe6a6a792b4233b56a71b51726555f",
+ "value": " 28.0/28.0 [00:00<00:00, 182B/s]"
+ }
+ },
+ "193d1fffdf504cd5a610fd7e36de48b0": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "model_module_version": "1.2.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "59f2dd49767248d9bc7287c510584386": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "model_module_version": "1.2.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "b9d60cf5b653439e8df60b592fc5059e": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "DescriptionStyleModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "c8d3ac3cb2c74f53bd8f68d82ea60f6c": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "model_module_version": "1.2.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "ccb64052caed4ed19cadd79e09d97972": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "ProgressStyleModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
+ },
+ "8e55e35a08b6419d9edcfcb2a81fd99f": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "model_module_version": "1.2.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "0ffe6a6a792b4233b56a71b51726555f": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "DescriptionStyleModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "e497e3fb759d4f49bce357053a6a4457": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "HBoxModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_5195a990292848e5bc38c12fb8542553",
+ "IPY_MODEL_cc1f35ae39944004b3915cae663c51cb",
+ "IPY_MODEL_ce5ab983090e41098444eb36c397e027"
+ ],
+ "layout": "IPY_MODEL_0b8321428edc4155a6fd096385609b54"
+ }
+ },
+ "5195a990292848e5bc38c12fb8542553": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "HTMLModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_9c2b0e5a4032410e9d05a360feec5209",
+ "placeholder": "",
+ "style": "IPY_MODEL_73997d55a3a144abb1e4c43dc97e320b",
+ "value": "Downloading (…)solve/main/vocab.txt: 100%"
+ }
+ },
+ "cc1f35ae39944004b3915cae663c51cb": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "FloatProgressModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "ProgressView",
+ "bar_style": "success",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_1d3d15f8ccca44cbb64a06ddb60017e4",
+ "max": 225062,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_116d37feb437461ea169556cfc6b621d",
+ "value": 225062
+ }
+ },
+ "ce5ab983090e41098444eb36c397e027": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "HTMLModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_e1fefcd310b6497395058972dfd448b9",
+ "placeholder": "",
+ "style": "IPY_MODEL_e75642fb03134b69ae00adb4febabd6b",
+ "value": " 225k/225k [00:00<00:00, 958kB/s]"
+ }
+ },
+ "0b8321428edc4155a6fd096385609b54": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "model_module_version": "1.2.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "9c2b0e5a4032410e9d05a360feec5209": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "model_module_version": "1.2.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "73997d55a3a144abb1e4c43dc97e320b": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "DescriptionStyleModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "1d3d15f8ccca44cbb64a06ddb60017e4": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "model_module_version": "1.2.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "116d37feb437461ea169556cfc6b621d": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "ProgressStyleModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
+ },
+ "e1fefcd310b6497395058972dfd448b9": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "model_module_version": "1.2.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "e75642fb03134b69ae00adb4febabd6b": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "DescriptionStyleModel",
+ "model_module_version": "1.5.0",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ }
+ }
+ }
+ },
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "source": [
+ "### Clone BiomedCLIP repo\n",
+ "\n",
+ "Will be merged to the open_clip main later"
+ ],
+ "metadata": {
+ "id": "e4ymt2jdGuPx"
+ }
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "J4N1Au3wz10u",
+ "outputId": "b995f3dd-22d8-47d8-ceca-c203e4c18c59"
+ },
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "Cloning into 'open_clip'...\n",
+ "remote: Enumerating objects: 2331, done.\u001b[K\n",
+ "remote: Counting objects: 100% (8/8), done.\u001b[K\n",
+ "remote: Compressing objects: 100% (8/8), done.\u001b[K\n",
+ "remote: Total 2331 (delta 0), reused 3 (delta 0), pack-reused 2323\u001b[K\n",
+ "Receiving objects: 100% (2331/2331), 7.92 MiB | 16.69 MiB/s, done.\n",
+ "Resolving deltas: 100% (1386/1386), done.\n"
+ ]
+ }
+ ],
+ "source": [
+ "!git clone -b naotous/biomed_clip_224px https://github.com/usuyama/open_clip.git"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "### Install libraries\n",
+ "**Make sure to restart the Colab runtime after installation**\n",
+ "\n",
+ "Colab Menu -> Runtime -> Restart runtime"
+ ],
+ "metadata": {
+ "id": "t8BYG2CFF6wD"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "!pip install -e ./open_clip transformers"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "nUAkSVJ90DQs",
+ "outputId": "519b19b5-f712-458f-bb06-ae8556a5241c"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
+ "Obtaining file:///content/open_clip\n",
+ " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+ "Collecting transformers\n",
+ " Downloading transformers-4.27.3-py3-none-any.whl (6.8 MB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.22.4)\n",
+ "Collecting huggingface-hub<1.0,>=0.11.0\n",
+ " Downloading huggingface_hub-0.13.3-py3-none-any.whl (199 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.8/199.8 KB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.10.0)\n",
+ "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n",
+ "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.0)\n",
+ "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.65.0)\n",
+ "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (6.0)\n",
+ "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
+ " Downloading tokenizers-0.13.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.27.1)\n",
+ "Requirement already satisfied: torch>=1.9.0 in /usr/local/lib/python3.9/dist-packages (from open-clip-torch==2.16.0) (1.13.1+cu116)\n",
+ "Requirement already satisfied: torchvision in /usr/local/lib/python3.9/dist-packages (from open-clip-torch==2.16.0) (0.14.1+cu116)\n",
+ "Collecting ftfy\n",
+ " Downloading ftfy-6.1.1-py3-none-any.whl (53 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.1/53.1 KB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hCollecting sentencepiece\n",
+ " Downloading sentencepiece-0.1.97-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m19.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: protobuf<4 in /usr/local/lib/python3.9/dist-packages (from open-clip-torch==2.16.0) (3.19.6)\n",
+ "Collecting timm\n",
+ " Downloading timm-0.6.12-py3-none-any.whl (549 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m549.1/549.1 KB\u001b[0m \u001b[31m14.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n",
+ "Requirement already satisfied: wcwidth>=0.2.5 in /usr/local/lib/python3.9/dist-packages (from ftfy->open-clip-torch==2.16.0) (0.2.6)\n",
+ "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (3.4)\n",
+ "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2.0.12)\n",
+ "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2022.12.7)\n",
+ "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.15)\n",
+ "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.9/dist-packages (from torchvision->open-clip-torch==2.16.0) (8.4.0)\n",
+ "Installing collected packages: tokenizers, sentencepiece, ftfy, huggingface-hub, transformers, timm, open-clip-torch\n",
+ " Running setup.py develop for open-clip-torch\n",
+ "Successfully installed ftfy-6.1.1 huggingface-hub-0.13.3 open-clip-torch-2.16.0 sentencepiece-0.1.97 timm-0.6.12 tokenizers-0.13.2 transformers-4.27.3\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "### Setup AzCopy for downloading files from our Azure storage"
+ ],
+ "metadata": {
+ "id": "-8l0giuDGUX-"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "!wget -O azcopy_linux.tar.gz https://aka.ms/downloadazcopy-v10-linux\n",
+ "!tar -xf azcopy_linux.tar.gz --strip-components=1 --wildcards '*/azcopy'\n",
+ "!./azcopy --version"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "dEB_ONMs0qn7",
+ "outputId": "3ece3f9b-c283-463c-890e-6e810e5af5ac"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "--2023-03-23 20:23:27-- https://aka.ms/downloadazcopy-v10-linux\n",
+ "Resolving aka.ms (aka.ms)... 23.12.134.45\n",
+ "Connecting to aka.ms (aka.ms)|23.12.134.45|:443... connected.\n",
+ "HTTP request sent, awaiting response... 301 Moved Permanently\n",
+ "Location: https://azcopyvnext.azureedge.net/release20230123/azcopy_linux_amd64_10.17.0.tar.gz [following]\n",
+ "--2023-03-23 20:23:28-- https://azcopyvnext.azureedge.net/release20230123/azcopy_linux_amd64_10.17.0.tar.gz\n",
+ "Resolving azcopyvnext.azureedge.net (azcopyvnext.azureedge.net)... 23.213.34.169, 23.213.34.191, 2600:1409:9800:21::17d8:93cb, ...\n",
+ "Connecting to azcopyvnext.azureedge.net (azcopyvnext.azureedge.net)|23.213.34.169|:443... connected.\n",
+ "HTTP request sent, awaiting response... 200 OK\n",
+ "Length: 13494500 (13M) [application/gzip]\n",
+ "Saving to: ‘azcopy_linux.tar.gz’\n",
+ "\n",
+ "azcopy_linux.tar.gz 100%[===================>] 12.87M --.-KB/s in 0.09s \n",
+ "\n",
+ "2023-03-23 20:23:28 (136 MB/s) - ‘azcopy_linux.tar.gz’ saved [13494500/13494500]\n",
+ "\n",
+ "azcopy version 10.17.0\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "### Download the model checkpoint and example images"
+ ],
+ "metadata": {
+ "id": "G4FRMeHQHW3c"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "!./azcopy copy --recursive \"https://hanoverdev.blob.core.windows.net/biomed-clip-share?sv=2021-10-04&st=2023-03-22T22%3A14%3A19Z&se=2024-03-23T22%3A14%3A00Z&sr=c&sp=rl&sig=lau70KJwG9ddkITH1CGhWv%2FDpRFKTpansQltuVWFhNY%3D\" ."
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "FfuizqV60V1Q",
+ "outputId": "3ccd96dc-8ef1-4399-cf7e-38154c43d395"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "INFO: Scanning...\n",
+ "INFO: Any empty folders will not be processed, because source and/or destination doesn't have full folder support\n",
+ "\n",
+ "Job 437698c5-72d3-6744-6cd2-d08967f0f4ab has started\n",
+ "Log file is located at: /root/.azcopy/437698c5-72d3-6744-6cd2-d08967f0f4ab.log\n",
+ "\n",
+ "100.0 %, 10 Done, 0 Failed, 0 Pending, 0 Skipped, 10 Total, \n",
+ "\n",
+ "\n",
+ "Job 437698c5-72d3-6744-6cd2-d08967f0f4ab summary\n",
+ "Elapsed Time (Minutes): 0.2336\n",
+ "Number of File Transfers: 10\n",
+ "Number of Folder Property Transfers: 0\n",
+ "Total Number of Transfers: 10\n",
+ "Number of File Transfers Completed: 10\n",
+ "Number of Folder Transfers Completed: 0\n",
+ "Number of File Transfers Failed: 0\n",
+ "Number of Folder Transfers Failed: 0\n",
+ "Number of File Transfers Skipped: 0\n",
+ "Number of Folder Transfers Skipped: 0\n",
+ "TotalBytesTransferred: 2353110162\n",
+ "Final Job Status: Completed\n",
+ "\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "!ls biomed-clip-share"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "4WOxBdKr0e_m",
+ "outputId": "468cd91f-9f81-464c-b747-f88d56960560"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "example_data models\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "### Load BiomedCLIP model"
+ ],
+ "metadata": {
+ "id": "ZQ7D3zXKGcaM"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "import glob\n",
+ "from collections import OrderedDict\n",
+ "\n",
+ "import torch\n",
+ "from PIL import Image\n",
+ "import open_clip\n",
+ "\n",
+ "# Timm contiual pretraining\n",
+ "model_name = 'PubMedBERT_256-timm-vit_base_patch16_224'\n",
+ "checkpoint = 'biomed-clip-share/models/2022_11_08-07_39_28-model_timm-vit_base_patch16_224-lr_0.0005-b_1024-j_8-p_amp/checkpoints/epoch_32.pt'\n",
+ "\n",
+ "model, transform_train, transform_val = open_clip.create_model_and_transforms(model_name)\n",
+ "checkpoint = torch.load(checkpoint, map_location=\"cpu\")\n",
+ "\n",
+ "tokenizer = open_clip.get_tokenizer(model_name)\n",
+ "context_length = 256\n",
+ "\n",
+ "new_state_dict = OrderedDict()\n",
+ "for k,v in checkpoint['state_dict'].items():\n",
+ " new_k = k.replace('module.', '')\n",
+ " new_state_dict[new_k] = v\n",
+ "model.load_state_dict(new_state_dict, strict=False) # can set this to be true except for timm models"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 236,
+ "referenced_widgets": [
+ "42279f0aaf454588a7f5ab4126f03f1f",
+ "9c04c4d9d6ed406d9ff488050806d15a",
+ "9d61c13f30af4391bf2a3012f9871e9e",
+ "9a7883cf26e1423bb216ae85aa271edf",
+ "f34ed1e1d2bc438795b1c8fd78a9e72a",
+ "7aeb12b1a3944acdb629b75db8237899",
+ "0d4eaf03c87a45b69dd0f9c3aad12d56",
+ "cec7e4851a104479b6dbce556061d1b3",
+ "2142fefcd627477bbd2b83ac59930cc5",
+ "229b382acaa540cc9fdacbe8f1d388b0",
+ "f8e8b814c568465ebdf95a8536005de5",
+ "526d0e6c57774754a343be6b0525ba19",
+ "a3f707f48a59480faac65f7314fafd47",
+ "cc17d31a4a97417d8a8773e9f0f20c5a",
+ "d3003bb831074b22a29256cc9b96df67",
+ "74602d4b276e4ce6a595f0956af6a85f",
+ "e0acabc0e90a48f2bc08b4aeaf1a6331",
+ "4fd3ebf062454067b7a7f888384ac873",
+ "2763d0cc0a1d430480540f7eea790c35",
+ "b174573848e54b14a878f8f4ee1637c8",
+ "f024455a82364bee980c79b91b31e023",
+ "424f3fc993d649e597d099b10f51de20",
+ "c0c47a877e664f4e81b9238c0623bd69",
+ "a89c457c6f904dd59f013ef048512841",
+ "b6afb2bed86c41599f934b04b1d45d63",
+ "cf5f5635be774821bdf720af21dc1fc3",
+ "193d1fffdf504cd5a610fd7e36de48b0",
+ "59f2dd49767248d9bc7287c510584386",
+ "b9d60cf5b653439e8df60b592fc5059e",
+ "c8d3ac3cb2c74f53bd8f68d82ea60f6c",
+ "ccb64052caed4ed19cadd79e09d97972",
+ "8e55e35a08b6419d9edcfcb2a81fd99f",
+ "0ffe6a6a792b4233b56a71b51726555f",
+ "e497e3fb759d4f49bce357053a6a4457",
+ "5195a990292848e5bc38c12fb8542553",
+ "cc1f35ae39944004b3915cae663c51cb",
+ "ce5ab983090e41098444eb36c397e027",
+ "0b8321428edc4155a6fd096385609b54",
+ "9c2b0e5a4032410e9d05a360feec5209",
+ "73997d55a3a144abb1e4c43dc97e320b",
+ "1d3d15f8ccca44cbb64a06ddb60017e4",
+ "116d37feb437461ea169556cfc6b621d",
+ "e1fefcd310b6497395058972dfd448b9",
+ "e75642fb03134b69ae00adb4febabd6b"
+ ]
+ },
+ "id": "v88W1wY01oxB",
+ "outputId": "0d163c47-ae2d-4c6e-f523-bea45400123f"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/plain": [
+ "Downloading (…)lve/main/config.json: 0%| | 0.00/385 [00:00, ?B/s]"
+ ],
+ "application/vnd.jupyter.widget-view+json": {
+ "version_major": 2,
+ "version_minor": 0,
+ "model_id": "42279f0aaf454588a7f5ab4126f03f1f"
+ }
+ },
+ "metadata": {}
+ },
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/plain": [
+ "Downloading pytorch_model.bin: 0%| | 0.00/440M [00:00, ?B/s]"
+ ],
+ "application/vnd.jupyter.widget-view+json": {
+ "version_major": 2,
+ "version_minor": 0,
+ "model_id": "526d0e6c57774754a343be6b0525ba19"
+ }
+ },
+ "metadata": {}
+ },
+ {
+ "output_type": "stream",
+ "name": "stderr",
+ "text": [
+ "Some weights of the model checkpoint at microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract were not used when initializing BertModel: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.bias']\n",
+ "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+ "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
+ ]
+ },
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/plain": [
+ "Downloading (…)okenizer_config.json: 0%| | 0.00/28.0 [00:00, ?B/s]"
+ ],
+ "application/vnd.jupyter.widget-view+json": {
+ "version_major": 2,
+ "version_minor": 0,
+ "model_id": "c0c47a877e664f4e81b9238c0623bd69"
+ }
+ },
+ "metadata": {}
+ },
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/plain": [
+ "Downloading (…)solve/main/vocab.txt: 0%| | 0.00/225k [00:00, ?B/s]"
+ ],
+ "application/vnd.jupyter.widget-view+json": {
+ "version_major": 2,
+ "version_minor": 0,
+ "model_id": "e497e3fb759d4f49bce357053a6a4457"
+ }
+ },
+ "metadata": {}
+ },
+ {
+ "output_type": "execute_result",
+ "data": {
+ "text/plain": [
+ "_IncompatibleKeys(missing_keys=['text.transformer.pooler.dense.weight', 'text.transformer.pooler.dense.bias'], unexpected_keys=[])"
+ ]
+ },
+ "metadata": {},
+ "execution_count": 1
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "### Example: Zero-shot classifications"
+ ],
+ "metadata": {
+ "id": "_11A5zFuGfkG"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "dataset_path = 'biomed-clip-share/example_data/biomed_image_classification_example_data'\n",
+ "template = 'this is a photo of '\n",
+ "labels = [\n",
+ " 'adenocarcinoma histopathology',\n",
+ " 'brain MRI',\n",
+ " 'covid line chart',\n",
+ " 'squamous cell carcinoma histopathology',\n",
+ " 'immunohistochemistry histopathology',\n",
+ " 'bone X-ray',\n",
+ " 'chest X-ray',\n",
+ " 'pie chart',\n",
+ " 'hematoxylin and eosin histopathology'\n",
+ "]\n",
+ "\n",
+ "import glob\n",
+ "test_imgs = glob.glob(dataset_path + '/*')\n",
+ "\n",
+ "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
+ "model.to(device)\n",
+ "model.eval()\n",
+ "\n",
+ "images = torch.stack([transform_val(Image.open(img)) for img in test_imgs]).to(device)\n",
+ "texts = tokenizer([template + l for l in labels], context_length=context_length).to(device)\n",
+ "with torch.no_grad():\n",
+ " image_features, text_features, logit_scale = model(images, texts)\n",
+ "\n",
+ " logits = (logit_scale * image_features @ text_features.t()).detach().softmax(dim=-1)\n",
+ " sorted_indices = torch.argsort(logits, dim=-1, descending=True)\n",
+ "\n",
+ " logits = logits.cpu().numpy()\n",
+ " sorted_indices = sorted_indices.cpu().numpy()\n",
+ "\n",
+ "top_k = -1\n",
+ "\n",
+ "for i, img in enumerate(test_imgs):\n",
+ " pred = labels[sorted_indices[i][0]]\n",
+ "\n",
+ " top_k = len(labels) if top_k == -1 else top_k\n",
+ " print(img.split('/')[-1] + ':')\n",
+ " for j in range(top_k):\n",
+ " jth_index = sorted_indices[i][j]\n",
+ " print(f'{labels[jth_index]}: {logits[i][jth_index]}')\n",
+ " print('\\n')"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "XSJw7Qpm1w-f",
+ "outputId": "08bbc1cd-c6b2-45aa-a4a2-759e0f0dd844"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "adenocarcinoma_histopathology.jpg:\n",
+ "adenocarcinoma histopathology: 0.7818863987922668\n",
+ "hematoxylin and eosin histopathology: 0.15517690777778625\n",
+ "immunohistochemistry histopathology: 0.06149514392018318\n",
+ "squamous cell carcinoma histopathology: 0.0014182085869833827\n",
+ "chest X-ray: 2.017213228100445e-05\n",
+ "brain MRI: 1.2181524198240368e-06\n",
+ "pie chart: 7.932688959044754e-07\n",
+ "bone X-ray: 7.436410101036017e-07\n",
+ "covid line chart: 4.482610052036762e-07\n",
+ "\n",
+ "\n",
+ "covid_line_chart.png:\n",
+ "covid line chart: 0.9493210315704346\n",
+ "adenocarcinoma histopathology: 0.01898195780813694\n",
+ "squamous cell carcinoma histopathology: 0.0175501499325037\n",
+ "immunohistochemistry histopathology: 0.006791787222027779\n",
+ "hematoxylin and eosin histopathology: 0.003417333820834756\n",
+ "brain MRI: 0.002629919210448861\n",
+ "chest X-ray: 0.0010041205678135157\n",
+ "bone X-ray: 0.00024685842799954116\n",
+ "pie chart: 5.6812208640621975e-05\n",
+ "\n",
+ "\n",
+ "bone_X-ray.jpg:\n",
+ "bone X-ray: 0.9037961959838867\n",
+ "hematoxylin and eosin histopathology: 0.07279316335916519\n",
+ "brain MRI: 0.013534954749047756\n",
+ "chest X-ray: 0.00821212213486433\n",
+ "immunohistochemistry histopathology: 0.001647887285798788\n",
+ "squamous cell carcinoma histopathology: 1.418814281350933e-05\n",
+ "covid line chart: 1.1351590956110158e-06\n",
+ "adenocarcinoma histopathology: 2.3802124360372545e-07\n",
+ "pie chart: 9.433303205241828e-08\n",
+ "\n",
+ "\n",
+ "pie_chart.png:\n",
+ "pie chart: 0.999992847442627\n",
+ "covid line chart: 6.056906840967713e-06\n",
+ "brain MRI: 6.212158041307703e-07\n",
+ "bone X-ray: 1.870277799298492e-07\n",
+ "chest X-ray: 1.4315827456812258e-07\n",
+ "immunohistochemistry histopathology: 7.397970591682679e-08\n",
+ "hematoxylin and eosin histopathology: 1.3329795045535775e-08\n",
+ "adenocarcinoma histopathology: 7.695367898463701e-09\n",
+ "squamous cell carcinoma histopathology: 4.512833662317917e-09\n",
+ "\n",
+ "\n",
+ "H_and_E_histopathology.jpg:\n",
+ "hematoxylin and eosin histopathology: 0.7953251600265503\n",
+ "immunohistochemistry histopathology: 0.19779996573925018\n",
+ "chest X-ray: 0.005973907187581062\n",
+ "bone X-ray: 0.0008049230673350394\n",
+ "adenocarcinoma histopathology: 9.133991261478513e-05\n",
+ "squamous cell carcinoma histopathology: 3.6423973597266013e-06\n",
+ "brain MRI: 6.688684948130685e-07\n",
+ "pie chart: 4.278819574210502e-07\n",
+ "covid line chart: 3.051619401617245e-08\n",
+ "\n",
+ "\n",
+ "brain_MRI.jpg:\n",
+ "brain MRI: 0.9565795660018921\n",
+ "hematoxylin and eosin histopathology: 0.041418157517910004\n",
+ "immunohistochemistry histopathology: 0.0019450499676167965\n",
+ "pie chart: 2.7151252652402036e-05\n",
+ "squamous cell carcinoma histopathology: 1.0223812751064543e-05\n",
+ "bone X-ray: 8.662499567435589e-06\n",
+ "chest X-ray: 7.96773747424595e-06\n",
+ "adenocarcinoma histopathology: 2.7692055937222904e-06\n",
+ "covid line chart: 3.420084908611898e-07\n",
+ "\n",
+ "\n",
+ "chest_X-ray.jpg:\n",
+ "chest X-ray: 0.9998347759246826\n",
+ "hematoxylin and eosin histopathology: 0.0001205605294671841\n",
+ "bone X-ray: 4.112880560569465e-05\n",
+ "immunohistochemistry histopathology: 1.0486423889233265e-06\n",
+ "adenocarcinoma histopathology: 9.66637117016944e-07\n",
+ "covid line chart: 9.508977996119938e-07\n",
+ "brain MRI: 3.232386518448038e-07\n",
+ "squamous cell carcinoma histopathology: 2.53368597213921e-07\n",
+ "pie chart: 3.6984038054299617e-09\n",
+ "\n",
+ "\n",
+ "squamous_cell_carcinoma_histopathology.jpeg:\n",
+ "squamous cell carcinoma histopathology: 0.9469489455223083\n",
+ "adenocarcinoma histopathology: 0.05259034037590027\n",
+ "hematoxylin and eosin histopathology: 0.0003988408425357193\n",
+ "immunohistochemistry histopathology: 6.187965482240543e-05\n",
+ "chest X-ray: 1.4099594380923008e-08\n",
+ "pie chart: 3.522500624519864e-10\n",
+ "bone X-ray: 2.9633814846441453e-10\n",
+ "brain MRI: 1.2720452469139332e-10\n",
+ "covid line chart: 1.8425603924565603e-12\n",
+ "\n",
+ "\n",
+ "IHC_histopathology.jpg:\n",
+ "immunohistochemistry histopathology: 0.9465934634208679\n",
+ "hematoxylin and eosin histopathology: 0.03232448548078537\n",
+ "brain MRI: 0.020657211542129517\n",
+ "adenocarcinoma histopathology: 0.000304735847748816\n",
+ "bone X-ray: 4.5735167077509686e-05\n",
+ "squamous cell carcinoma histopathology: 3.150868360535242e-05\n",
+ "covid line chart: 2.0559578842949122e-05\n",
+ "chest X-ray: 1.2715442608168814e-05\n",
+ "pie chart: 9.55282575887395e-06\n",
+ "\n",
+ "\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "
Expected outputs
\n",
+ "\n",
+ "\n",
+ "adenocarcinoma_histopathology.jpg:\n",
+ "adenocarcinoma histopathology: 0.7818863987922668\n",
+ "hematoxylin and eosin histopathology: 0.15517690777778625\n",
+ "immunohistochemistry histopathology: 0.06149514392018318\n",
+ "squamous cell carcinoma histopathology: 0.0014182085869833827\n",
+ "chest X-ray: 2.017213228100445e-05\n",
+ "brain MRI: 1.2181524198240368e-06\n",
+ "pie chart: 7.932688959044754e-07\n",
+ "bone X-ray: 7.436410101036017e-07\n",
+ "covid line chart: 4.482610052036762e-07\n",
+ "\n",
+ "\n",
+ "covid_line_chart.png:\n",
+ "covid line chart: 0.9493210315704346\n",
+ "adenocarcinoma histopathology: 0.01898195780813694\n",
+ "squamous cell carcinoma histopathology: 0.0175501499325037\n",
+ "immunohistochemistry histopathology: 0.006791787222027779\n",
+ "hematoxylin and eosin histopathology: 0.003417333820834756\n",
+ "brain MRI: 0.002629919210448861\n",
+ "chest X-ray: 0.0010041205678135157\n",
+ "bone X-ray: 0.00024685842799954116\n",
+ "pie chart: 5.6812208640621975e-05\n",
+ "\n",
+ "\n",
+ "bone_X-ray.jpg:\n",
+ "bone X-ray: 0.9037961959838867\n",
+ "hematoxylin and eosin histopathology: 0.07279316335916519\n",
+ "brain MRI: 0.013534954749047756\n",
+ "chest X-ray: 0.00821212213486433\n",
+ "immunohistochemistry histopathology: 0.001647887285798788\n",
+ "squamous cell carcinoma histopathology: 1.418814281350933e-05\n",
+ "covid line chart: 1.1351590956110158e-06\n",
+ "adenocarcinoma histopathology: 2.3802124360372545e-07\n",
+ "pie chart: 9.433303205241828e-08\n",
+ "\n",
+ "\n",
+ "pie_chart.png:\n",
+ "pie chart: 0.999992847442627\n",
+ "covid line chart: 6.056906840967713e-06\n",
+ "brain MRI: 6.212158041307703e-07\n",
+ "bone X-ray: 1.870277799298492e-07\n",
+ "chest X-ray: 1.4315827456812258e-07\n",
+ "immunohistochemistry histopathology: 7.397970591682679e-08\n",
+ "hematoxylin and eosin histopathology: 1.3329795045535775e-08\n",
+ "adenocarcinoma histopathology: 7.695367898463701e-09\n",
+ "squamous cell carcinoma histopathology: 4.512833662317917e-09\n",
+ "\n",
+ "\n",
+ "H_and_E_histopathology.jpg:\n",
+ "hematoxylin and eosin histopathology: 0.7953251600265503\n",
+ "immunohistochemistry histopathology: 0.19779996573925018\n",
+ "chest X-ray: 0.005973907187581062\n",
+ "bone X-ray: 0.0008049230673350394\n",
+ "adenocarcinoma histopathology: 9.133991261478513e-05\n",
+ "squamous cell carcinoma histopathology: 3.6423973597266013e-06\n",
+ "brain MRI: 6.688684948130685e-07\n",
+ "pie chart: 4.278819574210502e-07\n",
+ "covid line chart: 3.051619401617245e-08\n",
+ "\n",
+ "\n",
+ "brain_MRI.jpg:\n",
+ "brain MRI: 0.9565795660018921\n",
+ "hematoxylin and eosin histopathology: 0.041418157517910004\n",
+ "immunohistochemistry histopathology: 0.0019450499676167965\n",
+ "pie chart: 2.7151252652402036e-05\n",
+ "squamous cell carcinoma histopathology: 1.0223812751064543e-05\n",
+ "bone X-ray: 8.662499567435589e-06\n",
+ "chest X-ray: 7.96773747424595e-06\n",
+ "adenocarcinoma histopathology: 2.7692055937222904e-06\n",
+ "covid line chart: 3.420084908611898e-07\n",
+ "\n",
+ "\n",
+ "chest_X-ray.jpg:\n",
+ "chest X-ray: 0.9998347759246826\n",
+ "hematoxylin and eosin histopathology: 0.0001205605294671841\n",
+ "bone X-ray: 4.112880560569465e-05\n",
+ "immunohistochemistry histopathology: 1.0486423889233265e-06\n",
+ "adenocarcinoma histopathology: 9.66637117016944e-07\n",
+ "covid line chart: 9.508977996119938e-07\n",
+ "brain MRI: 3.232386518448038e-07\n",
+ "squamous cell carcinoma histopathology: 2.53368597213921e-07\n",
+ "pie chart: 3.6984038054299617e-09\n",
+ "\n",
+ "\n",
+ "squamous_cell_carcinoma_histopathology.jpeg:\n",
+ "squamous cell carcinoma histopathology: 0.9469489455223083\n",
+ "adenocarcinoma histopathology: 0.05259034037590027\n",
+ "hematoxylin and eosin histopathology: 0.0003988408425357193\n",
+ "immunohistochemistry histopathology: 6.187965482240543e-05\n",
+ "chest X-ray: 1.4099594380923008e-08\n",
+ "pie chart: 3.522500624519864e-10\n",
+ "bone X-ray: 2.9633814846441453e-10\n",
+ "brain MRI: 1.2720452469139332e-10\n",
+ "covid line chart: 1.8425603924565603e-12\n",
+ "\n",
+ "\n",
+ "IHC_histopathology.jpg:\n",
+ "immunohistochemistry histopathology: 0.9465934634208679\n",
+ "hematoxylin and eosin histopathology: 0.03232448548078537\n",
+ "brain MRI: 0.020657211542129517\n",
+ "adenocarcinoma histopathology: 0.000304735847748816\n",
+ "bone X-ray: 4.5735167077509686e-05\n",
+ "squamous cell carcinoma histopathology: 3.150868360535242e-05\n",
+ "covid line chart: 2.0559578842949122e-05\n",
+ "chest X-ray: 1.2715442608168814e-05\n",
+ "pie chart: 9.55282575887395e-06\n",
+ "
\n",
+ " "
+ ],
+ "metadata": {
+ "id": "kIZEaLJB5H6A"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "def plot_images_with_metadata(images, metadata):\n",
+ " num_images = len(images)\n",
+ " fig, axes = plt.subplots(nrows=num_images, ncols=1, figsize=(5, 5 * num_images))\n",
+ "\n",
+ " for i, (img_path, metadata) in enumerate(zip(images, metadata)):\n",
+ " img = Image.open(img_path)\n",
+ " ax = axes[i]\n",
+ " ax.imshow(img)\n",
+ " ax.axis('off')\n",
+ " ax.set_title(f\"{metadata['filename']}\\n{metadata['top_probs']}\", fontsize=14)\n",
+ "\n",
+ " plt.tight_layout()\n",
+ " plt.show()\n",
+ "\n",
+ "metadata_list = []\n",
+ "\n",
+ "top_k = 3\n",
+ "for i, img in enumerate(test_imgs):\n",
+ " pred = labels[sorted_indices[i][0]]\n",
+ " img_name = img.split('/')[-1]\n",
+ "\n",
+ " top_probs = []\n",
+ " top_k = len(labels) if top_k == -1 else top_k\n",
+ " for j in range(top_k):\n",
+ " jth_index = sorted_indices[i][j]\n",
+ " top_probs.append(f\"{labels[jth_index]}: {logits[i][jth_index] * 100:.1f}\")\n",
+ "\n",
+ " metadata = {'filename': img_name, 'top_probs': '\\n'.join(top_probs)}\n",
+ " metadata_list.append(metadata)\n",
+ "\n",
+ "plot_images_with_metadata(test_imgs, metadata_list)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 1000
+ },
+ "id": "zE5vznWj2CCf",
+ "outputId": "43c30caa-ec74-4f85-ca40-a1b308ccd7bd"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/plain": [
+ "