{ "cells": [ { "attachments": {}, "cell_type": "markdown", "metadata": { "id": "t8BYG2CFF6wD" }, "source": [ "### Install libraries\n", "**Make sure to restart the Colab runtime after installation**\n", "\n", "Colab Menu -> Runtime -> Restart runtime" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "nUAkSVJ90DQs", "outputId": "7f3d4fdb-8cab-4269-d10f-0e03c6e882d1" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: open_clip_torch==2.23.0 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (2.23.0)\n", "Requirement already satisfied: transformers==4.35.2 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (4.35.2)\n", "Requirement already satisfied: matplotlib in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (3.8.2)\n", "Requirement already satisfied: torch>=1.9.0 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from open_clip_torch==2.23.0) (2.1.1)\n", "Requirement already satisfied: torchvision in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from open_clip_torch==2.23.0) (0.16.1)\n", "Requirement already satisfied: regex in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from open_clip_torch==2.23.0) (2023.10.3)\n", "Requirement already satisfied: ftfy in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from open_clip_torch==2.23.0) (6.1.3)\n", "Requirement already satisfied: tqdm in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from open_clip_torch==2.23.0) (4.66.1)\n", "Requirement already satisfied: huggingface-hub in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from open_clip_torch==2.23.0) (0.19.4)\n", "Requirement already satisfied: sentencepiece in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from open_clip_torch==2.23.0) (0.1.99)\n", "Requirement already satisfied: protobuf in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from open_clip_torch==2.23.0) (4.25.1)\n", "Requirement already satisfied: timm in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from open_clip_torch==2.23.0) (0.9.11)\n", "Requirement already satisfied: filelock in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from transformers==4.35.2) (3.13.1)\n", "Requirement already satisfied: numpy>=1.17 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from transformers==4.35.2) (1.26.2)\n", "Requirement already satisfied: packaging>=20.0 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from transformers==4.35.2) (23.2)\n", "Requirement already satisfied: pyyaml>=5.1 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from transformers==4.35.2) (6.0.1)\n", "Requirement already satisfied: requests in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from transformers==4.35.2) (2.31.0)\n", "Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from transformers==4.35.2) (0.15.0)\n", "Requirement already satisfied: safetensors>=0.3.1 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from transformers==4.35.2) (0.4.0)\n", "Requirement already satisfied: contourpy>=1.0.1 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from matplotlib) (1.2.0)\n", "Requirement already satisfied: cycler>=0.10 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from matplotlib) (0.12.1)\n", "Requirement already satisfied: fonttools>=4.22.0 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from matplotlib) (4.45.1)\n", "Requirement already satisfied: kiwisolver>=1.3.1 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from matplotlib) (1.4.5)\n", "Requirement already satisfied: pillow>=8 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from matplotlib) (10.1.0)\n", "Requirement already satisfied: pyparsing>=2.3.1 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from matplotlib) (3.1.1)\n", "Requirement already satisfied: python-dateutil>=2.7 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from matplotlib) (2.8.2)\n", "Requirement already satisfied: fsspec>=2023.5.0 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from huggingface-hub->open_clip_torch==2.23.0) (2023.10.0)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from huggingface-hub->open_clip_torch==2.23.0) (4.8.0)\n", "Requirement already satisfied: six>=1.5 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n", "Requirement already satisfied: sympy in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch==2.23.0) (1.12)\n", "Requirement already satisfied: networkx in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch==2.23.0) (3.2.1)\n", "Requirement already satisfied: jinja2 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch==2.23.0) (3.1.2)\n", "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch==2.23.0) (12.1.105)\n", "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch==2.23.0) (12.1.105)\n", "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch==2.23.0) (12.1.105)\n", "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch==2.23.0) (8.9.2.26)\n", "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch==2.23.0) (12.1.3.1)\n", "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch==2.23.0) (11.0.2.54)\n", "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch==2.23.0) (10.3.2.106)\n", "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch==2.23.0) (11.4.5.107)\n", "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch==2.23.0) (12.1.0.106)\n", "Requirement already satisfied: nvidia-nccl-cu12==2.18.1 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch==2.23.0) (2.18.1)\n", "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch==2.23.0) (12.1.105)\n", "Requirement already satisfied: triton==2.1.0 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch==2.23.0) (2.1.0)\n", "Requirement already satisfied: nvidia-nvjitlink-cu12 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.9.0->open_clip_torch==2.23.0) (12.3.101)\n", "Requirement already satisfied: wcwidth<0.3.0,>=0.2.12 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from ftfy->open_clip_torch==2.23.0) (0.2.12)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from requests->transformers==4.35.2) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from requests->transformers==4.35.2) (3.4)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from requests->transformers==4.35.2) (2.1.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from requests->transformers==4.35.2) (2023.11.17)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from jinja2->torch>=1.9.0->open_clip_torch==2.23.0) (2.1.3)\n", "Requirement already satisfied: mpmath>=0.19 in /home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages (from sympy->torch>=1.9.0->open_clip_torch==2.23.0) (1.3.0)\n" ] } ], "source": [ "!pip install open_clip_torch==2.23.0 transformers==4.35.2 matplotlib" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": { "id": "kaOyeKkjqnri" }, "source": [ "## Load BiomedCLIP model" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "V8Yv9g_8EQ1W", "outputId": "3ec24c9b-4c4f-4c17-8d76-6cfd74bb8bdf" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/shezhan/anaconda3/envs/biomedclip/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], "source": [ "import open_clip\n", "\n", "model, preprocess_train, preprocess_val = open_clip.create_model_and_transforms('hf-hub:microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224')\n", "tokenizer = open_clip.get_tokenizer('hf-hub:microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224')" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": { "id": "bk0hm1R7qqU_" }, "source": [ "# Download sample images" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 67, "referenced_widgets": [ "692f8c386f9743a1a12f7d6c7959ca67", "3e0f188e73294f6ea4d1e28640cfdc22", "b754e18c5c49499d92db4803cfa426b7", "6743cbc5ca2c47e7be565e0d6cd933c9", "02aa2c49f2a94a7eb48794ed783c93e8", "ad84c0ed082d4ab7abf2815fc1910efa", "87a18840cc2c45ac824e8fe3d83d5150", "0b3b4fc0e99a47d0a494aee20166337f", "2de24c12eebd4054a3e6163fb6951986", "1c9af9a39e594c689590d09ae71baeb3", "182cc15b918a45d081543a6b3f182a07" ] }, "id": "qqafKW1kqgc4", "outputId": "34c29f78-32c5-4a6f-914e-30e8a07840a6" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "README.md: 100%|██████████| 4.13k/4.13k [00:00<00:00, 7.84MB/s]\n", "biomed-vlp-eval.svg: 100%|██████████| 63.4k/63.4k [00:00<00:00, 9.46MB/s]\n", "\n", "(…)e_data/adenocarcinoma_histopathology.jpg: 100%|██████████| 26.9k/26.9k [00:00<00:00, 8.89MB/s]\n", "\n", "\n", "(…)tion_example_data/IHC_histopathology.jpg: 100%|██████████| 181k/181k [00:00<00:00, 11.9MB/s]\n", "(…)_example_data/H_and_E_histopathology.jpg: 100%|██████████| 177k/177k [00:00<00:00, 5.38MB/s]\n", "\n", "\n", "biomed_clip_example.ipynb: 100%|██████████| 2.88M/2.88M [00:00<00:00, 26.7MB/s]\n", "LICENSE.md: 100%|██████████| 1.07k/1.07k [00:00<00:00, 9.03MB/s]\n", ".gitattributes: 100%|██████████| 1.48k/1.48k [00:00<00:00, 8.57MB/s]\n", "(…)assification_example_data/bone_X-ray.jpg: 100%|██████████| 7.44k/7.44k [00:00<00:00, 13.3MB/s]\n", "(…)lassification_example_data/brain_MRI.jpg: 100%|██████████| 128k/128k [00:00<00:00, 27.3MB/s]\n", "(…)cation_example_data/covid_line_chart.png: 100%|██████████| 6.30k/6.30k [00:00<00:00, 10.7MB/s]\n", "\n", "(…)lassification_example_data/pie_chart.png: 100%|██████████| 371k/371k [00:00<00:00, 29.6MB/s]\n", "\n", "special_tokens_map.json: 100%|██████████| 125/125 [00:00<00:00, 941kB/s]\n", "\n", "(…)ssification_example_data/chest_X-ray.jpg: 100%|██████████| 906k/906k [00:00<00:00, 4.06MB/s]\n", "vocab.txt: 100%|██████████| 225k/225k [00:00<00:00, 84.4MB/s]it/s]\n", "(…)amous_cell_carcinoma_histopathology.jpeg: 100%|██████████| 17.2k/17.2k [00:00<00:00, 24.9MB/s]\n", "tokenizer_config.json: 100%|██████████| 394/394 [00:00<00:00, 1.78MB/s]\n", "\n", "tokenizer.json: 100%|██████████| 679k/679k [00:00<00:00, 3.06MB/s]\n", "Fetching 20 files: 100%|██████████| 20/20 [00:01<00:00, 19.93it/s]\n" ] }, { "data": { "text/plain": [ "'/home/shezhan/repos/biomedclip/biomed-clip-share'" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from huggingface_hub import snapshot_download\n", "snapshot_download(\"microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224\", local_dir=\"biomed-clip-share\")" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "4WOxBdKr0e_m", "outputId": "2a05beae-6f5f-4c3c-ef59-b23210b6e1b5" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "biomed_clip_example.ipynb open_clip_config.json\ttokenizer_config.json\n", "biomed-vlp-eval.svg\t open_clip_pytorch_model.bin\ttokenizer.json\n", "example_data\t\t README.md\t\t\tvocab.txt\n", "LICENSE.md\t\t special_tokens_map.json\n" ] } ], "source": [ "!ls biomed-clip-share" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": { "id": "_11A5zFuGfkG" }, "source": [ "### Example: Zero-shot classifications" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "XSJw7Qpm1w-f", "outputId": "c8e69acc-09a6-41ac-a719-e0c2016e41d8" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "squamous_cell_carcinoma_histopathology.jpeg:\n", "squamous cell carcinoma histopathology: 0.9574294090270996\n", "adenocarcinoma histopathology: 0.042203135788440704\n", "hematoxylin and eosin histopathology: 0.00031521538039669394\n", "immunohistochemistry histopathology: 5.2207757107680663e-05\n", "chest X-ray: 1.2074539590400946e-08\n", "pie chart: 3.848998064714948e-10\n", "bone X-ray: 2.5361207578455947e-10\n", "brain MRI: 1.3335134385616954e-10\n", "covid line chart: 1.80010650482898e-12\n", "\n", "\n", "H_and_E_histopathology.jpg:\n", "hematoxylin and eosin histopathology: 0.7937036752700806\n", "immunohistochemistry histopathology: 0.19955486059188843\n", "chest X-ray: 0.005849139299243689\n", "bone X-ray: 0.0007925905520096421\n", "adenocarcinoma histopathology: 9.481503366259858e-05\n", "squamous cell carcinoma histopathology: 3.7734582747361856e-06\n", "brain MRI: 6.522547550957825e-07\n", "pie chart: 4.1520331706124125e-07\n", "covid line chart: 2.8973827781442196e-08\n", "\n", "\n", "bone_X-ray.jpg:\n", "bone X-ray: 0.9059222340583801\n", "hematoxylin and eosin histopathology: 0.07083114981651306\n", "brain MRI: 0.01343764178454876\n", "chest X-ray: 0.008196878246963024\n", "immunohistochemistry histopathology: 0.0015970191452652216\n", "squamous cell carcinoma histopathology: 1.3760063666268252e-05\n", "covid line chart: 1.082283802134043e-06\n", "adenocarcinoma histopathology: 2.334381434820898e-07\n", "pie chart: 9.2045546296049e-08\n", "\n", "\n", "adenocarcinoma_histopathology.jpg:\n", "adenocarcinoma histopathology: 0.7818822860717773\n", "hematoxylin and eosin histopathology: 0.1551811397075653\n", "immunohistochemistry histopathology: 0.06149505451321602\n", "squamous cell carcinoma histopathology: 0.0014182417653501034\n", "chest X-ray: 2.017233600781765e-05\n", "brain MRI: 1.2181716328996117e-06\n", "pie chart: 7.932791277198703e-07\n", "bone X-ray: 7.43639247957617e-07\n", "covid line chart: 4.482612894207705e-07\n", "\n", "\n", "covid_line_chart.png:\n", "covid line chart: 0.949321985244751\n", "adenocarcinoma histopathology: 0.01898154430091381\n", "squamous cell carcinoma histopathology: 0.017549900338053703\n", "immunohistochemistry histopathology: 0.0067916251718997955\n", "hematoxylin and eosin histopathology: 0.0034173112362623215\n", "brain MRI: 0.0026298766024410725\n", "chest X-ray: 0.0010041081113740802\n", "bone X-ray: 0.00024684693198651075\n", "pie chart: 5.6810858950484544e-05\n", "\n", "\n", "IHC_histopathology.jpg:\n", "immunohistochemistry histopathology: 0.9491111636161804\n", "hematoxylin and eosin histopathology: 0.02898590639233589\n", "brain MRI: 0.021494900807738304\n", "adenocarcinoma histopathology: 0.0002991782675962895\n", "bone X-ray: 4.3843381718033925e-05\n", "squamous cell carcinoma histopathology: 2.6241370505886152e-05\n", "covid line chart: 1.65638357429998e-05\n", "chest X-ray: 1.2055252227582969e-05\n", "pie chart: 1.0083938832394779e-05\n", "\n", "\n", "chest_X-ray.jpg:\n", "chest X-ray: 0.9998347759246826\n", "hematoxylin and eosin histopathology: 0.00012055937986588106\n", "bone X-ray: 4.1128412703983486e-05\n", "immunohistochemistry histopathology: 1.048652393365046e-06\n", "adenocarcinoma histopathology: 9.666111964179436e-07\n", "covid line chart: 9.508868856755726e-07\n", "brain MRI: 3.2324047083420737e-07\n", "squamous cell carcinoma histopathology: 2.533652150304988e-07\n", "pie chart: 3.6983756057651362e-09\n", "\n", "\n", "brain_MRI.jpg:\n", "brain MRI: 0.9565795063972473\n", "hematoxylin and eosin histopathology: 0.04141831025481224\n", "immunohistochemistry histopathology: 0.0019450720865279436\n", "pie chart: 2.7150734240422025e-05\n", "squamous cell carcinoma histopathology: 1.0223977369605564e-05\n", "bone X-ray: 8.662441359774675e-06\n", "chest X-ray: 7.967796591401566e-06\n", "adenocarcinoma histopathology: 2.7692635740095284e-06\n", "covid line chart: 3.420048813040921e-07\n", "\n", "\n", "pie_chart.png:\n", "pie chart: 0.999992847442627\n", "covid line chart: 6.056987785996171e-06\n", "brain MRI: 6.212234779923165e-07\n", "bone X-ray: 1.8702688464600215e-07\n", "chest X-ray: 1.431610030522279e-07\n", "immunohistochemistry histopathology: 7.39816812256322e-08\n", "hematoxylin and eosin histopathology: 1.3329999326572306e-08\n", "adenocarcinoma histopathology: 7.695529546936086e-09\n", "squamous cell carcinoma histopathology: 4.512937135103812e-09\n", "\n", "\n" ] } ], "source": [ "import glob\n", "from collections import OrderedDict\n", "\n", "import torch\n", "from PIL import Image\n", "import open_clip\n", "\n", "dataset_path = 'biomed-clip-share/example_data/biomed_image_classification_example_data'\n", "template = 'this is a photo of '\n", "labels = [\n", " 'adenocarcinoma histopathology',\n", " 'brain MRI',\n", " 'covid line chart',\n", " 'squamous cell carcinoma histopathology',\n", " 'immunohistochemistry histopathology',\n", " 'bone X-ray',\n", " 'chest X-ray',\n", " 'pie chart',\n", " 'hematoxylin and eosin histopathology'\n", "]\n", "\n", "test_imgs = glob.glob(dataset_path + '/*')\n", "\n", "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n", "model.to(device)\n", "model.eval()\n", "\n", "context_length = 256\n", "\n", "images = torch.stack([preprocess_val(Image.open(img)) for img in test_imgs]).to(device)\n", "texts = tokenizer([template + l for l in labels], context_length=context_length).to(device)\n", "with torch.no_grad():\n", " image_features, text_features, logit_scale = model(images, texts)\n", "\n", " logits = (logit_scale * image_features @ text_features.t()).detach().softmax(dim=-1)\n", " sorted_indices = torch.argsort(logits, dim=-1, descending=True)\n", "\n", " logits = logits.cpu().numpy()\n", " sorted_indices = sorted_indices.cpu().numpy()\n", "\n", "top_k = -1\n", "\n", "for i, img in enumerate(test_imgs):\n", " pred = labels[sorted_indices[i][0]]\n", "\n", " top_k = len(labels) if top_k == -1 else top_k\n", " print(img.split('/')[-1] + ':')\n", " for j in range(top_k):\n", " jth_index = sorted_indices[i][j]\n", " print(f'{labels[jth_index]}: {logits[i][jth_index]}')\n", " print('\\n')" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": { "id": "kIZEaLJB5H6A" }, "source": [ "
\n", "adenocarcinoma_histopathology.jpg:\n", "adenocarcinoma histopathology: 0.7818863987922668\n", "hematoxylin and eosin histopathology: 0.15517690777778625\n", "immunohistochemistry histopathology: 0.06149514392018318\n", "squamous cell carcinoma histopathology: 0.0014182085869833827\n", "chest X-ray: 2.017213228100445e-05\n", "brain MRI: 1.2181524198240368e-06\n", "pie chart: 7.932688959044754e-07\n", "bone X-ray: 7.436410101036017e-07\n", "covid line chart: 4.482610052036762e-07\n", "\n", "\n", "covid_line_chart.png:\n", "covid line chart: 0.9493210315704346\n", "adenocarcinoma histopathology: 0.01898195780813694\n", "squamous cell carcinoma histopathology: 0.0175501499325037\n", "immunohistochemistry histopathology: 0.006791787222027779\n", "hematoxylin and eosin histopathology: 0.003417333820834756\n", "brain MRI: 0.002629919210448861\n", "chest X-ray: 0.0010041205678135157\n", "bone X-ray: 0.00024685842799954116\n", "pie chart: 5.6812208640621975e-05\n", "\n", "\n", "bone_X-ray.jpg:\n", "bone X-ray: 0.9037961959838867\n", "hematoxylin and eosin histopathology: 0.07279316335916519\n", "brain MRI: 0.013534954749047756\n", "chest X-ray: 0.00821212213486433\n", "immunohistochemistry histopathology: 0.001647887285798788\n", "squamous cell carcinoma histopathology: 1.418814281350933e-05\n", "covid line chart: 1.1351590956110158e-06\n", "adenocarcinoma histopathology: 2.3802124360372545e-07\n", "pie chart: 9.433303205241828e-08\n", "\n", "\n", "pie_chart.png:\n", "pie chart: 0.999992847442627\n", "covid line chart: 6.056906840967713e-06\n", "brain MRI: 6.212158041307703e-07\n", "bone X-ray: 1.870277799298492e-07\n", "chest X-ray: 1.4315827456812258e-07\n", "immunohistochemistry histopathology: 7.397970591682679e-08\n", "hematoxylin and eosin histopathology: 1.3329795045535775e-08\n", "adenocarcinoma histopathology: 7.695367898463701e-09\n", "squamous cell carcinoma histopathology: 4.512833662317917e-09\n", "\n", "\n", "H_and_E_histopathology.jpg:\n", "hematoxylin and eosin histopathology: 0.7953251600265503\n", "immunohistochemistry histopathology: 0.19779996573925018\n", "chest X-ray: 0.005973907187581062\n", "bone X-ray: 0.0008049230673350394\n", "adenocarcinoma histopathology: 9.133991261478513e-05\n", "squamous cell carcinoma histopathology: 3.6423973597266013e-06\n", "brain MRI: 6.688684948130685e-07\n", "pie chart: 4.278819574210502e-07\n", "covid line chart: 3.051619401617245e-08\n", "\n", "\n", "brain_MRI.jpg:\n", "brain MRI: 0.9565795660018921\n", "hematoxylin and eosin histopathology: 0.041418157517910004\n", "immunohistochemistry histopathology: 0.0019450499676167965\n", "pie chart: 2.7151252652402036e-05\n", "squamous cell carcinoma histopathology: 1.0223812751064543e-05\n", "bone X-ray: 8.662499567435589e-06\n", "chest X-ray: 7.96773747424595e-06\n", "adenocarcinoma histopathology: 2.7692055937222904e-06\n", "covid line chart: 3.420084908611898e-07\n", "\n", "\n", "chest_X-ray.jpg:\n", "chest X-ray: 0.9998347759246826\n", "hematoxylin and eosin histopathology: 0.0001205605294671841\n", "bone X-ray: 4.112880560569465e-05\n", "immunohistochemistry histopathology: 1.0486423889233265e-06\n", "adenocarcinoma histopathology: 9.66637117016944e-07\n", "covid line chart: 9.508977996119938e-07\n", "brain MRI: 3.232386518448038e-07\n", "squamous cell carcinoma histopathology: 2.53368597213921e-07\n", "pie chart: 3.6984038054299617e-09\n", "\n", "\n", "squamous_cell_carcinoma_histopathology.jpeg:\n", "squamous cell carcinoma histopathology: 0.9469489455223083\n", "adenocarcinoma histopathology: 0.05259034037590027\n", "hematoxylin and eosin histopathology: 0.0003988408425357193\n", "immunohistochemistry histopathology: 6.187965482240543e-05\n", "chest X-ray: 1.4099594380923008e-08\n", "pie chart: 3.522500624519864e-10\n", "bone X-ray: 2.9633814846441453e-10\n", "brain MRI: 1.2720452469139332e-10\n", "covid line chart: 1.8425603924565603e-12\n", "\n", "\n", "IHC_histopathology.jpg:\n", "immunohistochemistry histopathology: 0.9465934634208679\n", "hematoxylin and eosin histopathology: 0.03232448548078537\n", "brain MRI: 0.020657211542129517\n", "adenocarcinoma histopathology: 0.000304735847748816\n", "bone X-ray: 4.5735167077509686e-05\n", "squamous cell carcinoma histopathology: 3.150868360535242e-05\n", "covid line chart: 2.0559578842949122e-05\n", "chest X-ray: 1.2715442608168814e-05\n", "pie chart: 9.55282575887395e-06\n", "\n", "