{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "collapsed_sections": [], "authorship_tag": "ABX9TyOjIcXwclnDxtt6VzX+P9Fq", "include_colab_link": true }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU", "widgets": { "application/vnd.jupyter.widget-state+json": { "f39f11d1075547bd81c2a71ab4e9d056": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_3c55a93ff9394478b696ad604223c406", "IPY_MODEL_ccf1f54ee22249028e2f8cc6d3397079", "IPY_MODEL_5743023322c2401bbea11750794498aa" ], "layout": "IPY_MODEL_593f36aefdb249608bee030b9905e2b0" } }, "3c55a93ff9394478b696ad604223c406": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_a820159a6d814a74a1ab3013521a41b4", "placeholder": "​", "style": "IPY_MODEL_d894666cefde4a6cb4864f9d6aecfdb8", "value": "100%" } }, "ccf1f54ee22249028e2f8cc6d3397079": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_cfc6ee0c109d428e96ac3778902104ab", "max": 1, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_095602ae14d141c9b74296bc18b390f7", "value": 1 } }, "5743023322c2401bbea11750794498aa": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_caac98cc706143d689e7eaddb25be4b4", "placeholder": "​", "style": "IPY_MODEL_88a7cf382ff64b78bbd440e3f82e02ea", "value": " 1/1 [00:00<00:00, 34.69it/s]" } }, "593f36aefdb249608bee030b9905e2b0": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a820159a6d814a74a1ab3013521a41b4": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d894666cefde4a6cb4864f9d6aecfdb8": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "cfc6ee0c109d428e96ac3778902104ab": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "095602ae14d141c9b74296bc18b390f7": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "caac98cc706143d689e7eaddb25be4b4": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "88a7cf382ff64b78bbd440e3f82e02ea": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "f2096bde6a524ff9a87c11c118a2c2ee": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_ae005c5b1ed6445097f27c9eadb44d00", "IPY_MODEL_9fa8126bacd749a3b374996c0bee7891", "IPY_MODEL_9558b9c30ce348928989e241942aecef" ], "layout": "IPY_MODEL_bd9374b595da45498874f2cc62cad206" } }, "ae005c5b1ed6445097f27c9eadb44d00": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_437cfda1d7ac4731a4673735ae8072e1", "placeholder": "​", "style": "IPY_MODEL_a7aa8c0ce09c4ee4bbccb2720b0ad4b4", "value": " 0%" } }, "9fa8126bacd749a3b374996c0bee7891": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "danger", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_ce8e7aa1b6154f78931aa32fd38c032f", "max": 1, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_dbd56444ab4a4d1799c5f164fae61e54", "value": 0 } }, "9558b9c30ce348928989e241942aecef": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_0a55eb4a977044a891795ffe23adfb28", "placeholder": "​", "style": "IPY_MODEL_5d0e9fb3a94a4f738545a84d470c846f", "value": " 0/1 [00:00<?, ?ba/s]" } }, "bd9374b595da45498874f2cc62cad206": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "437cfda1d7ac4731a4673735ae8072e1": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a7aa8c0ce09c4ee4bbccb2720b0ad4b4": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "ce8e7aa1b6154f78931aa32fd38c032f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "dbd56444ab4a4d1799c5f164fae61e54": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "0a55eb4a977044a891795ffe23adfb28": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "5d0e9fb3a94a4f738545a84d470c846f": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "view-in-github", "colab_type": "text" }, "source": [ "\"Open" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Ww6WXCvy-0lg", "outputId": "118a86a5-ed10-4970-f6c0-c6f1b76ff48b" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.23.1)\n", "Requirement already satisfied: datasets in /usr/local/lib/python3.7/dist-packages (2.6.1)\n", "Requirement already satisfied: evaluate in /usr/local/lib/python3.7/dist-packages (0.3.0)\n", "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.1)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n", "Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.1)\n", "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.13.1)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n", "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.13.0)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.1.1)\n", "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n", "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0.1)\n", "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.13)\n", "Requirement already satisfied: xxhash in /usr/local/lib/python3.7/dist-packages (from datasets) (3.1.0)\n", "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.18.0)\n", "Requirement already satisfied: dill<0.3.6 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.5.1)\n", "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (2022.8.2)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.7/dist-packages (from datasets) (3.8.3)\n", "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.3.5)\n", "Requirement already satisfied: asynctest==0.13.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (0.13.0)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (22.1.0)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.0.2)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.8.1)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (6.0.2)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.3.1)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.2.0)\n", "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.1.1)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.9.24)\n", "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n", "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n", "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.25.11)\n", "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.9.0)\n", "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2022.4)\n", "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n" ] } ], "source": [ "!pip install transformers==4.22.1 datasets==2.5.2 evaluate==0.2.2" ] }, { "cell_type": "code", "source": [ "import torch\n", "import numpy as np\n", "\n", "\n", "# 1. Dataset\n", "from datasets import load_dataset\n", "dataset = load_dataset(\"Adapting/abstract-keyphrases\")\n", "\n", "# 2. Model\n", "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n", "\n", "tokenizer = AutoTokenizer.from_pretrained(\"Adapting/KeyBartAdapter\")\n", "\n", "model = AutoModelForSeq2SeqLM.from_pretrained(\"Adapting/KeyBartAdapter\", revision = '9c3ed39c6ed5c7e141363e892d77cf8f589d5999')\n", "\n", "\n", "# 3. preprocess dataset\n", "dataset = dataset.shuffle()\n", "def preprocess_function(examples):\n", " inputs = examples['Abstract']\n", " targets = examples['Keywords']\n", " model_inputs = tokenizer(inputs, truncation=True)\n", "\n", " # Set up the tokenizer for targets\n", " with tokenizer.as_target_tokenizer():\n", " labels = tokenizer(targets, truncation=True)\n", "\n", " model_inputs[\"labels\"] = labels[\"input_ids\"]\n", " return model_inputs\n", "\n", "tokenized_dataset = dataset.map(\n", " preprocess_function,\n", " batched=True,\n", " remove_columns=dataset[\"train\"].column_names,\n", ")\n", "\n", "# 4. evaluation metrics\n", "def compute_metrics(eval_preds):\n", " preds = eval_preds.predictions\n", " labels = eval_preds.label_ids\n", " if isinstance(preds, tuple):\n", " preds = preds[0]\n", " print(preds.shape)\n", " if len(preds.shape) == 3:\n", " preds = preds.argmax(axis=-1)\n", " \n", " decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n", " # Replace -100 in the labels as we can't decode them.\n", " labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n", " decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n", "\n", " # Some simple post-processing\n", " decoded_preds = [a.strip().split(';') for a in decoded_preds]\n", " decoded_labels = [a.strip().split(';') for a in decoded_labels]\n", "\n", "\n", " precs, recalls, f_scores = [], [], []\n", " num_match, num_pred, num_gold = [], [], []\n", " for pred, label in zip(decoded_preds, decoded_labels):\n", " pred_set = set(pred)\n", " label_set = set(label)\n", " match_set = label_set.intersection(pred_set)\n", " p = float(len(match_set)) / float(len(pred_set)) if len(pred_set) > 0 else 0.0\n", " r = float(len(match_set)) / float(len(label_set)) if len(label_set) > 0 else 0.0\n", " f1 = float(2 * (p * r)) / (p + r) if (p + r) > 0 else 0.0\n", " precs.append(p)\n", " recalls.append(r)\n", " f_scores.append(f1)\n", " num_match.append(len(match_set))\n", " num_pred.append(len(pred_set))\n", " num_gold.append(len(label_set))\n", " \n", " # print(f'raw_PRED: {raw_pred}')\n", " print(f'PRED: num={len(pred_set)} - {pred_set}')\n", " print(f'GT: num={len(label_set)} - {label_set}')\n", " print(f'p={p}, r={r}, f1={f1}')\n", " print('-' * 20)\n", "\n", " result = {\n", " 'precision@M': np.mean(precs) * 100.0,\n", " 'recall@M': np.mean(recalls) * 100.0,\n", " 'fscore@M': np.mean(f_scores) * 100.0,\n", " 'num_match': np.mean(num_match),\n", " 'num_pred': np.mean(num_pred),\n", " 'num_gold': np.mean(num_gold),\n", " }\n", "\n", " result = {k: round(v, 2) for k, v in result.items()}\n", " return result\n", "\n", "# 5. train\n", "from transformers import DataCollatorForSeq2Seq,Seq2SeqTrainingArguments, Seq2SeqTrainer\n", "\n", "data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)\n", "\n", "model_name = 'KeyBartAdapter'\n", "num_epoch = 30\n", "\n", "args = Seq2SeqTrainingArguments(\n", " model_name,\n", " evaluation_strategy=\"epoch\",\n", " save_strategy=\"epoch\",\n", " learning_rate=2e-5,\n", " per_device_train_batch_size=4,\n", " per_device_eval_batch_size=4,\n", " weight_decay=0.01,\n", " save_total_limit=3,\n", " num_train_epochs=num_epoch,\n", " logging_steps=4,\n", " load_best_model_at_end=True,\n", " metric_for_best_model='fscore@M',\n", " predict_with_generate=True,\n", " fp16=torch.cuda.is_available(), # speeds up training on modern GPUs.\n", " # eval_accumulation_steps=10,\n", ")\n", "\n", "trainer = Seq2SeqTrainer(\n", " model,\n", " args,\n", " train_dataset=tokenized_dataset[\"train\"],\n", " eval_dataset=tokenized_dataset[\"train\"],\n", " data_collator=data_collator,\n", " tokenizer=tokenizer,\n", " compute_metrics=compute_metrics\n", ")\n", "\n", "trainer.train()\n" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000, "referenced_widgets": [ "f39f11d1075547bd81c2a71ab4e9d056", "3c55a93ff9394478b696ad604223c406", "ccf1f54ee22249028e2f8cc6d3397079", "5743023322c2401bbea11750794498aa", "593f36aefdb249608bee030b9905e2b0", "a820159a6d814a74a1ab3013521a41b4", "d894666cefde4a6cb4864f9d6aecfdb8", "cfc6ee0c109d428e96ac3778902104ab", "095602ae14d141c9b74296bc18b390f7", "caac98cc706143d689e7eaddb25be4b4", "88a7cf382ff64b78bbd440e3f82e02ea", "f2096bde6a524ff9a87c11c118a2c2ee", "ae005c5b1ed6445097f27c9eadb44d00", "9fa8126bacd749a3b374996c0bee7891", "9558b9c30ce348928989e241942aecef", "bd9374b595da45498874f2cc62cad206", "437cfda1d7ac4731a4673735ae8072e1", "a7aa8c0ce09c4ee4bbccb2720b0ad4b4", "ce8e7aa1b6154f78931aa32fd38c032f", "dbd56444ab4a4d1799c5f164fae61e54", "0a55eb4a977044a891795ffe23adfb28", "5d0e9fb3a94a4f738545a84d470c846f" ] }, "id": "OYPmfKRY-6tC", "outputId": "eaf77000-920e-4aec-e38c-1f9f3f3f4c1c" }, "execution_count": null, "outputs": [ { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "WARNING:datasets.builder:Using custom data configuration Adapting--abstract-keyphrases-4811abd1e624c6b0\n", "WARNING:datasets.builder:Found cached dataset csv (/root/.cache/huggingface/datasets/Adapting___csv/Adapting--abstract-keyphrases-4811abd1e624c6b0/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "f39f11d1075547bd81c2a71ab4e9d056", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/1 [00:00\n", " \n", " \n", " [378/390 38:43 < 01:14, 0.16 it/s, Epoch 29/30]\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossPrecision@mRecall@mFscore@mNum MatchNum PredNum Gold
11.4847000.82306355.00000037.33000043.5900001.2200002.2400003.280000
20.8368000.57559263.07000062.60000061.4300001.9600003.2400003.280000
30.6649000.43162878.07000068.97000071.8100002.1800002.8600003.280000
40.3905000.28010690.67000069.63000077.3400002.1600002.4200003.280000
50.3449000.31627192.17000070.20000078.5000002.2400002.4600003.280000
60.3554000.22360990.83000080.43000084.5800002.5600002.8600003.280000
70.3433000.32088388.00000080.43000083.3700002.5400002.9200003.280000
80.3082000.18644991.50000083.50000086.7100002.6400002.9200003.280000
90.1569000.25927291.43000086.77000088.5000002.7400003.0400003.280000
100.1300000.19101589.43000085.70000087.1200002.7000003.0800003.280000
110.1319000.14433289.93000086.87000088.0900002.7400003.1000003.280000
120.1511000.16092392.27000085.27000088.0700002.7000002.9600003.280000
130.1046000.18138891.77000085.53000088.0800002.7000002.9800003.280000
140.1102000.24793891.43000086.37000088.3100002.7200003.0200003.280000
150.1576000.24002292.27000087.60000089.3800002.7600003.0400003.280000
160.1265000.13378293.27000088.33000090.3200002.8000003.0400003.280000
170.0538000.15804091.43000089.00000089.9300002.8200003.1400003.280000
180.0560000.25000492.03000087.03000088.9700002.7400003.0200003.280000
190.0832000.16743593.37000086.77000089.3300002.7400002.9800003.280000
200.0992000.16018090.20000087.70000088.6800002.7600003.1200003.280000
210.0315000.14467191.37000087.43000088.9400002.7600003.0800003.280000
220.0621000.16700890.37000088.10000089.0100002.7800003.1400003.280000
230.0546000.12408291.27000088.50000089.5800002.8000003.1200003.280000
240.0335000.13674792.27000087.83000089.5100002.7800003.0600003.280000
250.0767000.12872793.27000087.17000089.5700002.7600003.0000003.280000
260.0922000.17543990.27000087.83000088.7700002.7800003.1400003.280000
270.0897000.18570890.93000088.50000089.4300002.8000003.1400003.280000
280.0229000.17171791.27000088.50000089.5800002.8000003.1200003.280000
290.0387000.14228691.27000088.50000089.5800002.8000003.1200003.280000

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=2 - {'transfer learning', 'natural language processing'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'short-term session', 'hypergraph attention network', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'descriptive text captions', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=0.5, r=0.3333333333333333, f1=0.4\n", "--------------------\n", "PRED: num=2 - {'diffusion-based generative model', 'human motion generation'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=0.5, r=0.3333333333333333, f1=0.4\n", "--------------------\n", "PRED: num=2 - {'self-attention mechanism', 'spiking Neural network(snn)'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=0.5, r=0.3333333333333333, f1=0.4\n", "--------------------\n", "PRED: num=2 - {'medical question understanding and answering system', 'knowledge grounding and semantic self-supervision'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=0.5, r=0.25, f1=0.3333333333333333\n", "--------------------\n", "PRED: num=2 - {'multi-modal image and text representations', 'diverse 3d object synthesis'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.5, r=0.25, f1=0.3333333333333333\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'graph neural network', 'feature fusion'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=0.5, r=0.3333333333333333, f1=0.4\n", "--------------------\n", "PRED: num=3 - {'transfer learning', 'multi-task learning', 'unsupervised learning'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'cognitive multilayer network', 'mental lexicon'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'pre-training', 'transferable spatiotemporal representation'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=0.5, r=0.3333333333333333, f1=0.4\n", "--------------------\n", "PRED: num=2 - {'cross-lingual transfer', 'machine translation'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=0.5, r=0.25, f1=0.3333333333333333\n", "--------------------\n", "PRED: num=3 - {'few-shot method', 'sentence transformer fine-tuning', 'sentence'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=2 - {'knowledge transferability', 'medical image domain'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=1.0, r=0.4, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'bayesian optimization', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'speech processing system', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {\"fitts' law\", 'expectation-conditional-maximization algorithm'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.5, r=0.3333333333333333, f1=0.4\n", "--------------------\n", "PRED: num=2 - {'cohesive structure', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=1.0, r=0.4, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'integrable model of quantum field theory', 'form factor programme', 'fermionic scattering state'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=2 - {'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'knowledge probing', 'knowledge graph completion'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=2 - {'multi-view Contrastive learning framework', 'vision-language Pre-trained model'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=2 - {'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'dialogue agent', 'implicit commonsense knowledge'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'quantum network', 'quantum computer', 'information theoretic security'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=2 - {'phoneme recognition', 'speech recognition'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'cross entropy loss', 'vision \\\\&language model', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=0.6666666666666666, r=1.0, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'low-frequency word prediction', 'frequency-aware token-level contrastive learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=2 - {'moderate length low-density parity-check code', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=0.5, r=0.3333333333333333, f1=0.4\n", "--------------------\n", "PRED: num=2 - {'slimmable network', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'contrastive learning', 'semi-supervised segmentation', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'legal citation network', 'legal Statute identification', 'heterogeneous graph'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=2 - {'multi-dialect automatic speech recognition', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'language-vision research and applications', 'deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=2 - {'semi-parametric language model', 'task-agnostic unlabeled corpus'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=0.5, r=0.3333333333333333, f1=0.4\n", "--------------------\n", "PRED: num=2 - {'cooperative environmental learning', 'multi-robot system'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'decision tree', 'time-incremental learning framework'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.5, r=0.3333333333333333, f1=0.4\n", "--------------------\n", "PRED: num=2 - {'non-referential face image quality assessment method', 'presentation attack detection'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language task', 'task-agnostic memory architecture'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=2 - {'semi-supervised learning', 'neural processes'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'automatic speech recognition(asr)', 'multi-head attention', 'sparse attention'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=2 - {'information removal', 'deep learning model'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=0.5, r=0.25, f1=0.3333333333333333\n", "--------------------\n", "PRED: num=2 - {'data budgeting', 'data investment'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithm', 'facial image database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=3 - {'convex optimization', 'deep learning', 'sparse recovery perspective'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'minimal alignment'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=2 - {'personalization', 'large text-to-image model'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-13\n", "Configuration saved in KeyBartAdapter/checkpoint-13/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-13/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-13/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-13/special_tokens_map.json\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=3 - {'transfer learning', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'session-based recommender system', 'dynamic item embedding', 'hypergraph attention network'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=0.3333333333333333, r=0.3333333333333333, f1=0.3333333333333333\n", "--------------------\n", "PRED: num=3 - {'text-to', 'descriptive text caption', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'generative model', 'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=0.75, r=1.0, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=4 - {'self-attention mechanism', '', 'biologically plausible structure', 'spiking neural network'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'knowledge grounding', 'semantic self-supervision', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'natural language', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'qualitative data', 'qualitative visualization', 'nLP capabilities'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=0.6666666666666666, r=1.0, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'graph neural network', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'cognitive multilayer network', 'c', 'cognitive network', 'mental lexicon'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.75, r=1.0, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=4 - {'spatiotemporal representation learning', '', 'pre-training', 'language semantics'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=4 - {'natural language processing task', 'cross-lingual transfer', 'language', 'machine translation'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=0.25, r=0.25, f1=0.25\n", "--------------------\n", "PRED: num=3 - {'prompt-free framework', 'sentence transformer fine-tuning', 'few-shot'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=4 - {'knowledge transferability', 'domain', 'pre-trained vision language model', 'medical image domain'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=0.5, r=0.4, f1=0.4444444444444445\n", "--------------------\n", "PRED: num=2 - {'bayesian optimization', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'speech processing system', 'multilingual supervision', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {\"fitts' law\", 'expectation-conditional-maximization algorithm'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.5, r=0.3333333333333333, f1=0.4\n", "--------------------\n", "PRED: num=3 - {'cohesive structure', 'maximal k-biplex enumer', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'operator-algebraic construction', '', 'integrable model', 'quantum field theory'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=0.25, r=0.3333333333333333, f1=0.28571428571428575\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'knowledge probing', 'knowledge graph', 'knowledge graph completion'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=0.3333333333333333, r=0.3333333333333333, f1=0.3333333333333333\n", "--------------------\n", "PRED: num=3 - {'', 'vision-language Pre-trained model', 'multi-view Contrastive learning'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'dialogue agent', 'implicit commonsense knowledge'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=6 - {'', 'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=0.8333333333333334, r=1.0, f1=0.9090909090909091\n", "--------------------\n", "PRED: num=4 - {'phoneme recognition', 'speech recognition', 'self-', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=0.75, r=1.0, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'token-level contrastive learning', 'neural machine translation', 'low-frequency word prediction'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=4 - {'low-density', 'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=0.75, r=1.0, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'medical AI', 'contrastive learning', 'semi-supervised segmentation network'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=4 - {'legal statute identification task', 'legal citation network', 'citation network', 'heterogeneous graph'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=0.5, r=1.0, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'speech recognition', 'multi-dialect automatic speech recognition', 'deep neural networks', 'multi'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.25, r=0.25, f1=0.25\n", "--------------------\n", "PRED: num=4 - {'language', 'language vision', 'language-vision research and application', 'deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=2 - {'semi-parametric language model', 'task-agnostic unlabeled'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=0.5, r=0.3333333333333333, f1=0.4\n", "--------------------\n", "PRED: num=3 - {'multi-robot system', 'cooperative environmental learning algorithm', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'', 'decision tree', 'time-incremental learning', 'cyber-physical system'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'non-referential face image quality assessment', 'quality assessment method', 'presentation attack detection'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=0.3333333333333333, r=0.3333333333333333, f1=0.3333333333333333\n", "--------------------\n", "PRED: num=4 - {'knowledge-intensive language task', 'knowledge', 'task-agnostic memory architecture', 'knowledge source'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'information removal', 'mutual information minimization', 'deep learning model'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=0.3333333333333333, r=0.25, f1=0.28571428571428575\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data investment'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'gender classification algorithm', 'benchmark database', 'gender minority subgroup'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=0.3333333333333333, r=0.5, f1=0.4\n", "--------------------\n", "PRED: num=4 - {'convex optimization', 'sparse recovery', 'deep learning', 'neural network'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=5 - {'genetic variant', 'genetics', 'gen', 'genes', 'genetic variation'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=0.2, r=0.5, f1=0.28571428571428575\n", "--------------------\n", "PRED: num=3 - {'personalization', 'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=0.6666666666666666, r=1.0, f1=0.8\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-26\n", "Configuration saved in KeyBartAdapter/checkpoint-26/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-26/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-26/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-26/special_tokens_map.json\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'hypergraph attention network', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'text-to', 'descriptive text caption', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'human motion generation', 'motion diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'self-attention mechanism', 'biologically plausible structure', 'spiking neural network'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'knowledge grounding', 'semantic self-supervision', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'synthesize', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'graph neural network', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'transfer learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'cognitive multilayer network', 'mental lexicon', 'quantitative mechanism'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'pretext task', 'pre-training', 'spatiotemporal representation', 'language semantics'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'language-family adapter', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'fine-tuning', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=2 - {'one-shot hybrid kg', 'bayesian optimization'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'handwritten text recognition', 'end-to-end'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'speech processing system', 'multilingual supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'expectation-cond', \"fitts' law\", 'two-component mixture structure'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'', 'cohesive structure', 'mining maximal subgraphs', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.5, r=0.4, f1=0.4444444444444445\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic construction', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'contrastive', 'vision-language pre-trained model', 'cross-modal representation'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=2 - {'free-text retrieval', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=4 - {'pre-trained conversational model', 'implicit commonsense', 'dialogue agent', ''}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.5, r=0.4, f1=0.4444444444444445\n", "--------------------\n", "PRED: num=4 - {'quantum network', 'quantum computer', 'information theory', 'communication complexity'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=0.8, f1=0.888888888888889\n", "--------------------\n", "PRED: num=4 - {'phoneme recognition', 'speech recognition', 'ph', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=0.75, r=1.0, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'contrastive learning', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'legal statute identification task', 'citation network', 'heterogeneous graph'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=0.6666666666666666, r=1.0, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'multi-dialect automatic speech recognition', 'deep neural networks'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.5, r=0.25, f1=0.3333333333333333\n", "--------------------\n", "PRED: num=3 - {'benchmarking', 'language-vision task', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.3333333333333333, r=0.25, f1=0.28571428571428575\n", "--------------------\n", "PRED: num=3 - {'semi-parametric', 'task-agnostic', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'non-referential face image quality assessment method', 'presentation attack detection'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'automatic speech recognition', 'sparse attention'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'privacy', 'deep learning', 'mutual information minimization'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data investment'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'gender classification algorithm', 'benchmark database', 'gender minority subgroup'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=0.3333333333333333, r=0.5, f1=0.4\n", "--------------------\n", "PRED: num=4 - {'convex optimization', 'relu networks', 'sparse recovery', 'deep learning'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=1.0, r=0.8, f1=0.888888888888889\n", "--------------------\n", "PRED: num=5 - {'genetic variant', 'genetics', 'gen', 'genotype', 'genetic variation'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=0.2, r=0.5, f1=0.28571428571428575\n", "--------------------\n", "PRED: num=3 - {'personalization', 'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=0.6666666666666666, r=1.0, f1=0.8\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-39\n", "Configuration saved in KeyBartAdapter/checkpoint-39/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-39/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-39/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-39/special_tokens_map.json\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=3 - {'transfer learning', 'transformer', 'natural language processing'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'hypergraph attention network', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'audio sample generation', 'descriptive text caption', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'human motion generation', 'motion diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'spiking neural network', 'self-attention mechanism'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'knowledge grounding and semantic self-supervision', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=0.5, r=0.25, f1=0.3333333333333333\n", "--------------------\n", "PRED: num=3 - {'synthesize', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'graph neural network', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'cognitive multilayer network', 'mental lexicon'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'spatiotemporal representation learning', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'fine-tuning', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=2 - {'bayesian optimization', 'stochastic expensive black box function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'handwritten text recognition', 'end-to-end'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'speech processing system', 'robust speech processing'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'expectation-cond', \"fitts' law\", 'two-component mixture structure'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic construction', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'vision-language pre-trained model', 'dual encoder', 'contrastive learning'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=2 - {'free-text retrieval', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'dial', 'implicit commonsense (cs) knowledge'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'quantum network', 'quantitative communication', 'quantum computer', 'information theory'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=0.8, f1=0.888888888888889\n", "--------------------\n", "PRED: num=2 - {'phoneme recognition', 'speech recognition'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'cross entropy loss', 'vision \\\\&language model', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=0.6666666666666666, r=1.0, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'reinforcement learning', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'segmentation of images', 'semi-supervised segmentation', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'multi-dialect automatic speech recognition', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'language-vision task', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'non-referential face image quality assessment method', 'face recognition system', 'quality assessment method'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=0.3333333333333333, r=0.3333333333333333, f1=0.3333333333333333\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semi-supervised learning', 'neural processes'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'privacy', 'deep learning'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'relu networks', 'deep learning'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=1.0, r=0.4, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relation'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-52\n", "Configuration saved in KeyBartAdapter/checkpoint-52/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-52/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-52/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-52/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-13] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=3 - {'transfer learning', 'transformer', 'natural language processing'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'auto-regressive generative', 'descriptive text caption', 'generating audio sample'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'spiking neural network', 'self-attention mechanism'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'medical question understanding and answering system', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'synthes', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'graph neural network', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'cognitive multilayer network', 'mental lexicon'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'fine-tuning', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'speech processing system', 'multilingual supervision', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'expectation-conditional-maximization', \"fitts' law\"}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=4 - {'maximal', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic construction', 'minkowski', 'quantum field theory'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=0.3333333333333333, r=0.3333333333333333, f1=0.3333333333333333\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=1 - {'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=0.3333333333333333, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'vision-language pre-trained model', 'dual encoder', 'contrastive learning'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=2 - {'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'external'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'quantum network', 'quantum computer', 'information theory', 'communication complexity'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=0.8, f1=0.888888888888889\n", "--------------------\n", "PRED: num=2 - {'phoneme recognition', 'speech recognition'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'reinforcement learning', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'acoustic model', 'multi-dialect automatic speech recognition', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=1 - {'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=0.3333333333333333, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'decision tree', 'real-time and human-interpretable decision-making'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'non-referential face image quality assessment method', 'quality assessment', 'face recognition system'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=1 - {'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'semi-supervised learning', 'neural processes'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'automatic speech recognition', 'sparse attention'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'convex optimization', 'deep learning', 'neural networks'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'large text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-65\n", "Configuration saved in KeyBartAdapter/checkpoint-65/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-65/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-65/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-65/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-26] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'hypergraph attention network', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'auto-regressive generative', 'descriptive text caption', 'generating audio sample'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'knowledge grounding', 'semantic self-supervision', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cognitive multilayer network', 'quantitative and interpretative framework', 'mental lexicon'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pre-training', 'language semantics'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'fine-tuning', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'speech processing system', 'multilingual supervision', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'expectation-cond', \"fitts' law\", 'two-component mixture structure'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k-biplex', 'cohesive structure', 'mining maximal subgraph', 'reverse'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'vision-language pre-trained model', 'cross-modal representation', 'contrastive learning'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=2 - {'free-text retrieval', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'external knowledge'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=4 - {'quantum network', 'quantum computer', 'information theory', 'communication complexity'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=0.8, f1=0.888888888888889\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'token-level', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'medical AI', 'contrastive learning', 'semi-supervised segmentation network'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'acoustic model', 'multi-dialect automatic speech recognition', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=4 - {'online distributed learning', 'cooperative environmental learning', 'multi-robot system', 'environment'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'non-referential face image quality assessment method', 'face recognition system', 'presentation attack detection'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=0.0, r=0.0, f1=0.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'convex optimization', 'relu network', 'sparse recovery', 'deep learning'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'large text-to-image diffusion model', '', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=0.3333333333333333, r=0.5, f1=0.4\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-78\n", "Configuration saved in KeyBartAdapter/checkpoint-78/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-78/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-78/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-78/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-39] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'auto-regressive generative', 'descriptive text caption', 'generating audio sample'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'self-attention mechanism', '', 'spiking neural network', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=0.75, r=1.0, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'medical question understanding and answering system', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'synthesize', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cognitive multilayer network', 'mental lexicon', 'mental network'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pre-training', 'language semantics'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'fine-tuning', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'speech processing', 'speech processing system', 'robust speech processing'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge graph', 'knowledge graph completion'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=0.5, r=0.3333333333333333, f1=0.4\n", "--------------------\n", "PRED: num=3 - {'cross-modal representation', 'vision-language pre-trained model', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'quantum network', 'quantum computer', 'information theory', 'communication complexity'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=0.8, f1=0.888888888888889\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'token-level', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'legal citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'multi dialectal corpus', '', 'multi-dialect automatic speech recognition', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'language-vision', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'non-referential face image quality assessment method', 'face recognition system', 'quality assessment method'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=0.3333333333333333, r=0.3333333333333333, f1=0.3333333333333333\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'bias domain', 'information removal', 'deep learning', 'privacy domain'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'convex optimization', 'sparse recovery', 'deep learning', 'neural networks'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=1.0, r=0.8, f1=0.888888888888889\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-91\n", "Configuration saved in KeyBartAdapter/checkpoint-91/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-91/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-91/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-91/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-52] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=3 - {'transfer learning', 'transformer', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'auto-regressive generative', 'descriptive text caption', 'generating audio sample'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'self-attention mechanism', 'spiking neural network', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'knowledge grounding', 'semantic self-supervision', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'transfer learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'cognitive multilayer network', 'quantitative and interpretative', 'mental lexicon'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pre-training', 'language semantics'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'robust speech processing', 'multilingual supervision', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=4 - {'', 'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=0.75, r=1.0, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'vision-language pre-trained model', 'dual encoder', 'contrastive learning'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'external knowledge'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=4 - {'quantum network', 'quantum computer', 'information theory', 'communication complexity'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=0.8, f1=0.888888888888889\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'segmentation of images', 'contrastive learning', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'acoustic model', 'multi-dialect automatic speech recognition', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'language-vision', 'open-source deep learning library', 'benchmarking'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'non-referential face image quality assessment method', 'face recognition', 'presentation attack detection'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=0.3333333333333333, r=0.3333333333333333, f1=0.3333333333333333\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semi-supervised learning', 'neural processes'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'convex optimization', 'relu networks', 'sparse recovery', 'deep learning'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=1.0, r=0.8, f1=0.888888888888889\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-104\n", "Configuration saved in KeyBartAdapter/checkpoint-104/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-104/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-104/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-104/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-65] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'auto-regressive generative', 'descriptive text caption', 'generating audio sample'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'medical question understanding and answering system', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cognitive multilayer network', 'quantitative and interpretative framework', 'mental lexicon'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cross-modal representation', 'vision-language pre-trained model', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'quantum network', 'quantum computer', 'information theory', 'communication complexity'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=0.8, f1=0.888888888888889\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'acoustic model', 'multi-dialect automatic speech recognition', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=0.75, r=1.0, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 're'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-117\n", "Configuration saved in KeyBartAdapter/checkpoint-117/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-117/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-117/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-117/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-78] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=3 - {'transfer learning', 'transformer', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'spiking neural network', 'self-attention mechanism'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'knowledge grounding', 'semantic self-supervision', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'cognitive multilayer network', '', 'mental lexicon', 'conceptual association'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pre-training', 'language semantics'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'medical prompt', 'medical image domain', 'large-scale pre-trained vision language model', 'knowledge'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'vision-language pre-trained model', 'cross-modal', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'quantum network', 'quantum computer', 'information theory', 'communication complexity'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=0.8, f1=0.888888888888889\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'multilingual', 'multi-dialect automatic speech recognition', 'multi', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=0.75, r=1.0, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-130\n", "Configuration saved in KeyBartAdapter/checkpoint-130/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-130/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-130/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-130/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-91] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'knowledge grounding', 'semantic self-supervision', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'cognitive multilayer network', 'c', 'mental lexicon', 'conceptual association'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pre-training', 'language semantics'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'vision-language pre-trained model', 'cross-modal', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'quantum network', 'quantum computer', 'information theory', 'communication complexity'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=0.8, f1=0.888888888888889\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'multilingual', 'multi-dialect automatic speech recognition', 'multi', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-143\n", "Configuration saved in KeyBartAdapter/checkpoint-143/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-143/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-143/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-143/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-104] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'spiking neural network', 'self-attention mechanism'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'knowledge grounding', 'semantic self-supervision', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'graph neural network', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cognitive multilayer network', 'mental lexicon', 'conceptual association'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'expectation-cond', \"fitts' law\", 'two-component mixture structure'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'vision-language pre-trained model', 'dual encoder', 'cross-mod'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'multilingual', 'multi-dialect automatic speech recognition', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'semi-parametric language model', 'task-agn', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'non-referential face image quality assessment method', 'quality assessment', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=0.3333333333333333, r=0.3333333333333333, f1=0.3333333333333333\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-156\n", "Configuration saved in KeyBartAdapter/checkpoint-156/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-156/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-156/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-156/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-130] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'knowledge grounding', 'semantic self-supervision', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cognitive multilayer network', 'mental lexicon', 'mental network'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'spatiotemporal representation learning', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'expectation-cond', \"fitts' law\", 'two-component mixture structure'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k-biplex', 'cohesive structure', 'mining maximal subgraph', 'reverse'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'vision-language pre-trained model', 'cross-modal', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', '', 'implicit commonsense (cs) knowledge'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'quantum network', 'quantum computer', 'information theory', 'communication complexity'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=0.8, f1=0.888888888888889\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'token-level', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'multilingual', 'multi-dialect automatic speech recognition', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'semi-parametric language model', 'task-agn', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'non-referential face image quality assessment method', 'quality assessment', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=0.3333333333333333, r=0.3333333333333333, f1=0.3333333333333333\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 're'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-169\n", "Configuration saved in KeyBartAdapter/checkpoint-169/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-169/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-169/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-169/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-143] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'medical question understanding and answering system', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'cognitive multilayer network', 'c', 'network structure', 'mental lexicon'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'vision-language pre-trained model', 'dual encoder', 'cross-mod'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', '', 'implicit commonsense (cs) knowledge'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'quantum network', 'quantum computer', 'information theory', 'communication complexity'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=0.8, f1=0.888888888888889\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'frequency-aware token', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'multilingual', 'multi-dialect automatic speech recognition', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'semi-parametric language model', 'task-agn', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-182\n", "Configuration saved in KeyBartAdapter/checkpoint-182/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-182/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-182/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-182/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-156] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'medical question understanding and answering system', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cognitive multilayer network', 'multi-layer network', 'mental lexicon'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end', 'feature'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'speech processing system', 'multilingual supervision', 'robust speech processing'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cross-modal representation', 'vision-language pre-trained model', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'quantum network', 'quantum computer', 'information theory', 'communication complexity'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=0.8, f1=0.888888888888889\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'multilingual', 'multi-dialect automatic speech recognition', 'multi', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=0.75, r=1.0, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-195\n", "Configuration saved in KeyBartAdapter/checkpoint-195/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-195/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-195/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-195/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-117] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'knowledge grounding', 'semantic self-supervision', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'graph neural network', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cognitive multilayer network', 'multi-layer network', 'mental lexicon'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end', 'feature'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'speech processing system', 'multilingual supervision', 'robust speech processing'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cross-modal representation', 'vision-language pre-trained model', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'acoustic model', '', 'multi-dialect automatic speech recognition', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-208\n", "Configuration saved in KeyBartAdapter/checkpoint-208/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-208/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-208/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-208/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-169] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'knowledge grounding', 'medical question understanding and answering', 'semantic', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'cognitive multilayer network', 'c', 'mental lexicon', 'mental network'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end', 'feature'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cross-modal representation', 'vision-language pre-trained model', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'acoustic model', '', 'multi-dialect automatic speech recognition', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-221\n", "Configuration saved in KeyBartAdapter/checkpoint-221/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-221/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-221/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-221/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-182] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'medical question understanding and answering system', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cognitive multilayer network', 'mental lexicon', 'mental network'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end', 'feature'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'expectation-cond', \"fitts' law\", 'two-component mixture structure'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'vision-language pre-trained model', 'cross-modal', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=5 - {'', 'quantitative communication', 'information theory', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'token-level', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'multi-dialect automatic speech recognition', 'deep neural network', 'multidialect'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'task-', 'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-234\n", "Configuration saved in KeyBartAdapter/checkpoint-234/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-234/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-234/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-234/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-195] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'medical question understanding and answering system', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'graph neural network', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cognitive multilayer network', 'mental lexicon'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end', 'feature'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'expectation-cond', \"fitts' law\", 'two-component mixture structure'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cross-modal representation', 'vision-language pre-trained model', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=5 - {'', 'quantitative communication', 'information theory', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'multi-dialect automatic speech recognition', 'deep neural network', 'multidialect'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'deep', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'task-', 'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-247\n", "Configuration saved in KeyBartAdapter/checkpoint-247/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-247/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-247/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-247/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-221] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'knowledge grounding', 'medical question understanding and answering', 'semantic self', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'cognitive multilayer network', 'cognition', 'c', 'mental lexicon'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end', 'feature'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'vision-language pre-trained model', 'cross-modal', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=5 - {'', 'quantitative communication', 'information theory', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'multi-dialect automatic speech recognition', 'deep neural network', 'multidialect'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'deep learning', 'open-source deep learning library', 'language learning'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 're'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-260\n", "Configuration saved in KeyBartAdapter/checkpoint-260/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-260/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-260/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-260/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-234] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'medical question understanding and answering system', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'cognitive multilayer network', 'cognition', 'mental lexicon', ''}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end', 'feature'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'expectation-cond', \"fitts' law\", 'two-component mixture structure'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'vision-language pre-trained model', 'dual encoder', 'contrastive learning'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=5 - {'', 'quantitative communication', 'information theory', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'multi-dialect automatic speech recognition', 'deep neural network', 'multidialect'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'deep', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'semi-parametric language model', 'task-agn', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 're'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-273\n", "Configuration saved in KeyBartAdapter/checkpoint-273/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-273/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-273/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-273/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-247] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'knowledge grounding', 'medical question understanding and answering', 'semantic self', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'cognitive multilayer network', 'cognition', 'mental lexicon', ''}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end', 'feature'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'vision-language pre-trained model', 'dual encoder', 'contrastive learning'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=5 - {'', 'quantitative communication', 'information theory', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'multi-dialect automatic speech recognition', 'deep neural network', 'multidialectal'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 're'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-286\n", "Configuration saved in KeyBartAdapter/checkpoint-286/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-286/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-286/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-286/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-260] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'knowledge grounding', 'medical question understanding and answering', 'semantic self', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'cognitive multilayer network', 'cognition', 'mental lexicon', ''}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end', 'feature'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cross-modal representation', 'vision-language pre-trained model', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'multi-dialect automatic speech recognition', 'deep neural network', 'multidialectal corpus'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-299\n", "Configuration saved in KeyBartAdapter/checkpoint-299/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-299/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-299/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-299/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-273] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'medical question understanding and answering system', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'cognitive multilayer network', 'cognition', 'mental lexicon', ''}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end', 'feature'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'expectation-cond', \"fitts' law\", 'two-component mixture structure'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cross-modal representation', 'vision-language pre-trained model', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'multi-dialect automatic speech recognition', 'deep neural network', 'multidialectal corpus'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'semi-parametric language model', 'task-agn', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-312\n", "Configuration saved in KeyBartAdapter/checkpoint-312/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-312/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-312/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-312/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-286] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'medical question understanding and answering system', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=1.0, r=0.5, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'graph neural network', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cognitive multilayer network', 'mental lexicon'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end', 'feature'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'expectation-cond', \"fitts' law\", 'two-component mixture structure'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cross-modal representation', 'vision-language pre-trained model', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'multi-dialect automatic speech recognition', 'deep neural network', 'multidialectal corpus'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'semi-parametric language model', 'task-agn', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=1.0, r=0.6666666666666666, f1=0.8\n", "--------------------\n", "PRED: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-325\n", "Configuration saved in KeyBartAdapter/checkpoint-325/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-325/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-325/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-325/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-299] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'knowledge grounding', 'medical question understanding and answering', 'semantic', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'cognitive multilayer network', 'cognition', 'c', 'mental lexicon'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end', 'feature'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cross-modal representation', 'vision-language pre-trained model', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'multilingual', 'multi-dialect automatic speech recognition', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'semi-parametric language model', 'task-agn', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-338\n", "Configuration saved in KeyBartAdapter/checkpoint-338/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-338/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-338/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-338/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-312] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'knowledge grounding', 'sem', 'medical question understanding and answering', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'cognitive multilayer network', 'cognition', 'mental lexicon', ''}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end', 'feature'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cross-modal representation', 'vision-language pre-trained model', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'multilingual', 'multi-dialect automatic speech recognition', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-351\n", "Configuration saved in KeyBartAdapter/checkpoint-351/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-351/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-351/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-351/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-325] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'knowledge grounding', 'medical question understanding and answering', 'semantic', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'cognitive multilayer network', 'cognition', 'mental lexicon', ''}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end', 'feature'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cross-modal representation', 'vision-language pre-trained model', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'multi-dialect automatic speech recognition', 'multi dialectal', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-364\n", "Configuration saved in KeyBartAdapter/checkpoint-364/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-364/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-364/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-364/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-338] due to args.save_total_limit\n", "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'knowledge grounding', 'medical question understanding and answering', 'semantic', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'cognitive multilayer network', 'cognition', 'c', 'mental lexicon'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end', 'feature'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cross-modal representation', 'vision-language pre-trained model', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'multi-dialect automatic speech recognition', 'multi dialectal', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "output_type": "stream", "name": "stderr", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-377\n", "Configuration saved in KeyBartAdapter/checkpoint-377/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-377/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-377/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-377/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-351] due to args.save_total_limit\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "" ], "text/html": [ "\n", "

\n", " \n", " \n", " [390/390 42:24, Epoch 30/30]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossPrecision@mRecall@mFscore@mNum MatchNum PredNum Gold
11.4847000.82306355.00000037.33000043.5900001.2200002.2400003.280000
20.8368000.57559263.07000062.60000061.4300001.9600003.2400003.280000
30.6649000.43162878.07000068.97000071.8100002.1800002.8600003.280000
40.3905000.28010690.67000069.63000077.3400002.1600002.4200003.280000
50.3449000.31627192.17000070.20000078.5000002.2400002.4600003.280000
60.3554000.22360990.83000080.43000084.5800002.5600002.8600003.280000
70.3433000.32088388.00000080.43000083.3700002.5400002.9200003.280000
80.3082000.18644991.50000083.50000086.7100002.6400002.9200003.280000
90.1569000.25927291.43000086.77000088.5000002.7400003.0400003.280000
100.1300000.19101589.43000085.70000087.1200002.7000003.0800003.280000
110.1319000.14433289.93000086.87000088.0900002.7400003.1000003.280000
120.1511000.16092392.27000085.27000088.0700002.7000002.9600003.280000
130.1046000.18138891.77000085.53000088.0800002.7000002.9800003.280000
140.1102000.24793891.43000086.37000088.3100002.7200003.0200003.280000
150.1576000.24002292.27000087.60000089.3800002.7600003.0400003.280000
160.1265000.13378293.27000088.33000090.3200002.8000003.0400003.280000
170.0538000.15804091.43000089.00000089.9300002.8200003.1400003.280000
180.0560000.25000492.03000087.03000088.9700002.7400003.0200003.280000
190.0832000.16743593.37000086.77000089.3300002.7400002.9800003.280000
200.0992000.16018090.20000087.70000088.6800002.7600003.1200003.280000
210.0315000.14467191.37000087.43000088.9400002.7600003.0800003.280000
220.0621000.16700890.37000088.10000089.0100002.7800003.1400003.280000
230.0546000.12408291.27000088.50000089.5800002.8000003.1200003.280000
240.0335000.13674792.27000087.83000089.5100002.7800003.0600003.280000
250.0767000.12872793.27000087.17000089.5700002.7600003.0000003.280000
260.0922000.17543990.27000087.83000088.7700002.7800003.1400003.280000
270.0897000.18570890.93000088.50000089.4300002.8000003.1400003.280000
280.0229000.17171791.27000088.50000089.5800002.8000003.1200003.280000
290.0387000.14228691.27000088.50000089.5800002.8000003.1200003.280000
300.0701000.14070091.27000088.50000089.5800002.8000003.1200003.280000

" ] }, "metadata": {} }, { "output_type": "stream", "name": "stderr", "text": [ "***** Running Evaluation *****\n", " Num examples = 50\n", " Batch size = 4\n" ] }, { "output_type": "stream", "name": "stdout", "text": [ "(50, 20)\n", "PRED: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "GT: num=4 - {'transfer learning', 'transformer', 'natural language processing', 'fine-tuning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "GT: num=3 - {'hypergraph attention network', 'recommender system', 'session-based recommendation system'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "GT: num=3 - {'descriptive text caption', 'generating audio sample', 'auto-regressive generative model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "GT: num=3 - {'human motion generation', 'motion diffusion model', 'diffusion model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "GT: num=3 - {'spiking neural network', 'self-attention mechanism', 'biological property'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'knowledge grounding', 'medical question understanding and answering', 'semantic', 'medical question answering system'}\n", "GT: num=4 - {'knowledge grounding', 'medical question understanding and answering system', 'semantic self-supervision', 'medical question answering system'}\n", "p=0.5, r=0.5, f1=0.5\n", "--------------------\n", "PRED: num=3 - {'synthesize diverse', 'multi-modal image and text representations', 'neural rendering'}\n", "GT: num=4 - {'natural language descriptions', 'synthesize diverse 3D objects', 'multi-modal image and text representations', 'neural rendering'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=2 - {'qualitative data', 'qualitative visualization'}\n", "GT: num=2 - {'qualitative data', 'qualitative visualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "GT: num=3 - {'graph neural network', 'topology design', 'feature selection and fusion strategy'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "GT: num=4 - {'transfer learning', 'multi-task learning', 'unsupervised learning', 'gaussian mixture model'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'cognitive multilayer network', 'cognition', 'c', 'mental lexicon'}\n", "GT: num=3 - {'cognitive multilayer network', 'cognitive network', 'mental lexicon'}\n", "p=0.5, r=0.6666666666666666, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "GT: num=3 - {'spatiotemporal representation learning', 'pretext task', 'pre-training'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "GT: num=4 - {'language-family adapter', 'multilingual model', 'machine translation', 'natural language processing'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "GT: num=3 - {'fine-tuning', 'prompt-free', 'sentence transformer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'', 'knowledge transferability', 'medical image domain', 'large-scale pre-trained vision language model'}\n", "GT: num=5 - {'knowledge transferability', 'medical prompt', 'medical image domain', 'domain transfer capability', 'large-scale pre-trained vision language model'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=4 - {'bayesian optimization', 'one-', 'stochastic expensive black box function', 'acquisition function'}\n", "GT: num=4 - {'one-shot hybrid kg', 'bayesian optimization', 'stochastic expensive black box function', 'acquisition function'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'convolutional neural network', 'handwritten text recognition', 'end-to-end', 'feature'}\n", "GT: num=4 - {'feature extraction', 'convolutional neural network', 'handwritten text recognition', 'end-to-end'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "GT: num=4 - {'speech processing system', 'multilingual supervision', 'robust speech processing', 'multitask supervision'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {\"fitts' law\", 'two-component mixture structure', 'expectation-conditional'}\n", "GT: num=3 - {'expectation-conditional-maximization', \"fitts' law\", 'two-component mixture structure'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=4 - {'maximal k', 'cohesive structure', 'mining maximal subgraph', 'bipartite graph'}\n", "GT: num=5 - {'mining maximal subgraph', 'bipartite graph', 'maximal k-biplex', 'cohesive structure', 'reverse search framework'}\n", "p=0.75, r=0.6, f1=0.6666666666666665\n", "--------------------\n", "PRED: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "GT: num=3 - {'operator-algebraic', 'quantum field theory', 'minkowski space'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "GT: num=3 - {'conditional diffusion transformer', 'data-driven approach', 'optimize neural networks'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "GT: num=3 - {'kg representation learning', 'knowledge graph', 'prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cross-modal representation', 'vision-language pre-trained model', 'dual encoder'}\n", "GT: num=5 - {'multi-view contrastive learning', 'contrastive learning', 'dual encoder', 'vision-language pre-trained model', 'cross-modal representation'}\n", "p=1.0, r=0.6, f1=0.7499999999999999\n", "--------------------\n", "PRED: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "GT: num=3 - {'free-text retrieval', 'knowledge base', 'question answering over knowledge base'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'pre-trained conversational model', 'implicit commonsense (cs) knowledge', 'dialogue'}\n", "GT: num=5 - {'implicit commonsense (cs) knowledge', 'pre-trained conversational model', 'dialogue agent', 'two-way learning', 'external knowledge'}\n", "p=0.6666666666666666, r=0.4, f1=0.5\n", "--------------------\n", "PRED: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "GT: num=5 - {'quantitative communication', 'information theory', 'communication complexity', 'quantum network', 'quantum computer'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "GT: num=3 - {'phoneme recognition', 'speech recognition', 'phonetic feature extraction'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "GT: num=2 - {'cross entropy loss', 'soft prompt learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "GT: num=2 - {'semantic segmentation', 'convolutional network architecture'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'neural machine translation', 'low frequency word prediction', 'token-level contrast', 'representation learning'}\n", "GT: num=4 - {'token-level contrastive learning', 'neural machine translation', 'low frequency word prediction', 'representation learning'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "GT: num=3 - {'reinforcement learning', 'markov decision process', 'sequential decoding'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "GT: num=2 - {'slimmable networks', 'self-supervised learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised segmentation network', 'segmentation of images', 'medical AI'}\n", "GT: num=4 - {'semi-supervised segmentation network', 'segmentation of images', 'contrastive learning', 'medical AI'}\n", "p=1.0, r=0.75, f1=0.8571428571428571\n", "--------------------\n", "PRED: num=2 - {'legal statute identification task', 'citation network'}\n", "GT: num=2 - {'legal statute identification task', 'citation network'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'multi-dialect automatic speech recognition', 'multi dialectal', 'deep neural network'}\n", "GT: num=4 - {'acoustic model', 'multi-dialect automatic speech recognition', 'multi-dialectal corpus', 'deep neural network'}\n", "p=0.6666666666666666, r=0.5, f1=0.5714285714285715\n", "--------------------\n", "PRED: num=4 - {'language-vision', 'language', 'deep learning', 'open-source deep learning library'}\n", "GT: num=4 - {'language-vision', 'language-vision tasks', 'deep learning', 'open-source deep learning library'}\n", "p=0.75, r=0.75, f1=0.75\n", "--------------------\n", "PRED: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "GT: num=3 - {'task-agnostic', 'semi-parametric language model', 'zero-shot generalization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "GT: num=3 - {'cooperative environmental learning', 'multi-robot system', 'distributed learning'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "GT: num=3 - {'decision tree', 'real-time and human-interpretable decision-making', 'time-incremental learning'}\n", "p=0.6666666666666666, r=0.6666666666666666, f1=0.6666666666666666\n", "--------------------\n", "PRED: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "GT: num=3 - {'Non-referential face image quality assessment method', 'quality assessment method', 'face recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "GT: num=2 - {'knowledge-intensive language tasks', 'benchmark'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "GT: num=3 - {'semi-supervised learning', 'neural processes', 'image classification task'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "GT: num=2 - {'sparse attention and monotonic attention', 'automatic speech recognition'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "GT: num=4 - {'information removal', 'privacy', 'deep learning', 'bias'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'data budgeting problem', 'data'}\n", "GT: num=2 - {'data budgeting problem', 'data'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "GT: num=2 - {'gender classification algorithms', 'benchmark database'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=5 - {'sparse recovery', 'convex optimization', 'deep learning', 'neural networks', 'relu'}\n", "GT: num=5 - {'sparse recovery', 'convex optimization', 'relu networks', 'deep learning', 'neural networks'}\n", "p=0.8, r=0.8, f1=0.8000000000000002\n", "--------------------\n", "PRED: num=2 - {'genetic variant', 'boolean relations'}\n", "GT: num=2 - {'genetic variant', 'boolean relations'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n", "PRED: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "GT: num=2 - {'text-to-image diffusion model', 'subject recontextualization'}\n", "p=1.0, r=1.0, f1=1.0\n", "--------------------\n" ] }, { "output_type": "stream", "name": "stderr", "text": [ "Saving model checkpoint to KeyBartAdapter/checkpoint-390\n", "Configuration saved in KeyBartAdapter/checkpoint-390/config.json\n", "Model weights saved in KeyBartAdapter/checkpoint-390/pytorch_model.bin\n", "tokenizer config file saved in KeyBartAdapter/checkpoint-390/tokenizer_config.json\n", "Special tokens file saved in KeyBartAdapter/checkpoint-390/special_tokens_map.json\n", "Deleting older checkpoint [KeyBartAdapter/checkpoint-364] due to args.save_total_limit\n", "\n", "\n", "Training completed. Do not forget to share your model on huggingface.co/models =)\n", "\n", "\n", "Loading best model from KeyBartAdapter/checkpoint-208 (score: 90.32).\n" ] }, { "output_type": "execute_result", "data": { "text/plain": [ "TrainOutput(global_step=390, training_loss=0.24197826716953363, metrics={'train_runtime': 2544.9523, 'train_samples_per_second': 0.589, 'train_steps_per_second': 0.153, 'total_flos': 961529921028096.0, 'train_loss': 0.24197826716953363, 'epoch': 30.0})" ] }, "metadata": {}, "execution_count": 3 } ] }, { "cell_type": "code", "source": [ "abstract = '''Large text-to-image models achieved a remarkable leap in the evolution of AI, enabling high-quality and diverse synthesis of images from a given text prompt. However, these models lack the ability to mimic the appearance of subjects in a given reference set and synthesize novel renditions of them in different contexts. In this work, we present a new approach for \"personalization\" of text-to-image diffusion models (specializing them to users' needs). Given as input just a few images of a subject, we fine-tune a pretrained text-to-image model (Imagen, although our method is not limited to a specific model) such that it learns to bind a unique identifier with that specific subject. Once the subject is embedded in the output domain of the model, the unique identifier can then be used to synthesize fully-novel photorealistic images of the subject contextualized in different scenes. By leveraging the semantic prior embedded in the model with a new autogenous class-specific prior preservation loss, our technique enables synthesizing the subject in diverse scenes, poses, views, and lighting conditions that do not appear in the reference images. We apply our technique to several previously-unassailable tasks, including subject recontextualization, text-guided view synthesis, appearance modification, and artistic rendering (all while preserving the subject's key features). Project page: https://dreambooth.github.io/'''" ], "metadata": { "id": "jfKXREnThJ_T" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "from transformers import Text2TextGenerationPipeline\n", "pipe = Text2TextGenerationPipeline(model=model,tokenizer=tokenizer,device= 0)\n", "\n", "\n", "pipe(abstract)" ], "metadata": { "id": "06Fard5UfHqQ", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "998a5db2-2026-429b-fffb-71443608ed37" }, "execution_count": null, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "[{'generated_text': 'text-to-image diffusion model;subject recontextualization'}]" ] }, "metadata": {}, "execution_count": 5 } ] }, { "cell_type": "code", "source": [ "# # 6. push to hub\n", "# commit_msg = f'{model_name}_{num_epoch}'\n", "# tokenizer.push_to_hub(commit_message=commit_msg, repo_id=model_name )\n", "# model.push_to_hub(commit_message=commit_msg, repo_id=model_name)" ], "metadata": { "id": "nYXWw7brJMhQ" }, "execution_count": null, "outputs": [] } ] }