Upload 2667 files

opencompass-my-api/.codespellrc

@@ -0,0 +1,5 @@
+[codespell]
+skip = *.ipynb
+count =
+quiet-level = 3
+ignore-words-list = nd, ans, ques, rouge, softwares, wit

opencompass-my-api/.gitignore

@@ -0,0 +1,129 @@
+
+output_*/
+outputs/
+icl_inference_output/
+.vscode/
+tmp/
+configs/eval_subjective_alignbench_test.py
+configs/openai_key.py
+configs/secrets.py
+configs/datasets/log.json
+configs/eval_debug*.py
+configs/viz_*.py
+data
+work_dirs
+models/*
+configs/internal/
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+*.ipynb
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+.idea
+
+# Auto generate documentation
+docs/en/_build/
+docs/zh_cn/_build/
+
+# .zip
+*.zip
+
+# sft config ignore list
+configs/sft_cfg/*B_*
+configs/sft_cfg/1B/*
+configs/sft_cfg/7B/*
+configs/sft_cfg/20B/*
+configs/sft_cfg/60B/*
+configs/sft_cfg/100B/*
+
+configs/cky/
+# in case llama clone in the opencompass
+llama/
+
+# in case ilagent clone in the opencompass
+ilagent/
+
+# ignore the config file for criticbench evaluation
+configs/sft_cfg/criticbench_eval/*
+
+# path of turbomind's model after runing `lmdeploy.serve.turbomind.deploy`
+turbomind/
+
+# cibench output
+*.db
+*.pth
+*.pt
+*.onnx
+*.gz
+*.gz.*
+*.png
+*.txt
+*.jpg
+*.json
+*.csv
+*.npy
+*.c
+
+# aliyun
+core.*

opencompass-my-api/.owners.yml

@@ -0,0 +1,14 @@
+assign:
+  issues: enabled
+  pull_requests: disabled
+  strategy:
+    # random
+    daily-shift-based
+  scedule:
+    '*/1 * * * *'
+  assignees:
+    - Leymore
+    - bittersweet1999
+    - yingfhu
+    - kennymckormick
+    - tonysy

opencompass-my-api/.pre-commit-config-zh-cn.yaml

@@ -0,0 +1,96 @@
+exclude: |
+    (?x)^(
+      tests/data/|
+      opencompass/models/internal/|
+      opencompass/utils/internal/|
+      opencompass/openicl/icl_evaluator/hf_metrics/|
+      opencompass/datasets/lawbench/utils|
+      opencompass/datasets/lawbench/evaluation_functions/|
+      opencompass/datasets/medbench/|
+      opencompass/datasets/teval/|
+      opencompass/datasets/NPHardEval/|
+      docs/zh_cn/advanced_guides/compassbench_intro.md
+    )
+repos:
+  - repo: https://gitee.com/openmmlab/mirrors-flake8
+    rev: 5.0.4
+    hooks:
+      - id: flake8
+        exclude: configs/
+  - repo: https://gitee.com/openmmlab/mirrors-isort
+    rev: 5.11.5
+    hooks:
+      - id: isort
+        exclude: configs/
+  - repo: https://gitee.com/openmmlab/mirrors-yapf
+    rev: v0.32.0
+    hooks:
+      - id: yapf
+        exclude: configs/
+  - repo: https://gitee.com/openmmlab/mirrors-codespell
+    rev: v2.2.1
+    hooks:
+      - id: codespell
+        exclude: |
+            (?x)^(
+                .*\.jsonl|
+                configs/
+            )
+  - repo: https://gitee.com/openmmlab/mirrors-pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: trailing-whitespace
+        exclude: |
+            (?x)^(
+              dicts/|
+              projects/.*?/dicts/|
+              configs/
+            )
+      - id: check-yaml
+      - id: end-of-file-fixer
+        exclude: |
+            (?x)^(
+              dicts/|
+              projects/.*?/dicts/|
+              configs/
+            )
+      - id: requirements-txt-fixer
+      - id: double-quote-string-fixer
+        exclude: configs/
+      - id: check-merge-conflict
+      - id: fix-encoding-pragma
+        args: ["--remove"]
+      - id: mixed-line-ending
+        args: ["--fix=lf"]
+      - id: mixed-line-ending
+        args: ["--fix=lf"]
+  - repo: https://gitee.com/openmmlab/mirrors-mdformat
+    rev: 0.7.9
+    hooks:
+      - id: mdformat
+        args: ["--number", "--table-width", "200"]
+        additional_dependencies:
+          - mdformat-openmmlab
+          - mdformat_frontmatter
+          - linkify-it-py
+        exclude: configs/
+  - repo: https://gitee.com/openmmlab/mirrors-docformatter
+    rev: v1.3.1
+    hooks:
+      - id: docformatter
+        args: ["--in-place", "--wrap-descriptions", "79"]
+  - repo: local
+    hooks:
+    -   id: update-dataset-suffix
+        name: dataset suffix updater
+        entry: ./tools/update_dataset_suffix.py
+        language: script
+        pass_filenames: true
+        require_serial: true
+        files: ^configs/datasets
+  # - repo: https://github.com/open-mmlab/pre-commit-hooks
+  #   rev: v0.2.0  # Use the ref you want to point at
+  #   hooks:
+  #     - id: check-algo-readme
+      # - id: check-copyright
+      #   args: ["mmocr", "tests", "tools"]  # these directories will be checked

opencompass-my-api/.pre-commit-config.yaml

@@ -0,0 +1,96 @@
+exclude: |
+    (?x)^(
+      tests/data/|
+      opencompass/models/internal/|
+      opencompass/utils/internal/|
+      opencompass/openicl/icl_evaluator/hf_metrics/|
+      opencompass/datasets/lawbench/utils|
+      opencompass/datasets/lawbench/evaluation_functions/|
+      opencompass/datasets/medbench/|
+      opencompass/datasets/teval/|
+      opencompass/datasets/NPHardEval/|
+      docs/zh_cn/advanced_guides/compassbench_intro.md
+    )
+repos:
+  - repo: https://github.com/PyCQA/flake8
+    rev: 5.0.4
+    hooks:
+      - id: flake8
+        exclude: configs/
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.11.5
+    hooks:
+      - id: isort
+        exclude: configs/
+  - repo: https://github.com/pre-commit/mirrors-yapf
+    rev: v0.32.0
+    hooks:
+      - id: yapf
+        exclude: configs/
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.1
+    hooks:
+      - id: codespell
+        exclude: |
+            (?x)^(
+                .*\.jsonl|
+                configs/
+            )
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: trailing-whitespace
+        exclude: |
+            (?x)^(
+              dicts/|
+              projects/.*?/dicts/|
+              configs/
+            )
+      - id: check-yaml
+      - id: end-of-file-fixer
+        exclude: |
+            (?x)^(
+              dicts/|
+              projects/.*?/dicts/|
+              configs/
+            )
+      - id: requirements-txt-fixer
+      - id: double-quote-string-fixer
+        exclude: configs/
+      - id: check-merge-conflict
+      - id: fix-encoding-pragma
+        args: ["--remove"]
+      - id: mixed-line-ending
+        args: ["--fix=lf"]
+      - id: mixed-line-ending
+        args: ["--fix=lf"]
+  - repo: https://github.com/executablebooks/mdformat
+    rev: 0.7.9
+    hooks:
+      - id: mdformat
+        args: ["--number", "--table-width", "200"]
+        additional_dependencies:
+          - mdformat-openmmlab
+          - mdformat_frontmatter
+          - linkify-it-py
+        exclude: configs/
+  - repo: https://github.com/myint/docformatter
+    rev: v1.3.1
+    hooks:
+      - id: docformatter
+        args: ["--in-place", "--wrap-descriptions", "79"]
+  - repo: local
+    hooks:
+    -   id: update-dataset-suffix
+        name: dataset suffix updater
+        entry: ./tools/update_dataset_suffix.py
+        language: script
+        pass_filenames: true
+        require_serial: true
+        files: ^configs/datasets
+  # - repo: https://github.com/open-mmlab/pre-commit-hooks
+  #   rev: v0.2.0  # Use the ref you want to point at
+  #   hooks:
+  #     - id: check-algo-readme
+      # - id: check-copyright
+      #   args: ["mmocr", "tests", "tools"]  # these directories will be checked

opencompass-my-api/.readthedocs.yml

@@ -0,0 +1,14 @@
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.8"
+
+formats:
+    - epub
+
+python:
+  install:
+    - requirements: requirements/docs.txt

opencompass-my-api/LICENSE

@@ -0,0 +1,203 @@
+Copyright 2020 OpenCompass Authors. All rights reserved.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2020 OpenCompass Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

opencompass-my-api/README.md

@@ -0,0 +1,520 @@
+<div align="center">
+  <img src="docs/en/_static/image/logo.svg" width="500px"/>
+  <br />
+  <br />
+
+[![docs](https://readthedocs.org/projects/opencompass/badge)](https://opencompass.readthedocs.io/en)
+[![license](https://img.shields.io/github/license/InternLM/opencompass.svg)](https://github.com/open-compass/opencompass/blob/main/LICENSE)
+
+<!-- [![PyPI](https://badge.fury.io/py/opencompass.svg)](https://pypi.org/project/opencompass/) -->
+
+[🌐Website](https://opencompass.org.cn/) |
+[📘Documentation](https://opencompass.readthedocs.io/en/latest/) |
+[🛠️Installation](https://opencompass.readthedocs.io/en/latest/get_started/installation.html) |
+[🤔Reporting Issues](https://github.com/open-compass/opencompass/issues/new/choose)
+
+English | [简体中文](README_zh-CN.md)
+
+</div>
+
+<p align="center">
+    👋 join us on <a href="https://discord.gg/KKwfEbFj7U" target="_blank">Discord</a> and <a href="https://r.vansin.top/?r=opencompass" target="_blank">WeChat</a>
+</p>
+
+## 📣 OpenCompass 2023 LLM Annual Leaderboard
+
+We are honored to have witnessed the tremendous progress of artificial general intelligence together with the community in the past year, and we are also very pleased that **OpenCompass** can help numerous developers and users.
+
+We announce the launch of the **OpenCompass 2023 LLM Annual Leaderboard** plan. We expect to release the annual leaderboard of the LLMs in January 2024, systematically evaluating the performance of LLMs in various capabilities such as language, knowledge, reasoning, creation, long-text, and agents.
+
+At that time, we will release rankings for both open-source models and commercial API models, aiming to provide a comprehensive, objective, and neutral reference for the industry and research community.
+
+We sincerely invite various large models to join the OpenCompass to showcase their performance advantages in different fields. At the same time, we also welcome researchers and developers to provide valuable suggestions and contributions to jointly promote the development of the LLMs. If you have any questions or needs, please feel free to [contact us](mailto:[email protected]). In addition, relevant evaluation contents, performance statistics, and evaluation methods will be open-source along with the leaderboard release.
+
+We have provided the more details of the CompassBench 2023 in [Doc](docs/zh_cn/advanced_guides/compassbench_intro.md).
+
+Let's look forward to the release of the OpenCompass 2023 LLM Annual Leaderboard!
+
+## 🧭	Welcome
+
+to **OpenCompass**!
+
+Just like a compass guides us on our journey, OpenCompass will guide you through the complex landscape of evaluating large language models. With its powerful algorithms and intuitive interface, OpenCompass makes it easy to assess the quality and effectiveness of your NLP models.
+
+🚩🚩🚩 Explore opportunities at OpenCompass! We're currently **hiring full-time researchers/engineers and interns**. If you're passionate about LLM and OpenCompass, don't hesitate to reach out to us via [email](mailto:[email protected]). We'd love to hear from you!
+
+🔥🔥🔥 We are delighted to announce that **the OpenCompass has been recommended by the Meta AI**, click [Get Started](https://ai.meta.com/llama/get-started/#validation) of Llama for more information.
+
+> **Attention**<br />
+> We launch the OpenCompass Collaboration project, welcome to support diverse evaluation benchmarks into OpenCompass!
+> Clike [Issue](https://github.com/open-compass/opencompass/issues/248) for more information.
+> Let's work together to build a more powerful OpenCompass toolkit!
+
+## 🚀 What's New <a><img width="35" height="20" src="https://user-images.githubusercontent.com/12782558/212848161-5e783dd6-11e8-4fe0-bbba-39ffb77730be.png"></a>
+
+- **\[2024.01.17\]** We supported the evaluation of [InternLM2](https://github.com/open-compass/opencompass/blob/main/configs/eval_internlm2_keyset.py) and [InternLM2-Chat](https://github.com/open-compass/opencompass/blob/main/configs/eval_internlm2_chat_keyset.py), InternLM2 showed extremely strong performance in these tests, welcome to try! 🔥🔥🔥.
+- **\[2024.01.17\]** We supported the needle in a haystack test with multiple needles, more information can be found [here](https://opencompass.readthedocs.io/en/latest/advanced_guides/needleinahaystack_eval.html#id8) 🔥🔥🔥.
+- **\[2023.12.28\]** We have enabled seamless evaluation of all models developed using [LLaMA2-Accessory](https://github.com/Alpha-VLLM/LLaMA2-Accessory), a powerful toolkit for comprehensive LLM development. 🔥🔥🔥.
+- **\[2023.12.22\]** We have released [T-Eval](https://github.com/open-compass/T-Eval), a step-by-step evaluation benchmark to gauge your LLMs on tool utilization. Welcome to our [Leaderboard](https://open-compass.github.io/T-Eval/leaderboard.html) for more details! 🔥🔥🔥.
+- **\[2023.12.10\]** We have released [VLMEvalKit](https://github.com/open-compass/VLMEvalKit), a toolkit for evaluating vision-language models (VLMs), currently support 20+ VLMs and 7 multi-modal benchmarks (including MMBench series).
+- **\[2023.12.10\]** We have supported Mistral AI's MoE LLM: **Mixtral-8x7B-32K**. Welcome to [MixtralKit](https://github.com/open-compass/MixtralKit) for more details about inference and evaluation.
+
+> [More](docs/en/notes/news.md)
+
+## ✨ Introduction
+
+![image](https://github.com/open-compass/opencompass/assets/22607038/f45fe125-4aed-4f8c-8fe8-df4efb41a8ea)
+
+OpenCompass is a one-stop platform for large model evaluation, aiming to provide a fair, open, and reproducible benchmark for large model evaluation. Its main features include:
+
+- **Comprehensive support for models and datasets**: Pre-support for 20+ HuggingFace and API models, a model evaluation scheme of 70+ datasets with about 400,000 questions, comprehensively evaluating the capabilities of the models in five dimensions.
+
+- **Efficient distributed evaluation**: One line command to implement task division and distributed evaluation, completing the full evaluation of billion-scale models in just a few hours.
+
+- **Diversified evaluation paradigms**: Support for zero-shot, few-shot, and chain-of-thought evaluations, combined with standard or dialogue-type prompt templates, to easily stimulate the maximum performance of various models.
+
+- **Modular design with high extensibility**: Want to add new models or datasets, customize an advanced task division strategy, or even support a new cluster management system? Everything about OpenCompass can be easily expanded!
+
+- **Experiment management and reporting mechanism**: Use config files to fully record each experiment, and support real-time reporting of results.
+
+## 📊 Leaderboard
+
+We provide [OpenCompass Leaderboard](https://opencompass.org.cn/rank) for the community to rank all public models and API models. If you would like to join the evaluation, please provide the model repository URL or a standard API interface to the email address `[email protected]`.
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 🛠️ Installation
+
+Below are the steps for quick installation and datasets preparation.
+
+### 💻 Environment Setup
+
+#### Open-source Models with GPU
+
+```bash
+conda create --name opencompass python=3.10 pytorch torchvision pytorch-cuda -c nvidia -c pytorch -y
+conda activate opencompass
+git clone https://github.com/open-compass/opencompass opencompass
+cd opencompass
+pip install -e .
+```
+
+#### API Models with CPU-only
+
+```bash
+conda create -n opencompass python=3.10 pytorch torchvision torchaudio cpuonly -c pytorch -y
+conda activate opencompass
+git clone https://github.com/open-compass/opencompass opencompass
+cd opencompass
+pip install -e .
+# also please install requiresments packages via `pip install -r requirements/api.txt` for API models if needed.
+```
+
+### 📂 Data Preparation
+
+```bash
+# Download dataset to data/ folder
+wget https://github.com/open-compass/opencompass/releases/download/0.2.2.rc1/OpenCompassData-core-20240207.zip
+unzip OpenCompassData-core-20240207.zip
+```
+
+Some third-party features, like Humaneval and Llama, may require additional steps to work properly, for detailed steps please refer to the [Installation Guide](https://opencompass.readthedocs.io/en/latest/get_started/installation.html).
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 🏗️ ️Evaluation
+
+After ensuring that OpenCompass is installed correctly according to the above steps and the datasets are prepared, you can evaluate the performance of the LLaMA-7b model on the MMLU and C-Eval datasets using the following command:
+
+```bash
+python run.py --models hf_llama_7b --datasets mmlu_ppl ceval_ppl
+```
+
+OpenCompass has predefined configurations for many models and datasets. You can list all available model and dataset configurations using the [tools](./docs/en/tools.md#list-configs).
+
+```bash
+# List all configurations
+python tools/list_configs.py
+# List all configurations related to llama and mmlu
+python tools/list_configs.py llama mmlu
+```
+
+You can also evaluate other HuggingFace models via command line. Taking LLaMA-7b as an example:
+
+```bash
+python run.py --datasets ceval_ppl mmlu_ppl \
+--hf-path huggyllama/llama-7b \  # HuggingFace model path
+--model-kwargs device_map='auto' \  # Arguments for model construction
+--tokenizer-kwargs padding_side='left' truncation='left' use_fast=False \  # Arguments for tokenizer construction
+--max-out-len 100 \  # Maximum number of tokens generated
+--max-seq-len 2048 \  # Maximum sequence length the model can accept
+--batch-size 8 \  # Batch size
+--no-batch-padding \  # Don't enable batch padding, infer through for loop to avoid performance loss
+--num-gpus 1  # Number of minimum required GPUs
+```
+
+> **Note**<br />
+> To run the command above, you will need to remove the comments starting from `# ` first.
+
+Through the command line or configuration files, OpenCompass also supports evaluating APIs or custom models, as well as more diversified evaluation strategies. Please read the [Quick Start](https://opencompass.readthedocs.io/en/latest/get_started/quick_start.html) to learn how to run an evaluation task.
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 📖 Dataset Support
+
+<table align="center">
+  <tbody>
+    <tr align="center" valign="bottom">
+      <td>
+        <b>Language</b>
+      </td>
+      <td>
+        <b>Knowledge</b>
+      </td>
+      <td>
+        <b>Reasoning</b>
+      </td>
+      <td>
+        <b>Examination</b>
+      </td>
+    </tr>
+    <tr valign="top">
+      <td>
+<details open>
+<summary><b>Word Definition</b></summary>
+
+- WiC
+- SummEdits
+
+</details>
+
+<details open>
+<summary><b>Idiom Learning</b></summary>
+
+- CHID
+
+</details>
+
+<details open>
+<summary><b>Semantic Similarity</b></summary>
+
+- AFQMC
+- BUSTM
+
+</details>
+
+<details open>
+<summary><b>Coreference Resolution</b></summary>
+
+- CLUEWSC
+- WSC
+- WinoGrande
+
+</details>
+
+<details open>
+<summary><b>Translation</b></summary>
+
+- Flores
+- IWSLT2017
+
+</details>
+
+<details open>
+<summary><b>Multi-language Question Answering</b></summary>
+
+- TyDi-QA
+- XCOPA
+
+</details>
+
+<details open>
+<summary><b>Multi-language Summary</b></summary>
+
+- XLSum
+
+</details>
+      </td>
+      <td>
+<details open>
+<summary><b>Knowledge Question Answering</b></summary>
+
+- BoolQ
+- CommonSenseQA
+- NaturalQuestions
+- TriviaQA
+
+</details>
+      </td>
+      <td>
+<details open>
+<summary><b>Textual Entailment</b></summary>
+
+- CMNLI
+- OCNLI
+- OCNLI_FC
+- AX-b
+- AX-g
+- CB
+- RTE
+- ANLI
+
+</details>
+
+<details open>
+<summary><b>Commonsense Reasoning</b></summary>
+
+- StoryCloze
+- COPA
+- ReCoRD
+- HellaSwag
+- PIQA
+- SIQA
+
+</details>
+
+<details open>
+<summary><b>Mathematical Reasoning</b></summary>
+
+- MATH
+- GSM8K
+
+</details>
+
+<details open>
+<summary><b>Theorem Application</b></summary>
+
+- TheoremQA
+- StrategyQA
+- SciBench
+
+</details>
+
+<details open>
+<summary><b>Comprehensive Reasoning</b></summary>
+
+- BBH
+
+</details>
+      </td>
+      <td>
+<details open>
+<summary><b>Junior High, High School, University, Professional Examinations</b></summary>
+
+- C-Eval
+- AGIEval
+- MMLU
+- GAOKAO-Bench
+- CMMLU
+- ARC
+- Xiezhi
+
+</details>
+
+<details open>
+<summary><b>Medical Examinations</b></summary>
+
+- CMB
+
+</details>
+      </td>
+    </tr>
+</td>
+    </tr>
+  </tbody>
+  <tbody>
+    <tr align="center" valign="bottom">
+      <td>
+        <b>Understanding</b>
+      </td>
+      <td>
+        <b>Long Context</b>
+      </td>
+      <td>
+        <b>Safety</b>
+      </td>
+      <td>
+        <b>Code</b>
+      </td>
+    </tr>
+    <tr valign="top">
+      <td>
+<details open>
+<summary><b>Reading Comprehension</b></summary>
+
+- C3
+- CMRC
+- DRCD
+- MultiRC
+- RACE
+- DROP
+- OpenBookQA
+- SQuAD2.0
+
+</details>
+
+<details open>
+<summary><b>Content Summary</b></summary>
+
+- CSL
+- LCSTS
+- XSum
+- SummScreen
+
+</details>
+
+<details open>
+<summary><b>Content Analysis</b></summary>
+
+- EPRSTMT
+- LAMBADA
+- TNEWS
+
+</details>
+      </td>
+      <td>
+<details open>
+<summary><b>Long Context Understanding</b></summary>
+
+- LEval
+- LongBench
+- GovReports
+- NarrativeQA
+- Qasper
+
+</details>
+      </td>
+      <td>
+<details open>
+<summary><b>Safety</b></summary>
+
+- CivilComments
+- CrowsPairs
+- CValues
+- JigsawMultilingual
+- TruthfulQA
+
+</details>
+<details open>
+<summary><b>Robustness</b></summary>
+
+- AdvGLUE
+
+</details>
+      </td>
+      <td>
+<details open>
+<summary><b>Code</b></summary>
+
+- HumanEval
+- HumanEvalX
+- MBPP
+- APPs
+- DS1000
+
+</details>
+      </td>
+    </tr>
+</td>
+    </tr>
+  </tbody>
+</table>
+
+## OpenCompass Ecosystem
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 📖 Model Support
+
+<table align="center">
+  <tbody>
+    <tr align="center" valign="bottom">
+      <td>
+        <b>Open-source Models</b>
+      </td>
+      <td>
+        <b>API Models</b>
+      </td>
+      <!-- <td>
+        <b>Custom Models</b>
+      </td> -->
+    </tr>
+    <tr valign="top">
+      <td>
+
+- [InternLM](https://github.com/InternLM/InternLM)
+- [LLaMA](https://github.com/facebookresearch/llama)
+- [Vicuna](https://github.com/lm-sys/FastChat)
+- [Alpaca](https://github.com/tatsu-lab/stanford_alpaca)
+- [Baichuan](https://github.com/baichuan-inc)
+- [WizardLM](https://github.com/nlpxucan/WizardLM)
+- [ChatGLM2](https://github.com/THUDM/ChatGLM2-6B)
+- [ChatGLM3](https://github.com/THUDM/ChatGLM3-6B)
+- [TigerBot](https://github.com/TigerResearch/TigerBot)
+- [Qwen](https://github.com/QwenLM/Qwen)
+- [BlueLM](https://github.com/vivo-ai-lab/BlueLM)
+- ...
+
+</td>
+<td>
+
+- OpenAI
+- Claude
+- ZhipuAI(ChatGLM)
+- Baichuan
+- ByteDance(YunQue)
+- Huawei(PanGu)
+- 360
+- Baidu(ERNIEBot)
+- MiniMax(ABAB-Chat)
+- SenseTime(nova)
+- Xunfei(Spark)
+- ……
+
+</td>
+
+</tr>
+  </tbody>
+</table>
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 🔜 Roadmap
+
+- [ ] Subjective Evaluation
+  - [ ] Release CompassAreana
+  - [ ] Subjective evaluation dataset.
+- [x] Long-context
+  - [ ] Long-context evaluation with extensive datasets.
+  - [ ] Long-context leaderboard.
+- [ ] Coding
+  - [ ] Coding evaluation leaderboard.
+  - [x] Non-python language evaluation service.
+- [ ] Agent
+  - [ ] Support various agenet framework.
+  - [ ] Evaluation of tool use of the LLMs.
+- [x] Robustness
+  - [x] Support various attack method
+
+## 👷‍♂️ Contributing
+
+We appreciate all contributions to improving OpenCompass. Please refer to the [contributing guideline](https://opencompass.readthedocs.io/en/latest/notes/contribution_guide.html) for the best practice.
+
+## 🤝 Acknowledgements
+
+Some code in this project is cited and modified from [OpenICL](https://github.com/Shark-NLP/OpenICL).
+
+Some datasets and prompt implementations are modified from [chain-of-thought-hub](https://github.com/FranxYao/chain-of-thought-hub) and [instruct-eval](https://github.com/declare-lab/instruct-eval).
+
+## 🖊️ Citation
+
+```bibtex
+@misc{2023opencompass,
+    title={OpenCompass: A Universal Evaluation Platform for Foundation Models},
+    author={OpenCompass Contributors},
+    howpublished = {\url{https://github.com/open-compass/opencompass}},
+    year={2023}
+}
+```
+
+<p align="right"><a href="#top">🔝Back to top</a></p>

opencompass-my-api/README_zh-CN.md

@@ -0,0 +1,522 @@
+<div align="center">
+  <img src="docs/zh_cn/_static/image/logo.svg" width="500px"/>
+  <br />
+  <br />
+
+[![docs](https://readthedocs.org/projects/opencompass/badge)](https://opencompass.readthedocs.io/zh_CN)
+[![license](https://img.shields.io/github/license/InternLM/opencompass.svg)](https://github.com/open-compass/opencompass/blob/main/LICENSE)
+
+<!-- [![PyPI](https://badge.fury.io/py/opencompass.svg)](https://pypi.org/project/opencompass/) -->
+
+[🌐Website](https://opencompass.org.cn/) |
+[📘Documentation](https://opencompass.readthedocs.io/zh_CN/latest/index.html) |
+[🛠️Installation](https://opencompass.readthedocs.io/zh_CN/latest/get_started/installation.html) |
+[🤔Reporting Issues](https://github.com/open-compass/opencompass/issues/new/choose)
+
+[English](/README.md) | 简体中文
+
+</div>
+
+<p align="center">
+    👋 加入我们的 <a href="https://discord.gg/KKwfEbFj7U" target="_blank">Discord</a> 和 <a href="https://r.vansin.top/?r=opencompass" target="_blank">微信社区</a>
+</p>
+
+## 📣 2023 年度榜单计划
+
+我们有幸与社区共同见证了通用人工智能在过去一年里的巨大进展，也非常高兴OpenCompass能够帮助广大大模型开发者和使用者。
+
+我们宣布将启动**OpenCompass 2023年度大模型榜单**发布计划。我们预计将于2024年1月发布大模型年度榜单，系统性评估大模型在语言、知识、推理、创作、长文本和智能体等多个能力维度的表现。
+
+届时，我们将发布开源模型和商业API模型能力榜单，以期为业界提供一份**全面、客观、中立**的参考。
+
+我们诚挚邀请各类大模型接入OpenCompass评测体系，以展示其在各个领域的性能优势。同时，也欢迎广大研究者、开发者向我们提供宝贵的意见和建议，共同推动大模型领域的发展。如有任何问题或需求，请随时[联系我们](mailto:[email protected])。此外，相关评测内容，性能数据，评测方法也将随榜单发布一并开源。
+
+我们提供了本次评测所使用的部分题目示例，详情请见[CompassBench 2023](docs/zh_cn/advanced_guides/compassbench_intro.md).
+
+<p>让我们共同期待OpenCompass 2023年度大模型榜单的发布，期待各大模型在榜单上的精彩表现！</p>
+
+## 🧭	欢迎
+
+来到**OpenCompass**！
+
+就像指南针在我们的旅程中为我们导航一样，我们希望OpenCompass能够帮助你穿越评估大型语言模型的重重迷雾。OpenCompass提供丰富的算法和功能支持，期待OpenCompass能够帮助社区更便捷地对NLP模型的性能进行公平全面的评估。
+
+🚩🚩🚩 欢迎加入 OpenCompass！我们目前**招聘全职研究人员/工程师和实习生**。如果您对 LLM 和 OpenCompass 充满热情，请随时通过[电子邮件](mailto:[email protected])与我们联系。我们非常期待与您交流！
+
+🔥🔥🔥 祝贺 **OpenCompass 作为大模型标准测试工具被Meta AI官方推荐**, 点击 Llama 的 [入门文档](https://ai.meta.com/llama/get-started/#validation) 获取更多信息.
+
+> **注意**<br />
+> 我们正式启动 OpenCompass 共建计划，诚邀社区用户为 OpenCompass 提供更具代表性和可信度的客观评测数据集!
+> 点击 [Issue](https://github.com/open-compass/opencompass/issues/248) 获取更多数据集.
+> 让我们携手共进，打造功能强大易用的大模型评测平台！
+
+## 🚀 最新进展 <a><img width="35" height="20" src="https://user-images.githubusercontent.com/12782558/212848161-5e783dd6-11e8-4fe0-bbba-39ffb77730be.png"></a>
+
+- **\[2024.01.17\]** 我们支持了 [InternLM2](https://github.com/open-compass/opencompass/blob/main/configs/eval_internlm2_chat_keyset.py) 和 [InternLM2-Chat](https://github.com/open-compass/opencompass/blob/main/configs/eval_internlm2_chat_keyset.py) 的相关评测，InternLM2 在这些测试中表现出非常强劲的性能，欢迎试用！🔥🔥🔥.
+- **\[2024.01.17\]** 我们支持了多根针版本的大海捞针测试，更多信息见[这里](https://opencompass.readthedocs.io/zh-cn/latest/advanced_guides/needleinahaystack_eval.html#id8)🔥🔥🔥.
+- **\[2023.12.28\]** 我们支持了对使用[LLaMA2-Accessory](https://github.com/Alpha-VLLM/LLaMA2-Accessory)（一款强大的LLM开发工具箱）开发的所有模型的无缝评估! 🔥🔥🔥.
+- **\[2023.12.22\]** 我们开源了[T-Eval](https://github.com/open-compass/T-Eval)用于评测大语言模型工具调用能力。欢迎访问T-Eval的官方[Leaderboard](https://open-compass.github.io/T-Eval/leaderboard.html)获取更多信息! 🔥🔥🔥.
+- **\[2023.12.10\]** 我们开源了多模评测框架 [VLMEvalKit](https://github.com/open-compass/VLMEvalKit)，目前已支持 20+ 个多模态大模型与包括 MMBench 系列在内的 7 个多模态评测集.
+- **\[2023.12.10\]** 我们已经支持了Mistral AI的MoE模型 **Mixtral-8x7B-32K**。欢迎查阅[MixtralKit](https://github.com/open-compass/MixtralKit)以获取更多关于推理和评测的详细信息.
+
+> [更多](docs/zh_cn/notes/news.md)
+
+## ✨ 介绍
+
+![image](https://github.com/open-compass/opencompass/assets/22607038/30bcb2e2-3969-4ac5-9f29-ad3f4abb4f3b)
+
+OpenCompass 是面向大模型评测的一站式平台。其主要特点如下：
+
+- **开源可复现**：提供公平、公开、可复现的大模型评测方案
+
+- **全面的能力维度**：五大维度设计，提供 70+ 个数据集约 40 万题的的模型评测方案，全面评估模型能力
+
+- **丰富的模型支持**：已支持 20+ HuggingFace 及 API 模型
+
+- **分布式高效评测**：一行命令实现任务分割和分布式评测，数小时即可完成千亿模型全量评测
+
+- **多样化评测范式**：支持零样本、小样本及思维链评测，结合标准型或对话型提示词模板，轻松激发各种模型最大性能
+
+- **灵活化拓展**：想增加新模型或数据集？想要自定义更高级的任务分割策略，甚至接入新的集群管理系统？OpenCompass 的一切均可轻松扩展！
+
+## 📊 性能榜单
+
+我们将陆续提供开源模型和API模型的具体性能榜单，请见 [OpenCompass Leaderboard](https://opencompass.org.cn/rank) 。如需加入评测，请提供模型仓库地址或标准的 API 接口至邮箱  `[email protected]`.
+
+<p align="right"><a href="#top">🔝返回顶部</a></p>
+
+## 🛠️ 安装
+
+下面展示了快速安装以及准备数据集的步骤。
+
+### 💻 环境配置
+
+#### 面向开源模型的GPU环境
+
+```bash
+conda create --name opencompass python=3.10 pytorch torchvision pytorch-cuda -c nvidia -c pytorch -y
+conda activate opencompass
+git clone https://github.com/open-compass/opencompass opencompass
+cd opencompass
+pip install -e .
+```
+
+#### 面向API模型测试的CPU环境
+
+```bash
+conda create -n opencompass python=3.10 pytorch torchvision torchaudio cpuonly -c pytorch -y
+conda activate opencompass
+git clone https://github.com/open-compass/opencompass opencompass
+cd opencompass
+pip install -e .
+# 如果需要使用各个API模型，请 `pip install -r requirements/api.txt` 安装API模型的相关依赖
+```
+
+### 📂 数据准备
+
+```bash
+# 下载数据集到 data/ 处
+wget https://github.com/open-compass/opencompass/releases/download/0.2.2.rc1/OpenCompassData-core-20240207.zip
+unzip OpenCompassData-core-20240207.zip
+```
+
+有部分第三方功能,如 Humaneval 以及 Llama,可能需要额外步骤才能正常运行，详细步骤请参考[安装指南](https://opencompass.readthedocs.io/zh_CN/latest/get_started/installation.html)。
+
+<p align="right"><a href="#top">🔝返回顶部</a></p>
+
+## 🏗️ ️评测
+
+确保按照上述步骤正确安装 OpenCompass 并准备好数据集后，可以通过以下命令评测 LLaMA-7b 模型在 MMLU 和 C-Eval 数据集上的性能：
+
+```bash
+python run.py --models hf_llama_7b --datasets mmlu_ppl ceval_ppl
+```
+
+OpenCompass 预定义了许多模型和数据集的配置，你可以通过 [工具](./docs/zh_cn/tools.md#ListConfigs) 列出所有可用的模型和数据集配置。
+
+```bash
+# 列出所有配置
+python tools/list_configs.py
+# 列出所有跟 llama 及 mmlu 相关的配置
+python tools/list_configs.py llama mmlu
+```
+
+你也可以通过命令行去评测其它 HuggingFace 模型。同样以 LLaMA-7b 为例：
+
+```bash
+python run.py --datasets ceval_ppl mmlu_ppl \
+--hf-path huggyllama/llama-7b \  # HuggingFace 模型地址
+--model-kwargs device_map='auto' \  # 构造 model 的参数
+--tokenizer-kwargs padding_side='left' truncation='left' use_fast=False \  # 构造 tokenizer 的参数
+--max-out-len 100 \  # 最长生成 token 数
+--max-seq-len 2048 \  # 模型能接受的最大序列长度
+--batch-size 8 \  # 批次大小
+--no-batch-padding \  # 不打开 batch padding，通过 for loop 推理，避免精度损失
+--num-gpus 1  # 运行该模型所需的最少 gpu 数
+```
+
+> **注意**<br />
+> 若需要运行上述命令，你需要删除所有从 `# ` 开始的注释。
+
+通过命令行或配置文件，OpenCompass 还支持评测 API 或自定义模型，以及更多样化的评测策略。请阅读[快速开始](https://opencompass.readthedocs.io/zh_CN/latest/get_started/quick_start.html)了解如何运行一个评测任务。
+
+更多教程请查看我们的[文档](https://opencompass.readthedocs.io/zh_CN/latest/index.html)。
+
+<p align="right"><a href="#top">🔝返回顶部</a></p>
+
+## 📖 数据集支持
+
+<table align="center">
+  <tbody>
+    <tr align="center" valign="bottom">
+      <td>
+        <b>语言</b>
+      </td>
+      <td>
+        <b>知识</b>
+      </td>
+      <td>
+        <b>推理</b>
+      </td>
+      <td>
+        <b>考试</b>
+      </td>
+    </tr>
+    <tr valign="top">
+      <td>
+<details open>
+<summary><b>字词释义</b></summary>
+
+- WiC
+- SummEdits
+
+</details>
+
+<details open>
+<summary><b>成语习语</b></summary>
+
+- CHID
+
+</details>
+
+<details open>
+<summary><b>语义相似度</b></summary>
+
+- AFQMC
+- BUSTM
+
+</details>
+
+<details open>
+<summary><b>指代消解</b></summary>
+
+- CLUEWSC
+- WSC
+- WinoGrande
+
+</details>
+
+<details open>
+<summary><b>翻译</b></summary>
+
+- Flores
+- IWSLT2017
+
+</details>
+
+<details open>
+<summary><b>多语种问答</b></summary>
+
+- TyDi-QA
+- XCOPA
+
+</details>
+
+<details open>
+<summary><b>多语种总结</b></summary>
+
+- XLSum
+
+</details>
+      </td>
+      <td>
+<details open>
+<summary><b>知识问答</b></summary>
+
+- BoolQ
+- CommonSenseQA
+- NaturalQuestions
+- TriviaQA
+
+</details>
+      </td>
+      <td>
+<details open>
+<summary><b>文本蕴含</b></summary>
+
+- CMNLI
+- OCNLI
+- OCNLI_FC
+- AX-b
+- AX-g
+- CB
+- RTE
+- ANLI
+
+</details>
+
+<details open>
+<summary><b>常识推理</b></summary>
+
+- StoryCloze
+- COPA
+- ReCoRD
+- HellaSwag
+- PIQA
+- SIQA
+
+</details>
+
+<details open>
+<summary><b>数学推理</b></summary>
+
+- MATH
+- GSM8K
+
+</details>
+
+<details open>
+<summary><b>定理应用</b></summary>
+
+- TheoremQA
+- StrategyQA
+- SciBench
+
+</details>
+
+<details open>
+<summary><b>综合推理</b></summary>
+
+- BBH
+
+</details>
+      </td>
+      <td>
+<details open>
+<summary><b>初中/高中/大学/职业考试</b></summary>
+
+- C-Eval
+- AGIEval
+- MMLU
+- GAOKAO-Bench
+- CMMLU
+- ARC
+- Xiezhi
+
+</details>
+
+<details open>
+<summary><b>医学考试</b></summary>
+
+- CMB
+
+</details>
+      </td>
+    </tr>
+</td>
+    </tr>
+  </tbody>
+  <tbody>
+    <tr align="center" valign="bottom">
+      <td>
+        <b>理解</b>
+      </td>
+      <td>
+        <b>长文本</b>
+      </td>
+      <td>
+        <b>安全</b>
+      </td>
+      <td>
+        <b>代码</b>
+      </td>
+    </tr>
+    <tr valign="top">
+      <td>
+<details open>
+<summary><b>阅读理解</b></summary>
+
+- C3
+- CMRC
+- DRCD
+- MultiRC
+- RACE
+- DROP
+- OpenBookQA
+- SQuAD2.0
+
+</details>
+
+<details open>
+<summary><b>内容总结</b></summary>
+
+- CSL
+- LCSTS
+- XSum
+- SummScreen
+
+</details>
+
+<details open>
+<summary><b>内容分析</b></summary>
+
+- EPRSTMT
+- LAMBADA
+- TNEWS
+
+</details>
+      </td>
+      <td>
+<details open>
+<summary><b>长文本理解</b></summary>
+
+- LEval
+- LongBench
+- GovReports
+- NarrativeQA
+- Qasper
+
+</details>
+      </td>
+      <td>
+<details open>
+<summary><b>安全</b></summary>
+
+- CivilComments
+- CrowsPairs
+- CValues
+- JigsawMultilingual
+- TruthfulQA
+
+</details>
+<details open>
+<summary><b>健壮性</b></summary>
+
+- AdvGLUE
+
+</details>
+      </td>
+      <td>
+<details open>
+<summary><b>代码</b></summary>
+
+- HumanEval
+- HumanEvalX
+- MBPP
+- APPs
+- DS1000
+
+</details>
+      </td>
+    </tr>
+</td>
+    </tr>
+  </tbody>
+</table>
+
+<p align="right"><a href="#top">🔝返回顶部</a></p>
+
+## 📖 模型支持
+
+<table align="center">
+  <tbody>
+    <tr align="center" valign="bottom">
+      <td>
+        <b>开源模型</b>
+      </td>
+      <td>
+        <b>API 模型</b>
+      </td>
+      <!-- <td>
+        <b>自定义模型</b>
+      </td> -->
+    </tr>
+    <tr valign="top">
+      <td>
+
+- [InternLM](https://github.com/InternLM/InternLM)
+- [LLaMA](https://github.com/facebookresearch/llama)
+- [Vicuna](https://github.com/lm-sys/FastChat)
+- [Alpaca](https://github.com/tatsu-lab/stanford_alpaca)
+- [Baichuan](https://github.com/baichuan-inc)
+- [WizardLM](https://github.com/nlpxucan/WizardLM)
+- [ChatGLM2](https://github.com/THUDM/ChatGLM2-6B)
+- [ChatGLM3](https://github.com/THUDM/ChatGLM3-6B)
+- [TigerBot](https://github.com/TigerResearch/TigerBot)
+- [Qwen](https://github.com/QwenLM/Qwen)
+- [BlueLM](https://github.com/vivo-ai-lab/BlueLM)
+- ……
+
+</td>
+<td>
+
+- OpenAI
+- Claude
+- ZhipuAI(ChatGLM)
+- Baichuan
+- ByteDance(YunQue)
+- Huawei(PanGu)
+- 360
+- Baidu(ERNIEBot)
+- MiniMax(ABAB-Chat)
+- SenseTime(nova)
+- Xunfei(Spark)
+- ……
+
+</td>
+
+</tr>
+  </tbody>
+</table>
+
+<p align="right"><a href="#top">🔝返回顶部</a></p>
+
+## 🔜 路线图
+
+- [ ] 主观评测
+  - [ ] 发布主观评测榜单
+  - [ ] 发布主观评测数据集
+- [x] 长文本
+  - [ ] 支持广泛的长文本评测集
+  - [ ] 发布长文本评测榜单
+- [ ] 代码能力
+  - [ ] 发布代码能力评测榜单
+  - [x] 提供非Python语言的评测服务
+- [ ] 智能体
+  - [ ] 支持丰富的智能体方案
+  - [ ] 提供智能体评测榜单
+- [x] 鲁棒性
+  - [x] 支持各类攻击方法
+
+## 👷‍♂️ 贡献
+
+我们感谢所有的贡献者为改进和提升 OpenCompass 所作出的努力。请参考[贡献指南](https://opencompass.readthedocs.io/zh_CN/latest/notes/contribution_guide.html)来了解参与项目贡献的相关指引。
+
+## 🤝 致谢
+
+该项目部分的代码引用并修改自 [OpenICL](https://github.com/Shark-NLP/OpenICL)。
+
+该项目部分的数据集和提示词实现修改自 [chain-of-thought-hub](https://github.com/FranxYao/chain-of-thought-hub), [instruct-eval](https://github.com/declare-lab/instruct-eval)
+
+## 🖊️ 引用
+
+```bibtex
+@misc{2023opencompass,
+    title={OpenCompass: A Universal Evaluation Platform for Foundation Models},
+    author={OpenCompass Contributors},
+    howpublished = {\url{https://github.com/open-compass/opencompass}},
+    year={2023}
+}
+```
+
+<p align="right"><a href="#top">🔝返回顶部</a></p>

opencompass-my-api/build/lib/opencompass/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.2.2'

opencompass-my-api/build/lib/opencompass/datasets/FinanceIQ.py

@@ -0,0 +1,39 @@
+import csv
+import os.path as osp
+
+from datasets import Dataset, DatasetDict
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class FinanceIQDataset(BaseDataset):
+
+    # @staticmethod
+    # def load(path: str):
+    #     from datasets import load_dataset
+    #     return load_dataset('csv', data_files={'test': path})
+
+    @staticmethod
+    def load(path: str, name: str):
+        dataset = DatasetDict()
+        for split in ['dev', 'test']:
+            raw_data = []
+            filename = osp.join(path, split, f'{name}.csv')
+            with open(filename, encoding='utf-8') as f:
+                reader = csv.reader(f)
+                _ = next(reader)  # skip the header
+                for row in reader:
+                    assert len(row) == 7
+                    raw_data.append({
+                        'question': row[1],
+                        'A': row[2],
+                        'B': row[3],
+                        'C': row[4],
+                        'D': row[5],
+                        'answer': row[6],
+                    })
+            dataset[split] = Dataset.from_list(raw_data)
+        return dataset

opencompass-my-api/build/lib/opencompass/datasets/GaokaoBench.py

@@ -0,0 +1,132 @@
+import json
+import re
+
+from datasets import Dataset
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class GaokaoBenchDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        with open(path, encoding='utf-8') as f:
+            data = json.load(f)
+        return Dataset.from_list(data['example'])
+
+
+valid_gaokao_bench_question_types = [
+    'single_choice', 'multi_choice', 'multi_question_choice',
+    'five_out_of_seven', 'cloze', 'subjective', 'correction'
+]
+
+
+class GaokaoBenchEvaluator(BaseEvaluator):
+
+    def __init__(self, question_type) -> None:
+        super().__init__()
+        assert question_type in valid_gaokao_bench_question_types
+        self.question_type = question_type
+
+    def do_predictions_postprocess(self, model_output, answer_lenth=None):
+        if self.question_type == 'single_choice':
+            model_answer = []
+            temp = re.findall(r'[A-D]', model_output[::-1])
+            if len(temp) != 0:
+                model_answer.append(temp[0])
+
+        elif self.question_type == 'multi_question_choice':
+            model_answer = []
+            temp = re.findall(r'【答案】\s*[:：]*\s*[A-Z]', model_output)
+
+            if len(temp) == answer_lenth:
+                for t in temp:
+                    model_answer.append(re.findall(r'[A-Z]', t)[0])
+            else:
+                temp = re.findall(r'[A-Z]', model_output)
+                if len(temp) > 0:
+                    for k in range(min(len(temp), answer_lenth)):
+                        model_answer.append(temp[k])
+
+        elif self.question_type == 'multi_choice':
+            model_answer = []
+            answer = ''
+            content = re.sub(r'\s+', '', model_output)
+            answer_index = content.find('【答案】')
+            if answer_index > 0:
+                temp = content[answer_index:]
+                if len(re.findall(r'[A-D]', temp)) > 0:
+                    for t in re.findall(r'[A-D]', temp):
+                        answer += t
+            else:
+                temp = content[-10:]
+                if len(re.findall(r'[A-D]', temp)) > 0:
+                    for t in re.findall(r'[A-D]', temp):
+                        answer += t
+            if len(answer) != 0:
+                model_answer.append(answer)
+
+        elif self.question_type == 'five_out_of_seven':
+            model_answer = []
+            temp = re.findall(r'[A-G]', model_output)
+            if len(temp) > 0:
+                for k in range(min(5, len(temp))):
+                    model_answer.append(temp[k])
+
+        return model_answer
+
+    def ensure_same_length(self, pred, refr):
+        if len(pred) == len(refr):
+            return pred
+        return ['Z'] * len(refr)
+
+    def score(self, predictions, references):
+        if self.question_type not in [
+                'single_choice', 'multi_choice', 'multi_question_choice',
+                'five_out_of_seven'
+        ]:
+            return {'score': 0}
+        elif self.question_type == 'multi_choice':
+            correct_score, total_score = 0, 0
+            for pred, refr in zip(predictions, references):
+                pred = self.do_predictions_postprocess(pred)
+                pred = self.ensure_same_length(pred, refr)
+                for p, r in zip(pred, refr):
+                    if p == r:
+                        correct_score += 2
+                    else:
+                        for i in p:
+                            if i not in r:
+                                break
+                        else:
+                            correct_score += 1
+                    total_score += 2
+            return {'score': correct_score / total_score * 100}
+        else:
+            correct_score, total_score = 0, 0
+            for pred, refr in zip(predictions, references):
+                if self.question_type == 'multi_question_choice':
+                    pred = self.do_predictions_postprocess(pred, len(refr))
+                else:
+                    pred = self.do_predictions_postprocess(pred)
+                pred = self.ensure_same_length(pred, refr)
+                for p, r in zip(pred, refr):
+                    if p == r:
+                        correct_score += 1
+                    total_score += 1
+            return {'score': correct_score / total_score * 100}
+
+
+for question_type in valid_gaokao_bench_question_types:
+    # fix classic closure problem
+    def _gaokao_register(question_type):
+        ICL_EVALUATORS.register_module(
+            name='GaokaoBenchEvaluator' + '_' + question_type,
+            module=lambda *args, **kwargs: GaokaoBenchEvaluator(
+                question_type=question_type, *args, **kwargs))
+
+    _gaokao_register(question_type)

opencompass-my-api/build/lib/opencompass/datasets/NPHardEval/__init__.py

@@ -0,0 +1,9 @@
+from .cmp_GCP_D import *  # noqa: F401, F403
+from .cmp_KSP import *  # noqa: F401, F403
+from .cmp_TSP_D import *  # noqa: F401, F403
+from .hard_GCP import *  # noqa: F401, F403
+from .hard_MSP import *  # noqa: F401, F403
+from .hard_TSP import *  # noqa: F401, F403
+from .p_BSP import *  # noqa: F401, F403
+from .p_EDP import *  # noqa: F401, F403
+from .p_SPP import *  # noqa: F401, F403

opencompass-my-api/build/lib/opencompass/datasets/NPHardEval/cmp_GCP_D.py

@@ -0,0 +1,161 @@
+import ast
+
+import networkx as nx
+from datasets import Dataset
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
+
+from ..base import BaseDataset
+from .prompts import gcp_dPrompts
+
+
+def q2text(q, p=gcp_dPrompts):
+    number_of_colors = q.split('\n')[0].split()[-2]  # last character of the first line
+    number_of_vertices = q.split('\n')[1].split(' ')[2]  # third word of the second line
+    prompt_text =   p['Intro'] + '\n' + \
+                    p['Initial_question'].format(total_vertices=number_of_vertices, number_of_colors=number_of_colors) + '\n' + \
+                    p['Output_content'] + '\n' + \
+                    p['Output_format'] + '\n' + \
+                    '\n The graph is below: \n'
+    for line in q.split('\n')[2:]:
+        vertex_list = line.split(' ')
+        this_line = 'Vertex {} is connected to vertex {}.'.format(
+            vertex_list[1], vertex_list[2])
+        prompt_text += this_line + '\n'
+    return prompt_text
+
+
+@LOAD_DATASET.register_module(force=True)
+class cmp_GCP_D_Dataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        raw_data = []
+        data_path = path
+        all_data = []
+        for file_num in range(10):
+            with open(data_path + 'decision_data_GCP_{}.txt'.format(file_num)) as f:
+                data = f.read()
+                sample = data.split('\n\n')[:-1]
+            all_data += zip([file_num + 1] * len(sample), sample)
+        for (level, q) in all_data:
+            prompt = q2text(q)
+            raw_data.append({
+                'prompt': prompt,
+                'q': str(level) + '####\n' + q,
+                'level': level
+            })
+        dataset = Dataset.from_list(raw_data)
+        return dataset
+
+
+@ICL_EVALUATORS.register_module(force=True)
+class cmp_GCP_D_Evaluator(BaseEvaluator):
+
+    def score(self, predictions, references):
+        assert len(predictions) == len(references)
+
+        result = {'pass': 0, 'fail': 0}
+        details = {}
+        for index, (q, output) in enumerate(zip(references, predictions)):
+            output_dict = {}
+            level = int(q.split('####\n')[0])
+            q = q.split('####\n')[-1]
+            try:
+                number_of_colors = int(q.split('\n')[0].split()[-2])
+                output, reasoning = self.parse_xml_to_dict(output)
+                output_dict['output'] = output
+                output_dict['correctness'], _ = self.gcp_decision_check(q, output, number_of_colors)
+            except Exception as e:
+                print(f'Attempt failed: {e}')
+                output_dict['correctness'] = False
+            output_dict['reasoning'] = reasoning
+
+            if output_dict['correctness']:
+                r = 'pass'
+            else:
+                r = 'fail'
+            result[r] += level
+            details[str(index)] = {'q': q, 'output': output, 'result': r}
+
+        result['score'] = result['pass'] / (result['pass'] + result['fail']) * 100
+        result['details'] = details
+        final_result = {'Weighted Accuracy': result['score']}
+        return final_result
+
+    def parse_xml_to_dict(self, xml_string):
+        try:
+            assert '<final_answer>' in xml_string
+            assert '</final_answer>' in xml_string
+            assert '<reasoning>' in xml_string
+            assert '</reasoning>' in xml_string
+            final_answer_start = xml_string.index('<final_answer>') + len('<final_answer>')
+            final_answer_end = xml_string.index('</final_answer>')
+            reasoning_start = xml_string.index('<reasoning>') + len('<reasoning>')
+            reasoning_end = xml_string.index('</reasoning>')
+            final_answer_element = xml_string[final_answer_start:final_answer_end].rstrip().strip().rstrip()
+            reasoning_element = xml_string[reasoning_start:reasoning_end].rstrip().strip().rstrip()
+            try:
+                final_answer_element = ast.literal_eval(final_answer_element)
+            except Exception:
+                final_answer_element = ''
+        except Exception:
+            final_answer_element = ''
+            reasoning_element = ''
+
+        return final_answer_element, reasoning_element
+
+    def read_dimacs_format(self, dimacs_str):
+        lines = dimacs_str.strip().split('\n')
+        p_line = next(line for line in lines if line.startswith('p'))
+        _, _, num_vertices, num_edges = p_line.split()
+        num_vertices, num_edges = int(num_vertices), int(num_edges)
+
+        adjacency_list = {i: set() for i in range(1, num_vertices + 1)}
+        for line in lines:
+            if line.startswith('e'):
+                _, vertex1, vertex2 = line.split()
+                vertex1, vertex2 = int(vertex1), int(vertex2)
+                if vertex1 in adjacency_list and vertex2 in adjacency_list:
+                    adjacency_list[vertex1].add(vertex2)
+                    adjacency_list[vertex2].add(vertex1)
+
+        return num_vertices, adjacency_list
+
+    def gcp_greedy_solution(self, adjacency_list):
+        """Provides a greedy solution to the GCP problem.
+
+        :param adjacency_list: A dictionary of the adjacency list.
+        :return: A tuple of (num_colors, coloring).
+        """
+        G = nx.Graph()
+        G.add_nodes_from(adjacency_list.keys())
+        for vertex, neighbors in adjacency_list.items():
+            for neighbor in neighbors:
+                G.add_edge(vertex, neighbor)
+        coloring = nx.coloring.greedy_color(G, strategy='largest_first')
+        num_colors = max(coloring.values()) + 1
+        return num_colors, coloring
+
+    def gcp_decision_check(self, dimacs_str, answer, k_colors):
+        """Check if the given GCP instance is feasible with k_colors.
+
+        :param dimacs_str: The DIMACS format string of the GCP instance.
+        :param answer: The answer returned by the model.
+        :param k_colors: The target number of colors.
+        :return: A tuple of (is_correct, message).
+        """
+        num_vertices, adjacency_list = self.read_dimacs_format(dimacs_str)
+        try:
+            is_feasible = answer.get('Feasible', 'no').lower() == 'yes'
+        except Exception:
+            return False, 'Feasible key not found'
+        num_colors, coloring = self.gcp_greedy_solution(adjacency_list)
+        exist_optimal = num_colors <= k_colors
+        if is_feasible != exist_optimal:
+            if exist_optimal:
+                return False, f'Feasibility mismatch: {coloring}'
+            else:
+                return False, f'Feasibility mismatch: {is_feasible} vs {exist_optimal}'
+        return True, 'Feasible' if is_feasible else 'Infeasible'

opencompass-my-api/build/lib/opencompass/datasets/NPHardEval/cmp_KSP.py

@@ -0,0 +1,183 @@
+import ast
+import json
+
+from datasets import Dataset
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
+
+from ..base import BaseDataset
+from .prompts import kspPrompts
+
+
+def q2text(q, p=kspPrompts):
+    knapsack_capacity = q['knapsack_capacity']
+    items = q['items']
+    prompt_text = p['Intro'] + '\n' + \
+                p['Initial_question'].format(knapsack_capacity=knapsack_capacity) + '\n' + \
+                p['Output_content'] + '\n' + \
+                p['Output_format'] + \
+                '\n The items details are as below: \n'
+    for item in items:
+        this_line = f"Item {item['id']} has weight {item['weight']} and value {item['value']}."
+        prompt_text += this_line + '\n'
+    return prompt_text
+
+
+@LOAD_DATASET.register_module(force=True)
+class cmp_KSP_Dataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        raw_data = []
+        data_path = path
+        all_data = []
+        with open(data_path + 'ksp_instances.json', 'r') as f:
+            data = json.load(f)
+            for sample in data:
+                level = len(sample['items']) - 3
+                all_data.append((level, sample))
+        for (level, q) in all_data:
+            prompt = q2text(q)
+            raw_data.append({
+                'prompt': prompt,
+                'q': str(level) + '####\n' + json.dumps(q),
+                'level': level
+            })
+        dataset = Dataset.from_list(raw_data)
+        return dataset
+
+
+@ICL_EVALUATORS.register_module(force=True)
+class cmp_KSP_Evaluator(BaseEvaluator):
+
+    def score(self, predictions, references):
+        assert len(predictions) == len(references)
+
+        result = {'pass': 0, 'fail': 0}
+        details = {}
+        for index, (q, output) in enumerate(zip(references, predictions)):
+            output_dict = {}
+            level = int(q.split('####\n')[0])
+            q = json.loads(q.split('####\n')[-1])
+            try:
+                llm_string = q
+                output, reasoning = self.parse_xml_to_dict(llm_string)
+                output_dict['output'] = output
+                output_dict['correctness'], _ = self.kspCheck(q, output)
+                output_dict['reasoning'] = reasoning
+                output_dict['level'] = level
+            except Exception as e:
+                print(f'Attempt failed: {e}')
+            if output_dict:
+                if output_dict['correctness']:
+                    r = 'pass'
+                else:
+                    r = 'fail'
+            else:
+                print(f'Failed to run {q}')
+                r = 'fail'
+
+            result[r] += level
+            details[str(index)] = {'q': q, 'output': output, 'result': r}
+
+        result['score'] = result['pass'] / (result['pass'] + result['fail']) * 100
+        result['details'] = details
+        final_result = {'Weighted Accuracy': result['score']}
+        return final_result
+
+    def parse_xml_to_dict(self, xml_string):
+        try:
+            assert '<final_answer>' in xml_string
+            assert '</final_answer>' in xml_string
+            assert '<reasoning>' in xml_string
+            assert '</reasoning>' in xml_string
+            final_answer_start = xml_string.index('<final_answer>') + len('<final_answer>')
+            final_answer_end = xml_string.index('</final_answer>')
+            reasoning_start = xml_string.index('<reasoning>') + len('<reasoning>')
+            reasoning_end = xml_string.index('</reasoning>')
+            final_answer_element = xml_string[final_answer_start:final_answer_end].rstrip().strip().rstrip()
+            reasoning_element = xml_string[reasoning_start:reasoning_end].rstrip().strip().rstrip()
+            try:
+                final_answer_element = ast.literal_eval(final_answer_element)
+            except Exception:
+                final_answer_element = ''
+        except Exception:
+            final_answer_element = ''
+            reasoning_element = ''
+
+        return final_answer_element, reasoning_element
+
+    def ksp_optimal_solution(self, knapsacks, capacity):
+        """Provides the optimal solution for the KSP instance with dynamic
+        programming.
+
+        :param knapsacks: A dictionary of the knapsacks.
+        :param capacity: The capacity of the knapsack.
+        :return: The optimal value.
+        """
+        # num_knapsacks = len(knapsacks)
+
+        # Create a one-dimensional array to store intermediate solutions
+        dp = [0] * (capacity + 1)
+
+        for itemId, (weight, value) in knapsacks.items():
+            for w in range(capacity, weight - 1, -1):
+                dp[w] = max(dp[w], value + dp[w - weight])
+
+        return dp[capacity]
+
+    # KSP
+    def kspCheck(self, instance, solution):
+        """Validates the solution for the KSP instance.
+
+        :param instance: A dictionary of the KSP instance.
+        :param solution: A dictionary of the solution.
+        :return: A tuple of (is_correct, message).
+        """
+        # Change string key to integer key and value to boolean
+        items = instance.get('items', [])
+        knapsacks = {
+            item['id']: (item['weight'], item['value'])
+            for item in items
+        }
+
+        ksp_optimal_value = self.ksp_optimal_solution(
+            knapsacks, instance['knapsack_capacity'])
+
+        try:
+            is_feasible = (solution.get('Feasible', '').lower() == 'yes')
+        except Exception:
+            return False, f'Output format is incorrect.'
+        if is_feasible != (ksp_optimal_value > 0):
+            return False, f'The solution is {is_feasible} but the optimal solution is {ksp_optimal_value > 0}.'
+
+        total_value = int(solution.get('TotalValue', -1))
+        selectedItems = list(map(int, solution.get('SelectedItemIds', [])))
+
+        if len(set(selectedItems)) != len(selectedItems):
+            return False, f'Duplicate items are selected.'
+
+        total_weight = 0
+        cum_value = 0
+
+        # Calculate total weight and value of selected items
+        for item in selectedItems:
+            if knapsacks.get(item, False):
+                weight, value = knapsacks[item]
+                total_weight += weight
+                cum_value += value
+            else:
+                return False, f'Item {item} does not exist.'
+
+        # Check if the item weight exceeds the knapsack capacity
+        if total_weight > instance['knapsack_capacity']:
+            return False, f"Total weight {total_weight} exceeds knapsack capacity {instance['knapsack_capacity']}."
+
+        if total_value != cum_value:
+            return False, f'The total value {total_value} does not match the cumulative value {cum_value} of the selected items.'
+
+        if total_value != ksp_optimal_value:
+            return False, f'The total value {total_value} does not match the optimal value {ksp_optimal_value}.'
+
+        return True, f'The solution is valid with total weight {total_weight} and total value {total_value}.'

opencompass-my-api/build/lib/opencompass/datasets/NPHardEval/cmp_TSP_D.py

@@ -0,0 +1,150 @@
+import ast
+import json
+
+import networkx as nx
+import pandas as pd
+from datasets import Dataset
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
+
+from ..base import BaseDataset
+from .prompts import tsp_dPrompts
+
+
+def q2text(adj_matrix, distance_limit, p=tsp_dPrompts):
+    total_cities = adj_matrix.shape[0]  # exclude the last row
+    prompt_text = p['Intro'] + '\n' + \
+                p['Initial_question'].format(total_cities=total_cities, distance_limit=distance_limit) + '\n' + \
+                p['Output_content'] + '\n' + \
+                p['Output_format'] + '\n' + \
+                'The distances between cities are below: \n'
+
+    for i in range(adj_matrix.shape[0]):
+        for j in range(adj_matrix.shape[1]):
+            if i < j:  # only use the upper triangle
+                this_line = 'The distance between City {} and City {} is {}.'.format(i, j, adj_matrix[i, j])
+                prompt_text += this_line + '\n'
+    return prompt_text
+
+
+@LOAD_DATASET.register_module(force=True)
+class cmp_TSP_D_Dataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        raw_data = []
+        data_path = path
+        all_data = []
+        for level in range(10):
+            for file_num in range(10):
+                df = pd.read_csv(data_path + 'decision_data_TSP_level_{}_instance_{}.csv'.format(level, file_num + 1),
+                    header=None,
+                    index_col=False)
+                all_data.append((level + 1, df))
+
+        for (level, q) in all_data:
+            threshold = q.iloc[-1, 0]  # therashold is the last row
+            distance_matrix = q.iloc[:
+                                     -1].values  # distance matrix is the rest of the rows
+            prompt = q2text(distance_matrix, threshold)
+            raw_data.append({
+                'prompt': prompt,
+                'q': str(level) + '####\n' + json.dumps(q.to_json()),
+                'level': level
+            })
+        dataset = Dataset.from_list(raw_data)
+        return dataset
+
+
+@ICL_EVALUATORS.register_module(force=True)
+class cmp_TSP_D_Evaluator(BaseEvaluator):
+
+    def score(self, predictions, references):
+        assert len(predictions) == len(references)
+
+        result = {'pass': 0, 'fail': 0}
+        details = {}
+        tsp_d_Results = []
+        for index, (q, llm_string) in enumerate(zip(references, predictions)):
+            output_dict = {}
+            output, reasoning = self.parse_xml_to_dict(llm_string)
+            level = int(q.split('####\n')[0])
+            q = json.loads(q.split('####\n')[-1])
+            q = pd.DataFrame(eval(q))
+            threshold = q.iloc[-1, 0]  # therashold is the last row
+            distance_matrix = q.iloc[:-1].values  # distance matrix is the rest of the rows
+            output_dict['output'] = output
+            try:
+                output_dict['correctness'], _ = self.tsp_decision_check(distance_matrix, threshold, output)
+            except Exception as e:
+                print(f'Check failed: {e}')
+                output_dict['correctness'] = False
+            output_dict['reasoning'] = reasoning
+            output_dict['level'] = level
+            if output_dict:
+                tsp_d_Results.append(output_dict)
+                if output_dict['correctness']:
+                    r = 'pass'
+                else:
+                    r = 'fail'
+
+            result[r] += level
+            details[str(index)] = {'q': q, 'output': output, 'result': r}
+
+        result['score'] = result['pass'] / (result['pass'] + result['fail']) * 100
+        result['details'] = details
+        final_result = {'Weighted Accuracy': result['score']}
+        return final_result
+
+    def parse_xml_to_dict(self, xml_string):
+        try:
+            assert '<final_answer>' in xml_string
+            assert '</final_answer>' in xml_string
+            assert '<reasoning>' in xml_string
+            assert '</reasoning>' in xml_string
+            final_answer_start = xml_string.index('<final_answer>') + len('<final_answer>')
+            final_answer_end = xml_string.index('</final_answer>')
+            reasoning_start = xml_string.index('<reasoning>') + len('<reasoning>')
+            reasoning_end = xml_string.index('</reasoning>')
+            final_answer_element = xml_string[final_answer_start:final_answer_end].rstrip().strip().rstrip()
+            reasoning_element = xml_string[reasoning_start:reasoning_end].rstrip().strip().rstrip()
+            try:
+                final_answer_element = ast.literal_eval(final_answer_element)
+            except Exception:
+                final_answer_element = ''
+        except Exception:
+            final_answer_element = ''
+            reasoning_element = ''
+
+        return final_answer_element, reasoning_element
+
+    def tsp_approx(self, distance_matrix):
+        """Returns an approximate solution to the TSP problem.
+
+        :param distance_matrix: A 2D numpy array representing the distance matrix.
+        :return: A list of the cities in the order they were visited.
+        """
+        G = nx.from_numpy_array(distance_matrix)
+        return nx.approximation.traveling_salesman_problem(G)
+
+    def tsp_decision_check(self, distance_matrix, threshold, tour):
+        """Checks if a given TSP tour is valid and within the threshold
+        distance.
+
+        :param distance_matrix: A 2D numpy array representing the distance matrix.
+        :param threshold: The maximum distance allowed.
+        :param tour: A dictionary containing the feasibility.
+        """
+        try:
+            is_feasible = tour.get('Feasible', 'no').lower() == 'yes'
+        except Exception:
+            return False, 'Output format incorrect'
+
+        # Calculate the approxed distance of the tour
+        tours = self.tsp_approx(distance_matrix)
+        tour_distance = sum(distance_matrix[tours[i], tours[i + 1]] for i in range(len(tours) - 1)) + distance_matrix[tours[-1], tours[0]]
+
+        if is_feasible != (tour_distance <= threshold):
+            return False, f'Feasibility mismatch: {is_feasible} vs {tour_distance} > {threshold}'
+        return True, 'Feasible: {} <= {}'.format(tour_distance, threshold)

opencompass-my-api/build/lib/opencompass/datasets/NPHardEval/hard_GCP.py

@@ -0,0 +1,189 @@
+import ast
+import xml.etree.ElementTree as ET
+
+from datasets import Dataset
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
+
+from ..base import BaseDataset
+from .prompts import gcpPrompts
+
+
+def q2text(q, p=gcpPrompts):  # q is the data for the HP-hard question, p is the prompt
+    # print(q)
+    chromatic_number = q.split('\n')[0][-1]  # last character of the first line
+    number_of_vertices = q.split('\n')[1].split(' ')[2]  # third word of the second line
+    prompt_text = p['Intro'] + '\n' \
+        + p['Initial_question'].format(max_vertices=number_of_vertices,max_colors=chromatic_number) + '\n' \
+        + p['Output_content'] + '\n' \
+        + p['Output_format'] + \
+        '\n The graph is below: \n'
+    for line in q.split('\n')[2:]:
+        vertex_list = line.split(' ')
+        this_line = 'Vertex {} is connected to vertex {}.'.format(vertex_list[1], vertex_list[2])
+        prompt_text += this_line + '\n'
+
+    return prompt_text
+
+
+@LOAD_DATASET.register_module(force=True)
+class hard_GCP_Dataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        raw_data = []
+        data_path = path
+        all_data = []
+        for file_num in range(10):
+            with open(data_path + 'synthesized_data_GCP_{}.txt'.format(file_num)) as f:
+                data = f.read()
+                sample = data.split('\n\n')[:-1]
+            all_data += zip([file_num + 1] * len(sample), sample)
+        for (level, q) in all_data:
+            prompt = q2text(q)
+            raw_data.append({
+                'prompt': prompt,
+                'q': str(level) + '####\n' + q,
+                'level': level
+            })
+        dataset = Dataset.from_list(raw_data)
+        return dataset
+
+
+@ICL_EVALUATORS.register_module(force=True)
+class hard_GCP_Evaluator(BaseEvaluator):
+
+    def score(self, predictions, references):
+        assert len(predictions) == len(references)
+
+        result = {'pass': 0, 'fail': 0}
+        details = {}
+        for index, (q, output) in enumerate(zip(references, predictions)):
+            output_dict = {}
+            level = int(q.split('####\n')[0])
+            q = q.split('####\n')[-1]
+
+            output_dict['output'] = output
+            try:
+                output_dict['correctness'] = self.gcpCheck(q, output)
+            except Exception as e:
+                print(f'Check failed: {e}')
+                output_dict['correctness'] = False
+            output_dict['level'] = level
+
+            if output_dict['correctness']:
+                r = 'pass'
+            else:
+                r = 'fail'
+            result[r] += level
+            details[str(index)] = {'q': q, 'output': output, 'result': r}
+
+        result['score'] = result['pass'] / (result['pass'] + result['fail']) * 100
+        result['details'] = details
+        final_result = {'Weighted Accuracy': result['score']}
+        return final_result
+
+    def parse_xml_to_dict(self, xml_string):
+        try:
+            # Parse the XML string
+            root = ET.fromstring(xml_string)
+
+            # Find the 'final_answer' tag
+            final_answer_element = root.find('final_answer')
+
+            # Find the 'reasoning' tag
+            reasoning_element = root.find('reasoning')
+        except Exception:
+            try:
+                assert '<final_answer>' in xml_string
+                assert '</final_answer>' in xml_string
+                assert '<reasoning>' in xml_string
+                assert '</reasoning>' in xml_string
+                final_answer_start = xml_string.index('<final_answer>') + len('<final_answer>')
+                final_answer_end = xml_string.index('</final_answer>')
+                reasoning_start = xml_string.index('<reasoning>') + len('<reasoning>')
+                reasoning_end = xml_string.index('</reasoning>')
+                final_answer_element = xml_string[final_answer_start:final_answer_end]
+                reasoning_element = xml_string[reasoning_start:reasoning_end]
+            except Exception:
+                final_answer_element = ''
+                reasoning_element = ''
+
+        return final_answer_element, reasoning_element
+
+    def gcpCheck(self, dimacs_str, answer_str):
+        num_vertices, adjacency_list = self.read_dimacs_format(dimacs_str)
+        answer_colors = self.parse_answer(answer_str)
+        # print(adjacency_list)
+        # print(answer_colors)
+
+        # Check if all colors in the answer are valid
+        for vertex, neighbors in adjacency_list.items():
+            for neighbor in neighbors:
+                try:
+                    if answer_colors[vertex] == answer_colors[neighbor]:
+                        print(f'Invalid coloring: Vertex {vertex} and {neighbor} have the same color.')
+                        return False
+                except:
+                    print(f'Invalid input.')  # dealing with hullucination
+                    return False
+
+        print(f'Valid coloring found with {len(set(answer_colors.values()))} colors: {answer_colors}')
+        return True
+
+    def read_dimacs_format(self, dimacs_str):
+        lines = dimacs_str.strip().split('\n')
+        # Read the number of vertices and edges
+        p_line = next(line for line in lines if line.startswith('p'))
+        _, _, num_vertices, num_edges = p_line.split()
+        num_vertices, num_edges = int(num_vertices), int(num_edges)
+
+        # Create adjacency list
+        adjacency_list = {i: set() for i in range(1, num_vertices + 1)}
+
+        # Read the edges and ignore those that reference non-existing vertices
+        for line in lines:
+            if line.startswith('e'):
+                _, vertex1, vertex2 = line.split()
+                vertex1, vertex2 = int(vertex1), int(vertex2)
+                if vertex1 in adjacency_list and vertex2 in adjacency_list:
+                    adjacency_list[vertex1].add(vertex2)
+                    adjacency_list[vertex2].add(vertex1)
+
+        return num_vertices, adjacency_list
+
+    def parse_answer(self, llm_string):
+        # # Convert the answer string to a dictionary
+        # answer_dict = {}
+        # # Remove the braces and split the string by commas
+        # entries = answer_str.strip("}{").split(', ')
+        # for entry in entries:
+        #     vertex, color = entry.split(':')
+        #     answer_dict[int(vertex)] = color
+        # return answer_dict
+
+        all_answers, reasoning_element = self.parse_xml_to_dict(llm_string)
+
+        if all_answers == '':
+            return {}
+        elif all_answers is None:
+            return {}
+        else:
+            if isinstance(all_answers, str):
+                try:
+                    all_answers = ast.literal_eval(all_answers)
+                except Exception:
+                    try:
+                        all_answers = ast.literal_eval('{' + all_answers + '}')
+                    except Exception:
+                        return {}
+            else:
+                all_answers = ast.literal_eval(all_answers.text)
+        # answer_dict = {}
+        # for pair in all_answers:
+        #     vertex, color = pair.split(":")
+        #     answer_dict[int(vertex)] = color
+        # convert key type to int
+        all_answers = {int(k): v for k, v in all_answers.items()}
+        return all_answers  # answer_dict

opencompass-my-api/build/lib/opencompass/datasets/NPHardEval/hard_MSP.py

@@ -0,0 +1,203 @@
+import ast
+import json
+import xml.etree.ElementTree as ET
+
+from datasets import Dataset
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
+
+from ..base import BaseDataset
+from .prompts import mspPrompts
+
+
+def q2text(q, p=mspPrompts):  # q is the data for the HP-hard question, p is the prompt
+    total_participants = q['participants']
+    total_timeslots = q['time_slots']
+    prompt_text = p['Intro'] + '\n' \
+        + p['Initial_question'].format(total_participants=total_participants,total_timeslots=total_timeslots) + '\n' \
+        + p['Output_content'] + '\n' \
+        + p['Output_format'] + \
+        '\n The meetings and participants details are as below: \n'
+    meetings = q['meetings']
+    participants = q['participants']
+    for meeting in meetings:
+        this_line = 'Meeting {} is with duration {}.'.format(meeting['id'], meeting['duration'])
+        prompt_text += this_line + '\n'
+    for j in participants.keys():
+        this_line = 'Participant {} is available at time slots {} and has meetings {}.'.format(j, participants[j]['available_slots'], participants[j]['meetings'])
+        prompt_text += this_line + '\n'
+    return prompt_text
+
+
+@LOAD_DATASET.register_module(force=True)
+class hard_MSP_Dataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        raw_data = []
+        data_path = path
+        all_data = []
+        with open(data_path + 'msp_instances.json', 'r') as f:
+            data = json.load(f)
+            all_data = zip([int(d['complexity_level']) for d in data], data)
+
+        for (level, q) in all_data:
+            prompt = q2text(q)
+            raw_data.append({
+                'prompt': prompt,
+                'q': str(level) + '####\n' + json.dumps(q),
+                'level': level
+            })
+        dataset = Dataset.from_list(raw_data)
+        return dataset
+
+
+@ICL_EVALUATORS.register_module(force=True)
+class hard_MSP_Evaluator(BaseEvaluator):
+
+    def score(self, predictions, references):
+        assert len(predictions) == len(references)
+
+        result = {'pass': 0, 'fail': 0}
+        for index, (q, output) in enumerate(zip(references, predictions)):
+            output_dict = {}
+            level = int(q.split('####\n')[0])
+            q = json.loads(q.split('####\n')[-1])
+
+            output_dict['output'] = output
+            output_dict['level'] = level
+            try:
+                output_dict['correctness'], _ = self.mspCheck(q, output)
+            except Exception as e:
+                print(f'Check failed: {e}')
+                output_dict['correctness'] = False
+
+            if output_dict['correctness']:
+                r = 'pass'
+            else:
+                r = 'fail'
+            result[r] += level
+
+        result['score'] = result['pass'] / (result['pass'] + result['fail']) * 100
+        final_result = {'Weighted Accuracy': result['score']}
+        return final_result
+
+    def mspCheck(self, instance, llm_string):
+        """Validate the MSP solution.
+
+        Parameters:
+        - instance: The MSP instance as a dictionary.
+        - solution: A dictionary with meeting ids as keys and lists of scheduled time slots as values.
+
+        Returns:
+        - A tuple (is_valid, message). is_valid is True if the solution is valid, False otherwise.
+        message contains information about the validity of the solution.
+        """
+        # print(llm_string)
+        solution, reasoning_element = self.parse_xml_to_dict(llm_string)
+        # print(solution.text)
+
+        # convert solution to dictionary
+        if solution == '':
+            return False, None
+        elif solution is None:
+            return False, None
+        else:
+            if isinstance(solution, str):
+                try:
+                    solution = ast.literal_eval(solution)
+                    if solution is None:
+                        return False, None
+                except Exception:
+                    try:
+                        solution = ast.literal_eval('{' + solution + '}')
+                        if solution is None:
+                            return False, None
+                    except Exception:
+                        return False, None
+            else:
+                try:
+                    solution = ast.literal_eval(solution.text)
+                    if solution is None:
+                        return False, None
+                except Exception:
+                    return False, None
+        # convert key type to int
+        if isinstance(solution, dict):
+            print(solution)
+            solution = {int(k): v for k, v in solution.items()}
+        else:
+            return False, None
+
+        # Check if all meetings are scheduled within the available time slots
+        for meeting in instance['meetings']:
+            m_id = meeting['id']
+            duration = meeting['duration']
+            scheduled_slots = solution.get(m_id, None)
+
+            # Check if the meeting is scheduled
+            if scheduled_slots is None:
+                return False, f'Meeting {m_id} is not scheduled.'
+
+            # Check if the meeting fits within the number of total time slots
+            if any(slot >= instance['time_slots'] for slot in scheduled_slots):
+                return False, f'Meeting {m_id} does not fit within the available time slots.'
+
+            # Check if the scheduled slots are contiguous and fit the meeting duration
+            if len(scheduled_slots) != duration or not all(scheduled_slots[i] + 1 == scheduled_slots[i + 1]
+                    for i in range(len(scheduled_slots) - 1)):
+                return False, f'Meeting {m_id} is not scheduled in contiguous time slots fitting its duration.'
+
+            # Check if all participants are available at the scheduled time
+            for p_id, participant in instance['participants'].items():
+                if m_id in participant['meetings']:
+                    if not all(slot in participant['available_slots'] for slot in scheduled_slots):
+                        return False, f'Participant {p_id} is not available for meeting {m_id} at the scheduled time.'
+
+        # Check if any participant is double-booked
+        participants_schedule = {p_id: [] for p_id in instance['participants']}
+        for m_id, time_slots in solution.items():
+            try:
+                duration = next(meeting['duration'] for meeting in instance['meetings'] if meeting['id'] == m_id)
+                if len(time_slots) != duration:
+                    return False, f'Meeting {m_id} duration does not match the number of scheduled time slots.'
+                for p_id, participant in instance['participants'].items():
+                    if m_id in participant['meetings']:
+                        participants_schedule[p_id].extend(time_slots)
+            except Exception:
+                return False, f'Meeting {m_id} is not in the instance or program error.'
+
+        for p_id, slots in participants_schedule.items():
+            if len(slots) != len(set(slots)):
+                return False, f'Participant {p_id} is double-booked.'
+
+        return True, 'The solution is valid.'
+
+    def parse_xml_to_dict(self, xml_string):
+        try:
+            # Parse the XML string
+            root = ET.fromstring(xml_string)
+
+            # Find the 'final_answer' tag
+            final_answer_element = root.find('final_answer')
+
+            # Find the 'reasoning' tag
+            reasoning_element = root.find('reasoning')
+        except:
+            try:
+                assert '<final_answer>' in xml_string
+                assert '</final_answer>' in xml_string
+                assert '<reasoning>' in xml_string
+                assert '</reasoning>' in xml_string
+                final_answer_start = xml_string.index('<final_answer>') + len('<final_answer>')
+                final_answer_end = xml_string.index('</final_answer>')
+                reasoning_start = xml_string.index('<reasoning>') + len('<reasoning>')
+                reasoning_end = xml_string.index('</reasoning>')
+                final_answer_element = xml_string[final_answer_start:final_answer_end]
+                reasoning_element = xml_string[reasoning_start:reasoning_end]
+            except:
+                final_answer_element = ''
+                reasoning_element = ''
+
+        return final_answer_element, reasoning_element

opencompass-my-api/build/lib/opencompass/datasets/NPHardEval/hard_TSP.py

@@ -0,0 +1,211 @@
+import ast
+import json
+import xml.etree.ElementTree as ET
+
+import numpy as np
+import pandas as pd
+from datasets import Dataset
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
+
+from ..base import BaseDataset
+from .prompts import tspPrompts
+
+
+def q2text(q, p=tspPrompts):  # q is the data for the HP-hard question, p is the prompt
+    total_cities = q.shape[0]
+    prompt_text = p['Intro'] + '\n' \
+        + p['Initial_question'].format(total_cities=total_cities) + '\n' \
+        + p['Output_content'] + '\n' \
+        + p['Output_format'] + \
+        '\n The distances between cities are below: \n'
+    for i in range(q.shape[0]):
+        for j in range(q.shape[1]):
+            if i < j:  # only use the upper triangle
+                this_line = 'The path between City {} and City {} is with distance {}.'.format(i, j, q.iloc[i, j])
+                prompt_text += this_line + '\n'
+    return prompt_text
+
+
+@LOAD_DATASET.register_module(force=True)
+class hard_TSP_Dataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        raw_data = []
+        data_path = path
+        all_data = []
+        for level in range(10):
+            for file_num in range(10):
+                # read np array
+                df = pd.read_csv(data_path + 'synthesized_data_TSP_level_{}_instance_{}.csv'.format(level, file_num + 1),
+                    header=None,
+                    index_col=False)
+                # transform df to
+                all_data.append((level + 1, df))
+        for (level, q) in all_data:
+            prompt = q2text(q)
+            raw_data.append({
+                'prompt': prompt,
+                'q': str(level) + '####\n' + json.dumps(q.to_json()),
+                'level': level
+            })
+        dataset = Dataset.from_list(raw_data)
+        return dataset
+
+
+@ICL_EVALUATORS.register_module(force=True)
+class hard_TSP_Evaluator(BaseEvaluator):
+
+    def score(self, predictions, references):
+        assert len(predictions) == len(references)
+
+        result = {'pass': 0, 'fail': 0}
+        for index, (q, output) in enumerate(zip(references, predictions)):
+            output_dict = {}
+            level = int(q.split('####\n')[0])
+            q = json.loads(q.split('####\n')[-1])
+            q = pd.DataFrame(eval(q))
+
+            output_dict['output'] = output
+            try:
+                output_dict['correctness'], _ = self.tspCheck(q, output)
+            except Exception as e:
+                print(f'Check failed: {e}')
+                output_dict['correctness'] = False
+            output_dict['level'] = level
+
+            if output_dict['correctness']:
+                r = 'pass'
+            else:
+                r = 'fail'
+            result[r] += level
+
+        result['score'] = result['pass'] / (result['pass'] + result['fail']) * 100
+        final_result = {'Weighted Accuracy': result['score']}
+        return final_result
+
+    def parse_xml_to_dict(self, xml_string):
+        try:
+            # Parse the XML string
+            root = ET.fromstring(xml_string)
+
+            # Find the 'final_answer' tag
+            final_answer_element = root.find('final_answer')
+
+            # Find the 'reasoning' tag
+            reasoning_element = root.find('reasoning')
+        except:
+            try:
+                assert '<final_answer>' in xml_string
+                assert '</final_answer>' in xml_string
+                assert '<reasoning>' in xml_string
+                assert '</reasoning>' in xml_string
+                final_answer_start = xml_string.index('<final_answer>') + len('<final_answer>')
+                final_answer_end = xml_string.index('</final_answer>')
+                reasoning_start = xml_string.index('<reasoning>') + len('<reasoning>')
+                reasoning_end = xml_string.index('</reasoning>')
+                final_answer_element = xml_string[final_answer_start:final_answer_end]
+                reasoning_element = xml_string[reasoning_start:reasoning_end]
+            except:
+                final_answer_element = ''
+                reasoning_element = ''
+
+        return final_answer_element, reasoning_element
+
+    def tspCheck(self, distance_matrix, llm_string):
+        """Check if the TSP solution is complete and if the distance matches
+        the greedy solution.
+
+        :param tour_string: String representing the TSP tour in the format "0->1->2->...->N->0"
+        :param distance_matrix: 2D numpy array representing the distances between cities
+        :return: Boolean indicating whether the tour is complete and matches the greedy distance
+        """
+        # convert distance_matrix to numpy array
+        distance_matrix = np.array(distance_matrix)
+
+        # Convert the tour string to a list of integers
+        # print(llm_string)
+        final_answer_element, reasoning_element = self.parse_xml_to_dict(llm_string)
+        # convert solution to dictionary
+        if final_answer_element == '':
+            return False, ''
+        elif final_answer_element is None:
+            return False, ''
+        else:
+            if isinstance(final_answer_element, str):
+                try:
+                    tour_string = ast.literal_eval(final_answer_element)['Path']
+                    if tour_string is None:
+                        return False, ''
+                except Exception:
+                    try:
+                        tour_string = ast.literal_eval('{' + final_answer_element + '}')['Path']
+                        if tour_string is None:
+                            return False, ''
+                    except Exception:
+                        return False, ''
+            else:
+                try:
+                    tour_string = ast.literal_eval(final_answer_element.text)['Path']
+                    if tour_string is None:
+                        return False, ''
+                except Exception:
+                    return False, ''
+        try:
+            tour = list(map(int, tour_string.split('->')))
+        except Exception:
+            return False, ''
+        # we could also prinpt `reasoning_element` to see the reasoning of the answer
+        # we could also print the final distance of the tour by `final_answer_element['Distance']`
+
+        # Check if tour is a cycle
+        if tour[0] != tour[-1]:
+            return False, 'The tour must start and end at the same city.'
+
+        # Check if all cities are visited
+        if len(tour) != len(distance_matrix) + 1:
+            return False, 'The tour does not visit all cities exactly once.'
+
+        # Calculate the distance of the provided tour
+        tour_distance = sum(distance_matrix[tour[i]][tour[i + 1]]
+                            for i in range(len(tour) - 1))
+
+        # Find the greedy tour distance for comparison
+        greedy_tour, greedy_distance = self.greedy_tsp(distance_matrix)
+
+        # Check if the provided tour distance is equal to the greedy tour distance
+        if tour_distance != greedy_distance:
+            return False, f'The tour distance ({tour_distance}) does not match the greedy solution ({greedy_distance}).'
+
+        return True, 'The solution is complete and matches the greedy solution distance.'
+
+    def greedy_tsp(self, distance_matrix):
+        """Solve the Traveling Salesman Problem using a greedy algorithm.
+
+        :param distance_matrix: 2D numpy array where the element at [i, j] is the distance between city i and j
+        :return: A tuple containing a list of the cities in the order they were visited and the total distance
+        """
+        num_cities = distance_matrix.shape[0]
+        unvisited_cities = set(range(num_cities))
+        current_city = np.random.choice(list(unvisited_cities))
+        tour = [current_city]
+        total_distance = 0
+
+        while unvisited_cities:
+            unvisited_cities.remove(current_city)
+            if unvisited_cities:
+                # Find the nearest unvisited city
+                distances_to_unvisited = distance_matrix[current_city][list(unvisited_cities)]
+                nearest_city = list(unvisited_cities)[np.argmin(distances_to_unvisited)]
+                tour.append(nearest_city)
+                # Update the total distance
+                total_distance += distance_matrix[current_city, nearest_city]
+                current_city = nearest_city
+
+        # Return to start
+        total_distance += distance_matrix[current_city, tour[0]]
+        tour.append(tour[0])
+
+        return tour, total_distance

opencompass-my-api/build/lib/opencompass/datasets/NPHardEval/p_BSP.py

@@ -0,0 +1,124 @@
+import ast
+import json
+
+from datasets import Dataset
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
+
+from ..base import BaseDataset
+from .prompts import bspPrompts
+
+
+def q2text(q, p=bspPrompts):
+    target_value = q['target']
+    # TO-DO: fix data not being sorted
+    array = sorted(q['array'])
+    prompt_text = p['Intro'] + '\n' + \
+                  p['Initial_question'].format(target_value=target_value) + '\n' + \
+                  p['Output_content'] + '\n' + \
+                  p['Output_format'] + \
+                  '\n The sorted array elements are: ' + ', '.join(map(str, array)) + '\n'
+
+    return prompt_text
+
+
+@LOAD_DATASET.register_module(force=True)
+class p_BSP_Dataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        raw_data = []
+        data_path = path
+        all_data, newdata = [], []
+        with open(data_path + 'bsp_instances.json', 'r') as f:
+            data = json.load(f)
+            for sample in data:
+                level = len(sample['array']) - 2
+                all_data.append((level, sample))
+
+        for level, q in all_data:
+            prompt = q2text(q)
+            raw_data.append({
+                'prompt': prompt,
+                'q': str(level) + '####\n' + json.dumps(q),
+                'level': level
+            })
+        dataset = Dataset.from_list(raw_data)
+        return dataset
+
+
+@ICL_EVALUATORS.register_module(force=True)
+class p_BSP_Evaluator(BaseEvaluator):
+
+    def score(self, predictions, references):
+        assert len(predictions) == len(references)
+
+        result = {'pass': 0, 'fail': 0}
+        for index, (q, output) in enumerate(zip(references, predictions)):
+            output_dict = {}
+            level = int(q.split('####\n')[0])
+            q = json.loads(q.split('####\n')[-1])
+            output, reasoning = self.parse_xml_to_dict(output)
+            output_dict['output'] = output
+            try:
+                output_dict['correctness'], _ = self.bsp_check(q, output)
+            except Exception as e:
+                print(f'Check failed: {e}')
+                output_dict['correctness'] = False
+            output_dict['reasoning'] = reasoning
+            output_dict['level'] = level
+
+            if output_dict['correctness']:
+                r = 'pass'
+            else:
+                r = 'fail'
+            result[r] += level
+
+        result['score'] = result['pass'] / (result['pass'] + result['fail']) * 100
+        final_result = {'Weighted Accuracy': result['score']}
+        return final_result
+
+    def parse_xml_to_dict(self, xml_string):
+        try:
+            assert '<final_answer>' in xml_string
+            assert '</final_answer>' in xml_string
+            assert '<reasoning>' in xml_string
+            assert '</reasoning>' in xml_string
+            final_answer_start = xml_string.index('<final_answer>') + len('<final_answer>')
+            final_answer_end = xml_string.index('</final_answer>')
+            reasoning_start = xml_string.index('<reasoning>') + len('<reasoning>')
+            reasoning_end = xml_string.index('</reasoning>')
+            final_answer_element = xml_string[final_answer_start:final_answer_end].rstrip().strip().rstrip()
+            reasoning_element = xml_string[reasoning_start:reasoning_end].rstrip().strip().rstrip()
+            try:
+                final_answer_element = ast.literal_eval(final_answer_element)
+            except Exception:
+                final_answer_element = ''
+        except Exception:
+            final_answer_element = ''
+            reasoning_element = ''
+
+        return final_answer_element, reasoning_element
+
+    def bsp_check(self, instance, solution):
+        """Check if the binary search solution is valid.
+
+        :param instance: The instance dictionary with array and target value.
+        :param solution: The solution dictionary with the position of the target value.
+        :return: A tuple of (is_correct, message).
+        """
+        array = sorted(instance['array'])
+        target_value = instance['target']
+        solution, reasoning = self.parse_xml_to_dict(solution)
+        if isinstance(solution, str):
+            return False, f'The solution is invalid.'
+        try:
+            position = int(solution['Position'])
+        except Exception:
+            return False, f'The solution is invalid.'
+        if position == -1 or position >= len(array):
+            return False, f'The solution is invalid.'
+        elif array[position] != target_value:
+            return False, f'The target index is incorrect.'
+        return True, 'The solution is valid.'

opencompass-my-api/build/lib/opencompass/datasets/NPHardEval/p_EDP.py

@@ -0,0 +1,145 @@
+import ast
+import json
+
+from datasets import Dataset
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
+
+from ..base import BaseDataset
+from .prompts import edpPrompts
+
+
+def q2text(q, p=edpPrompts):
+    string_a = q['string_a']
+    string_b = q['string_b']
+    prompt_text = p['Intro'] + '\n' + \
+                  p['Initial_question'].format(string_a=string_a, string_b=string_b) + '\n' + \
+                  p['Output_content'] + '\n' + \
+                  p['Output_format']
+    return prompt_text
+
+
+@LOAD_DATASET.register_module(force=True)
+class p_EDP_Dataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        raw_data = []
+        data_path = path
+        all_data = []
+        with open(data_path + 'edp_instances.json', 'r') as f:
+            data = json.load(f)
+            for sample in data:
+                level = len(sample['string_a']) - 2
+                all_data.append((level, sample))
+
+        for level, q in all_data:
+            prompt = q2text(q)
+            raw_data.append({
+                'prompt': prompt,
+                'q': str(level) + '####\n' + json.dumps(q),
+                'level': level
+            })
+        dataset = Dataset.from_list(raw_data)
+        return dataset
+
+
+@ICL_EVALUATORS.register_module(force=True)
+class p_EDP_Evaluator(BaseEvaluator):
+
+    def score(self, predictions, references):
+        assert len(predictions) == len(references)
+
+        result = {'pass': 0, 'fail': 0}
+        for index, (q, output) in enumerate(zip(references, predictions)):
+            output_dict = {}
+            level = int(q.split('####\n')[0])
+            q = json.loads(q.split('####\n')[-1])
+            output, reasoning = self.parse_xml_to_dict(output)
+            output_dict['output'] = output
+            try:
+                output_dict['correctness'], _ = self.edp_check(q, output)
+            except Exception as e:
+                print(f'Check failed: {e}')
+                output_dict['correctness'] = False
+            output_dict['reasoning'] = reasoning
+            output_dict['level'] = level
+
+            if output_dict['correctness']:
+                r = 'pass'
+            else:
+                r = 'fail'
+            result[r] += level
+
+        result['score'] = result['pass'] / (result['pass'] + result['fail']) * 100
+        final_result = {'Weighted Accuracy': result['score']}
+        return final_result
+
+    def compute_min_edit_distance(self, string_a, string_b):
+        """Computes the minimum edit distance between two strings using dynamic
+        programming."""
+        m, n = len(string_a), len(string_b)
+        dp = [[0] * (n + 1) for _ in range(m + 1)]
+
+        for i in range(m + 1):
+            for j in range(n + 1):
+                if i == 0:
+                    dp[i][j] = j
+                elif j == 0:
+                    dp[i][j] = i
+                elif string_a[i - 1] == string_b[j - 1]:
+                    dp[i][j] = dp[i - 1][j - 1]
+                else:
+                    dp[i][j] = 1 + min(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1])
+        return dp[m][n]
+
+    def edp_check(self, instance, solution):
+        """Check if the edit distance solution is valid.
+
+        :param instance: The instance dictionary with 'string_a' and 'string_b'.
+        :param solution: The solution dictionary with the reported 'edit_distance'.
+        :return: A tuple of (is_correct, message).
+        """
+        string_a = instance['string_a']
+        string_b = instance['string_b']
+        try:
+            reported_distance = int(solution.get('Operations', -1))
+        except Exception:
+            reported_distance = -1
+
+        actual_distance = self.compute_min_edit_distance(string_a, string_b)
+
+        if reported_distance == -1:
+            return False, 'No solution provided.'
+        elif reported_distance != actual_distance:
+            return False, f'The reported edit distance ({reported_distance}) is incorrect. Actual distance: {actual_distance}.'
+        return True, 'The solution is valid.'
+
+    def parse_xml_to_dict(self, xml_string):
+        try:
+            assert '<final_answer>' in xml_string
+            assert '</final_answer>' in xml_string
+            # assert '<reasoning>' in xml_string
+            # assert '</reasoning>' in xml_string
+            final_answer_start = xml_string.index('<final_answer>') + len('<final_answer>')
+            final_answer_end = xml_string.index('</final_answer>')
+            # reasoning_start = xml_string.index('<reasoning>') + len('<reasoning>')
+            # reasoning_end = xml_string.index('</reasoning>')
+            final_answer_element = xml_string[final_answer_start:final_answer_end].rstrip().strip().rstrip()
+            assert '{' in final_answer_element
+            assert '}' in final_answer_element
+            dic_start = final_answer_element.index('{')
+            dic_end = final_answer_element.index('}')
+            final_answer_element = final_answer_element[dic_start:dic_end + 1].rstrip().strip().rstrip()
+            reasoning_element = xml_string
+            # reasoning_element = xml_string[reasoning_start:reasoning_end].rstrip().strip().rstrip()
+            try:
+                final_answer_element = ast.literal_eval(final_answer_element)
+            except Exception:
+                final_answer_element = ''
+        except Exception:
+            final_answer_element = ''
+            reasoning_element = ''
+
+        return final_answer_element, reasoning_element

opencompass-my-api/build/lib/opencompass/datasets/NPHardEval/p_SPP.py

@@ -0,0 +1,196 @@
+import ast
+import json
+
+import networkx as nx
+from datasets import Dataset
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
+
+from ..base import BaseDataset
+from .prompts import sppPrompts
+
+
+def q2text(q, p=sppPrompts):
+    # start_node = q['start_node']
+    # end_node = q['end_node']
+    # TO-DO: fix later
+    start_node = q['nodes'][0]
+    end_node = q['nodes'][-1]
+    edges = q['edges']
+    prompt_text = p['Intro'] + '\n' + \
+                  p['Initial_question'].format(start_node=start_node, end_node=end_node) + '\n' + \
+                  p['Output_content'] + '\n' + \
+                  p['Output_format'] + \
+                  "\n The graph's edges and weights are as follows: \n"
+    for edge in edges:
+        this_line = f"Edge from {edge['from']} to {edge['to']} has a weight of {edge['weight']}."
+        prompt_text += this_line + '\n'
+    return prompt_text
+
+
+@LOAD_DATASET.register_module(force=True)
+class p_SPP_Dataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        raw_data = []
+        data_path = path
+        all_data = []
+        with open(data_path + 'spp_instances.json', 'r') as f:
+            data = json.load(f)
+            all_data = zip([int(d['complexity_level']) for d in data], data)
+        for level, q in all_data:
+            prompt = q2text(q)
+            raw_data.append({
+                'prompt': prompt,
+                'q': str(level) + '####\n' + json.dumps(q),
+                'level': level
+            })
+        dataset = Dataset.from_list(raw_data)
+        return dataset
+
+
+@ICL_EVALUATORS.register_module(force=True)
+class p_SPP_Evaluator(BaseEvaluator):
+
+    def score(self, predictions, references):
+        assert len(predictions) == len(references)
+
+        result = {'pass': 0, 'fail': 0}
+        for index, (q, output) in enumerate(zip(references, predictions)):
+            output_dict = {}
+            level = int(q.split('####\n')[0])
+            q = json.loads(q.split('####\n')[-1])
+            output, reasoning = self.parse_xml_to_dict(output)
+            output_dict['output'] = output
+            try:
+                output_dict['correctness'], _ = self.spp_check(q, output)
+            except Exception as e:
+                print(f'Check failed: {e}')
+                output_dict['correctness'] = False
+            output_dict['reasoning'] = reasoning
+            output_dict['level'] = level
+
+            if output_dict['correctness']:
+                r = 'pass'
+            else:
+                r = 'fail'
+            result[r] += level
+
+        result['score'] = result['pass'] / (result['pass'] + result['fail']) * 100
+        final_result = {'Weighted Accuracy': result['score']}
+        return final_result
+
+    def parse_xml_to_dict(self, xml_string):
+        try:
+            assert '<final_answer>' in xml_string
+            assert '</final_answer>' in xml_string
+            # assert '<reasoning>' in xml_string
+            # assert '</reasoning>' in xml_string
+            final_answer_start = xml_string.index('<final_answer>') + len('<final_answer>')
+            final_answer_end = xml_string.index('</final_answer>')
+            # reasoning_start = xml_string.index('<reasoning>') + len('<reasoning>')
+            # reasoning_end = xml_string.index('</reasoning>')
+            final_answer_element = xml_string[final_answer_start:final_answer_end].rstrip().strip().rstrip()
+            assert '{' in final_answer_element
+            assert '}' in final_answer_element
+            dic_start = final_answer_element.index('{')
+            dic_end = final_answer_element.index('}')
+            final_answer_element = final_answer_element[dic_start:dic_end + 1].rstrip().strip().rstrip()
+            # reasoning_element = xml_string[reasoning_start:reasoning_end].rstrip().strip().rstrip()
+            try:
+                final_answer_element = ast.literal_eval(final_answer_element)
+                reasoning_element = xml_string
+            except Exception:
+                final_answer_element = ''
+                reasoning_element = xml_string
+        except Exception:
+            final_answer_element = ''
+            reasoning_element = ''
+
+        return final_answer_element, reasoning_element
+
+    def ssp_optimal_solution(self, instance, source, target):
+        """Provides the optimal solution for the SSP instance.
+
+        :param instance: The SSP instance as a dictionary with 'nodes' and 'edges'.
+        :param source: The source node.
+        :param target: The destination node.
+        :return: The optimal shortest path length and path.
+        """
+        G = nx.Graph()
+        G.add_nodes_from(instance['nodes'])
+        G.add_weighted_edges_from([(edge['from'], edge['to'], edge['weight'])
+                                   for edge in instance['edges']])
+        shortest_path_length = None
+        shortest_path = None
+        if nx.has_path(G, source=source, target=target):
+            shortest_path_length = nx.shortest_path_length(G, source=source, target=target, weight='weight')
+            shortest_path = nx.shortest_path(G, source=source, target=target, weight='weight')
+        return shortest_path_length, shortest_path
+
+    # SPP
+    def spp_check(self, instance, solution, start_node=None, end_node=None):
+        """Validate the solution of the SPP problem.
+
+        :param instance: The instance dictionary with nodes and edges.
+        :param solution: The solution dictionary with the path and total distance.
+        :param start_node: The start node.
+        :param end_node: The end node.
+        :return: A tuple of (is_correct, message).
+        """
+        # Get the start and end nodes
+        # Currently, the start and end nodes are the first and last nodes in the instance
+        if start_node is None:
+            start_node = instance['nodes'][0]
+        if end_node is None:
+            end_node = instance['nodes'][-1]
+
+        # Convert solution to dictionary
+        try:
+            path_string = solution.get('Path', '')
+            cost_string = solution.get('TotalDistance', '')
+        except Exception:
+            return False, 'The solution is not a dictionary.'
+
+        # Calculate the optimal solution
+        ssp_optimal_length, ssp_optimal_path = self.ssp_optimal_solution(
+            instance, start_node, end_node)
+        if ssp_optimal_length is None:
+            if isinstance(cost_string, int) or cost_string.isdigit():
+                return False, f'No path between from node {start_node} to node {end_node}.'
+            else:
+                return True, 'No path found from node {start_node} to node {end_node}.'
+
+        try:
+            path = list(map(int, path_string.split('->')))
+            total_cost = int(cost_string)
+        except Exception:
+            return False, 'The solution is not a valid dictionary.'
+
+        # Check if path starts and ends with the correct nodes
+        if not path or path[0] != start_node or path[-1] != end_node:
+            return False, 'The path does not start or end at the correct nodes.'
+
+        # Check if the path is continuous and calculate the cost
+        calculated_cost = 0
+        is_in_edge = lambda edge, from_node, to_node: (edge['from'] == from_node and edge['to'] == to_node) or (edge['from'] == to_node and edge['to'] == from_node)
+        for i in range(len(path) - 1):
+            from_node, to_node = path[i], path[i + 1]
+            edge = next((edge for edge in instance['edges'] if is_in_edge(edge, from_node, to_node)), None)
+
+            if not edge:
+                return False, f'No edge found from node {from_node} to node {to_node}.'
+
+            calculated_cost += edge['weight']
+
+        # Check if the calculated cost matches the total cost provided in the solution
+        if calculated_cost != total_cost:
+            return False, f'The calculated cost ({calculated_cost}) does not match the provided total cost ({total_cost}).'
+
+        if calculated_cost != ssp_optimal_length:
+            # spp_optimal_path = '->'.join(map(str, ssp_optimal_path))
+            return False, f'The calculated cost ({calculated_cost}) does not match the optimal solution ({ssp_optimal_length}): {ssp_optimal_path}.'
+
+        return True, 'The solution is valid.'

opencompass-my-api/build/lib/opencompass/datasets/NPHardEval/prompts.py

@@ -0,0 +1,96 @@
+# Overall fewshot prompts
+FEW_SHOT_SELF = 'Please refer to a few examples of this problem and the corresponding reasoning process. The examples are:'
+FEW_SHOT_OTHERS = 'Please refer to a few examples of another problem and the corresponding reasoning process. The problem is {initial_question}. {output_content}. The examples are:'
+
+# P problems
+sppPrompts = {
+    'Intro': 'The Shortest Path Problem (SPP) involves finding the shortest path between two nodes in a weighted graph.',
+    'Initial_question': "You need to find the shortest path between node {start_node} and node {end_node} in a graph. The graph's edges and their weights are given.",
+    'Output_content': 'Please provide the shortest path from {start_node} to {end_node} and its total distance. Offer a concise step-by-step explanation of your reasoning process. Aim for brevity and clarity in your response.',
+    'Output_format': "Your output should be enclosed within <root></root> tags. Include your reasoning in <reasoning></reasoning> tags and the final path and total distance in <final_answer></final_answer> tags, like <final_answer>{'Path': 'START->...->END', 'TotalDistance': 'INT_TOTAL_DISTANCE'}</final_answer>.",
+    'Few_shot_self': FEW_SHOT_SELF,
+    'Few_shot_others': FEW_SHOT_OTHERS
+}
+
+mfpPrompts = {
+    'Intro': 'The Maximum Flow Problem (MFP) seeks to find the maximum possible flow from a source node to a sink node in a flow network, subject to capacity constraints on the edges.',
+    'Initial_question': 'Determine the maximum flow from the source node {source_node} to the sink node {sink_node} in the given flow network. The capacities of the edges are provided.',
+    'Output_content': 'Please indicate the maximum flow value and the flow for each edge. Provide a brief explanation of your methodology. Keep your response concise and focused.',
+    'Output_format': "Enclose your output within <root></root> tags. Present your reasoning in <reasoning></reasoning> tags and the final maximum flow and edge flows in <final_answer></final_answer> tags, like <final_answer>{'MaxFlow': 'MAX_FLOW_VALUE', 'Flows': {'NODE_1->NODE_2': FLOW, ...}}</final_answer>.",
+    'Few_shot_self': FEW_SHOT_SELF,
+    'Few_shot_others': FEW_SHOT_OTHERS
+}
+
+bspPrompts = {
+    'Intro': 'The Binary Search Problem (BSP) deals with finding the position of a target value within a sorted array using a binary search algorithm, which efficiently narrows down the search range.',
+    'Initial_question': 'Find the position of the target value {target_value} in the sorted array. The index begins with 0. The array elements are provided.',
+    'Output_content': 'Please identify the position of the target value in the array. Offer a brief, step-by-step account of your search process. Aim for conciseness in your response.',
+    'Output_format': "Your output should be enclosed in <root></root> tags. Include your search process in <reasoning></reasoning> tags and the final position of the target value in <final_answer></final_answer> tags, like <final_answer>{'Position': 'TARGET_POSITION'}</final_answer>.",
+    'Few_shot_self': FEW_SHOT_SELF,
+    'Few_shot_others': FEW_SHOT_OTHERS
+}
+
+edpPrompts = {
+    'Intro': 'The Edit Distance Problem (EDP) involves finding the minimum number of operations required to transform one string into another, where each operation is either an insertion, deletion, or substitution of a single character.',
+    'Initial_question': 'Find the minimum number of operations required to transform the first string {string_a} into the second string {string_b}. The operations are insertion, deletion, and substitution of a single character, each requiring 1 edit operation.',
+    'Output_content': 'Please provide the minimum number of operations required to transform the first string into the second string. Offer a brief explanation of your methodology. Keep your response concise and focused.',
+    'Output_format': "Enclose your output within <root></root> tags. Present your reasoning in <reasoning></reasoning> tags and the final minimum number of operations in <final_answer></final_answer> tags, like <final_answer>{'Operations': 'MINIMUM_NUMBER_OF_OPERATIONS'}</final_answer>.",
+    'Few_shot_self': FEW_SHOT_SELF,
+    'Few_shot_others': FEW_SHOT_OTHERS
+}
+
+# NP-complete problems
+tsp_dPrompts = {
+    'Intro': 'The Traveling Salesman Problem (Decision Version, TSP-D) focuses on determining if a salesman can complete a route, visiting each city at least once, with the total travel distance being less than a specified value.',
+    'Initial_question': "Check if it's possible for a salesman to visit each of the {total_cities} cities at least once and return to the starting city with the total distance less than {distance_limit}. The distances between each pair of cities are given.",
+    'Output_content': 'Provide a yes or no answer, with a succinct explanation of your decision process. Focus on clarity and brevity in your response.',
+    'Output_format': "Enclose your output in <root></root> tags. Present your reasoning in <reasoning></reasoning> tags and the final yes/no answer in <final_answer></final_answer> tags, like <final_answer>{'Feasible': 'YES_OR_NO'}</final_answer>.",
+    'Few_shot_self': FEW_SHOT_SELF,
+    'Few_shot_others': FEW_SHOT_OTHERS
+}
+
+gcp_dPrompts = {
+    'Intro': 'The Graph Coloring Problem (Decision Version, GCP-D) involves determining if it is possible to color the vertices of a graph using a given number of colors, ensuring no two adjacent vertices have the same color.',
+    'Initial_question': 'Find out if the vertices of a graph with {total_vertices} vertices can be colored using only {number_of_colors} colors, such that no adjacent vertices share the same color.',
+    'Output_content': 'Provide a yes or no answer, along with a concise explanation of your reasoning. Keep your explanation focused and brief.',
+    'Output_format': "Enclose your output in <root></root> tags. Include your reasoning in <reasoning></reasoning> tags and the final yes/no answer in <final_answer></final_answer> tags, like <final_answer>{'Feasible': 'YES_OR_NO'}</final_answer>.",
+    'Few_shot_self': FEW_SHOT_SELF,
+    'Few_shot_others': FEW_SHOT_OTHERS
+}
+
+kspPrompts = {
+    'Intro': 'The 0-1 Knapsack Problem (KSP) asks whether a subset of items, each with a given weight and value, can be chosen to fit into a knapsack of fixed capacity, maximizing the total value without exceeding the capacity.',
+    'Initial_question': 'Determine if a subset of items can be selected to fit into a knapsack with a capacity of {knapsack_capacity}, maximizing value without exceeding the capacity. Item weights and values are provided.',
+    'Output_content': 'Indicate if an optimal subset exists and its total value. Offer a concise explanation of your selection process. Aim for clarity and brevity in your response.',
+    'Output_format': "Your output should be enclosed within <root></root> tags. Include your selection process in <reasoning></reasoning> tags and the final decision and total value in <final_answer></final_answer> tags, like <final_answer>{'Feasible': 'YES_OR_NO', 'TotalValue': 'TOTAL_VALUE', 'SelectedItemIds': [0, 1]}</final_answer>.",
+    'Few_shot_self': FEW_SHOT_SELF,
+    'Few_shot_others': FEW_SHOT_OTHERS
+}
+
+# NP-hard problems
+tspPrompts = {
+    'Intro': 'The traveling salesman problem (TSP) is a classic optimization problem that aims to find the shortest possible route that visits a set of cities, with each city being visited exactly once and the route returning to the original city.',
+    'Initial_question': 'You must find the shortest path that visits all {total_cities} cities, labelled from 1 to {total_cities}. The distances between each pair of cities are provided.',
+    'Output_content': 'Please list each city in the order they are visited. Provide the total distance of the trip. You should also provide very short step by step reasoning. Do not use multiple lines and try your best to save output tokens.',
+    'Output_format': "Your output should contain two parts enclosed by <root></root>. First, your step by step reasoning like <reasoning>The reasoning process</reasoning>. Second, the final output of the result path and total distance wrapped by final_answer tag, like <final_answer>{'Path': '0->1->2->...->N->0', 'TotalDistance': 'INT_TOTAL_DISTANCE'}</final_answer>",
+    'Few_shot_self': FEW_SHOT_SELF,
+    'Few_shot_others': FEW_SHOT_OTHERS
+}
+
+gcpPrompts = {
+    'Intro': 'Graph coloring refers to the problem of coloring vertices of a graph in such a way that no two adjacent vertices have the same color. ',
+    'Initial_question': 'There are {max_vertices} vertices 1 to {max_vertices} in a graph. You may use {max_colors} colors with alphabats from A, B, C,... to color the graph.',
+    'Output_content': "Please label every vertex, even if it is disconnected from the rest of the graph. Please provide each vertex's color. Do not skip any vertices. You should also provide very short step by step reasoning. Do not use multiple lines and try your best to save output tokens.",
+    'Output_format': "Your output should contain two parts enclosed by <root></root>. First, your step by step reasoning wrapped by <reasoning></reasoning>. Second, the final output of all vertex numbers and their associated colors, wrapped by final_answer tag, like <final_answer>{0:'COLOR_1', 1:'COLOR_2', ...}</final_answer>.",
+    'Few_shot_self': FEW_SHOT_SELF,
+    'Few_shot_others': FEW_SHOT_OTHERS
+}
+
+mspPrompts = {
+    'Intro': 'The meeting scheduling problem (MSP) is a type of constraint satisfaction problem where the goal is to find a suitable time slot for a meeting that all participants can attend without conflicts in their schedules.',
+    'Initial_question': "There are {total_participants} participants with their available time slots. There are {total_timeslots} consecutive non-overlapping time slots. Let's assume all meetings has duration of 1.",
+    'Output_content': 'Please provide a time slot where all participants can attend the meeting. You should also provide very short step by step reasoning. Do not use multiple lines and try your best to save output tokens.',
+    'Output_format': 'Your output should contain two parts enclosed by <root></root>. First, your step by step reasoning wrapped by <reasoning></reasoning>. Second, the final output of meeting numbers followed by a list of slots, like <final_answer>{0:[1,2], 1:[4], ...}</final_answer>.',
+    'Few_shot_self': FEW_SHOT_SELF,
+    'Few_shot_others': FEW_SHOT_OTHERS
+}

opencompass-my-api/build/lib/opencompass/datasets/NPHardEval/utils.py

@@ -0,0 +1,43 @@
+import ast
+import xml.etree.ElementTree as ET
+
+
+def append_root_tags(string):
+    if not string.strip().startswith('<root>'):
+        string = '<root>\n' + string
+    if not string.strip().endswith('</root>'):
+        string += '\n</root>'
+    return string
+
+
+def parse_xml_to_dict(xml_string):
+    final_answer_element = ''
+    reasoning_element = ''
+
+    try:
+        # Parse the XML string
+        root = ET.fromstring(xml_string)
+
+        # Find the 'final_answer' tag
+        final_answer_element = root.find('final_answer').text
+
+        # Find the 'reasoning' tag
+        reasoning_element = root.find('reasoning').text
+    except Exception:
+        try:
+            assert '<final_answer>' in xml_string
+            assert '</final_answer>' in xml_string
+            assert '<reasoning>' in xml_string
+            assert '</reasoning>' in xml_string
+            final_answer_start = xml_string.index('<final_answer>') + len('<final_answer>')
+            final_answer_end = xml_string.index('</final_answer>')
+            reasoning_start = xml_string.index('<reasoning>') + len('<reasoning>')
+            reasoning_end = xml_string.index('</reasoning>')
+            final_answer_element = xml_string[final_answer_start:final_answer_end]
+            reasoning_element = xml_string[reasoning_start:reasoning_end]
+        except Exception:
+            final_answer_element = ''
+            reasoning_element = ''
+
+    final_answer_element = ast.literal_eval(final_answer_element.strip())
+    return final_answer_element, reasoning_element

opencompass-my-api/build/lib/opencompass/datasets/OpenFinData.py

@@ -0,0 +1,47 @@
+import json
+import os.path as osp
+
+from datasets import Dataset
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class OpenFinDataDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str, name: str):
+        with open(osp.join(path, f'{name}.json'), 'r') as f:
+            data = json.load(f)
+            return Dataset.from_list(data)
+
+
+@ICL_EVALUATORS.register_module()
+class OpenFinDataKWEvaluator(BaseEvaluator):
+
+    def __init__(self, ):
+        super().__init__()
+
+    def score(self, predictions, references):
+        assert len(predictions) == len(references)
+
+        scores = []
+        results = dict()
+
+        for i in range(len(references)):
+            all_hit = True
+            judgement = references[i].split('、')
+            for item in judgement:
+                if item not in predictions[i]:
+                    all_hit = False
+                    break
+            if all_hit:
+                scores.append(True)
+            else:
+                scores.append(False)
+
+        results['accuracy'] = round(sum(scores) / len(scores), 4) * 100
+        return results

opencompass-my-api/build/lib/opencompass/datasets/TheoremQA.py

@@ -0,0 +1,38 @@
+import re
+
+from datasets import load_dataset
+
+from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class TheoremQADataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        return load_dataset('csv', data_files={'test': path})
+
+
+@TEXT_POSTPROCESSORS.register_module('TheoremQA')
+def TheoremQA_postprocess(text: str) -> str:
+    text = text.strip()
+    matches = re.findall(r'answer is ([^\s]+)', text)
+    if len(matches) == 0:
+        return text
+    else:
+        text = matches[0].strip().strip('.,?!\"\';:')
+        return text
+
+
+def TheoremQA_postprocess_v2(text: str) -> str:
+    prediction = text.strip().strip('\n').split('\n')[-1]
+    tmp = ''
+    for entry in prediction.split(' ')[::-1]:
+        if entry == 'is' or entry == 'be' or entry == 'are' or entry.endswith(
+                ':'):
+            break
+        tmp = entry + ' ' + tmp
+    prediction = tmp.strip().strip('.')
+    return prediction

opencompass-my-api/build/lib/opencompass/datasets/advglue.py

@@ -0,0 +1,174 @@
+import json
+from typing import List, Union
+
+from datasets import Dataset, concatenate_datasets
+
+from opencompass.openicl.icl_evaluator import AccEvaluator
+
+from .base import BaseDataset
+
+
+class AdvDataset(BaseDataset):
+    """Base adv GLUE dataset. Adv GLUE is built on GLUE dataset. The main
+    purpose is to eval the accuracy drop on original set and adv set.
+
+    Args:
+        subset (str): The subset task of adv GLUE dataset.
+        filter_keys (str): The keys to be filtered to create the original
+            set for comparison.
+    """
+
+    def __init__(
+        self,
+        subset: str,
+        filter_keys: Union[str, List[str]],
+        **kwargs,
+    ):
+        self.subset = subset
+        if isinstance(filter_keys, str):
+            filter_keys = [filter_keys]
+        self.filter_keys = filter_keys
+        super().__init__(**kwargs)
+
+    def aug_with_original_data(self, dataset):
+        """Create original dataset and concat to the end."""
+        # Remove data without original reference
+        dataset = dataset.filter(
+            lambda x: any([x[k] for k in self.filter_keys]))
+
+        def ori_preprocess(example):
+            for k in self.filter_keys:
+                if example[k]:
+                    new_k = k.split('original_')[-1]
+                    example[new_k] = example[k]
+                    example['type'] = 'original'
+            return example
+
+        original_dataset = dataset.map(ori_preprocess)
+
+        return concatenate_datasets([dataset, original_dataset])
+
+    def load(self, path):
+        """Load dataset and aug with original dataset."""
+
+        with open(path, 'r') as f:
+            raw_data = json.load(f)
+            subset = raw_data[self.subset]
+
+            # In case the missing keys in first example causes Dataset
+            # to ignore them in the following examples when building.
+            for k in self.filter_keys:
+                if k not in subset[0]:
+                    subset[0][k] = None
+
+            dataset = Dataset.from_list(raw_data[self.subset])
+
+        dataset = self.aug_with_original_data(dataset)
+
+        def choices_process(example):
+            example['label_option'] = chr(ord('A') + example['label'])
+            return example
+
+        dataset = dataset.map(choices_process)
+        return dataset
+
+
+# label 0 for A. negative
+# label 1 for B. positive
+class AdvSst2Dataset(AdvDataset):
+    """Adv GLUE sst2 dataset."""
+
+    def __init__(self, **kwargs):
+        super().__init__(subset='sst2',
+                         filter_keys='original_sentence',
+                         **kwargs)
+
+
+# label 0 for not_duplicate, A. no
+# label 1 for duplicate, B. yes
+class AdvQqpDataset(AdvDataset):
+    """Adv GLUE qqp dataset."""
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            subset='qqp',
+            filter_keys=['original_question1', 'original_question2'],
+            **kwargs)
+
+
+# # label 0 for entailment, A. yes
+# # label 1 for neutral, B. maybe
+# # label 2 for contradiction, C. no
+class AdvMnliDataset(AdvDataset):
+    """Adv GLUE mnli dataset."""
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            subset='mnli',
+            filter_keys=['original_premise', 'original_hypothesis'],
+            **kwargs)
+
+
+# # label 0 for entailment, A. yes
+# # label 1 for neutral, B. maybe
+# # label 2 for contradiction, C. no
+class AdvMnliMMDataset(AdvDataset):
+    """Adv GLUE mnli mm dataset."""
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            subset='mnli-mm',
+            filter_keys=['original_premise', 'original_hypothesis'],
+            **kwargs)
+
+
+# # label 0 for entailment, A. yes
+# # label 1 for not entailment, B. no
+class AdvQnliDataset(AdvDataset):
+    """Adv GLUE qnli dataset."""
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            subset='qnli',
+            filter_keys=['original_question', 'original_sentence'],
+            **kwargs)
+
+
+# # label 0 for entailment, A. yes
+# # label 1 for not entailment, B. no
+class AdvRteDataset(AdvDataset):
+    """Adv GLUE rte dataset."""
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            subset='rte',
+            filter_keys=['original_sentence1', 'original_sentence2'],
+            **kwargs)
+
+
+class AccDropEvaluator(AccEvaluator):
+    """Eval accuracy drop."""
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    def score(self, predictions: List, references: List) -> dict:
+        """Calculate scores and accuracy.
+
+        Args:
+            predictions (List): List of probabilities for each class of each
+                sample.
+            references (List): List of target labels for each sample.
+
+        Returns:
+            dict: calculated scores.
+        """
+
+        n = len(predictions)
+        assert n % 2 == 0, 'Number of examples should be even.'
+        acc_after = super().score(predictions[:n // 2], references[:n // 2])
+        acc_before = super().score(predictions[n // 2:], references[n // 2:])
+        acc_drop = 1 - acc_after['accuracy'] / acc_before['accuracy']
+        return dict(acc_drop=acc_drop,
+                    acc_after=acc_after['accuracy'],
+                    acc_before=acc_before['accuracy'])

opencompass-my-api/build/lib/opencompass/datasets/afqmcd.py

@@ -0,0 +1,21 @@
+import json
+
+from datasets import Dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class AFQMCDataset_V2(BaseDataset):
+
+    @staticmethod
+    def load(path):
+        data = []
+        with open(path, 'r', encoding='utf-8') as f:
+            for line in f:
+                line = json.loads(line)
+                line['label'] = 'AB'[int(line['label'])]
+                data.append(line)
+        return Dataset.from_list(data)

opencompass-my-api/build/lib/opencompass/datasets/agieval/__init__.py

@@ -0,0 +1,3 @@
+# flake8: noqa
+
+from .agieval import *  # noqa: F401, F403

opencompass-my-api/build/lib/opencompass/datasets/agieval/agieval.py

@@ -0,0 +1,99 @@
+import json
+import os.path as osp
+
+from datasets import Dataset
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
+
+from ..base import BaseDataset
+from .math_equivalence import is_equiv
+from .post_process import parse_math_answer
+
+
+@LOAD_DATASET.register_module()
+class AGIEvalDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str, name: str, setting_name: str):
+        from .dataset_loader import load_dataset, load_dataset_as_result_schema
+
+        assert setting_name in 'zero-shot', 'only support zero-shot setting'
+        dataset_wo_label = load_dataset(name, setting_name, path)
+        dataset_with_label = load_dataset_as_result_schema(name, path)
+        dataset = []
+        for d1, d2 in zip(dataset_wo_label, dataset_with_label):
+            dataset.append({
+                'id': d2.index,
+                'problem_input': d1['context'],
+                'label': d2.label,
+            })
+        dataset = Dataset.from_list(dataset)
+        return dataset
+
+
+@LOAD_DATASET.register_module()
+class AGIEvalDataset_v2(BaseDataset):
+
+    @staticmethod
+    def load(path: str, name: str, setting_name: str):
+        assert setting_name in 'zero-shot', 'only support zero-shot setting'
+        filename = osp.join(path, name + '.jsonl')
+        with open(filename, encoding='utf-8') as f:
+            data = [json.loads(line.strip()) for line in f]
+        dataset = []
+        for item in data:
+            passage = item['passage'] if item['passage'] else ''
+            question = passage + item['question']
+            options = '\n'.join(item['options']) if item['options'] else ''
+            if item['label']:
+                if isinstance(item['label'], list):
+                    label = ''.join(item['label'])
+                else:
+                    label = item['label']
+            else:
+                label = item['answer']
+            d = {'question': question, 'options': options, 'label': label}
+            dataset.append(d)
+        dataset = Dataset.from_list(dataset)
+        return dataset
+
+
+@ICL_EVALUATORS.register_module()
+class AGIEvalEvaluator(BaseEvaluator):
+
+    def score(self, predictions, references):
+        predictions = [parse_math_answer('', pred) for pred in predictions]
+        details = []
+        cnt = 0
+        for pred, ref in zip(predictions, references):
+            detail = {'pred': pred, 'answer': ref, 'correct': False}
+            if is_equiv(pred, ref):
+                cnt += 1
+                detail['correct'] = True
+            details.append(detail)
+        score = cnt / len(predictions) * 100
+        return {'score': score, 'details': details}
+
+
+@ICL_EVALUATORS.register_module()
+class AGIEvalEvaluator_mcq(BaseEvaluator):
+
+    def score(self, predictions, references):
+        if len(predictions) != len(references):
+            return {
+                'error': 'predictions and references have different '
+                'length'
+            }
+        details = []
+        cnt = 0
+        for pred, ref in zip(predictions, references):
+            detail = {'pred': pred, 'answer': ref, 'correct': False}
+            if pred == ref:
+                cnt += 1
+                detail['correct'] = True
+            details.append(detail)
+
+        score = cnt / len(predictions) * 100
+
+        return {'score': score, 'details': details}

opencompass-my-api/build/lib/opencompass/datasets/agieval/constructions.py

@@ -0,0 +1,104 @@
+# flake8: noqa
+import pandas as pd
+
+
+class TaskSchema(object):
+
+    def __init__(self,
+                 passage=None,
+                 question=None,
+                 options=None,
+                 label=None,
+                 answer=None,
+                 other=None):
+        self.passage = passage
+        self.question = question
+        self.options = options
+        self.label = label
+        self.answer = answer
+        self.other = other
+
+    def to_dict(self):
+        return {
+            'passage': self.passage,
+            'question': self.question,
+            'options': self.options,
+            'label': self.label,
+            'answer': self.answer,
+            'other': self.other
+        }
+
+
+# define README.json
+class AgiInstance(object):
+
+    def __init__(self, task_description, data_source, task_schema, output,
+                 evaluation_metric, task_example):
+        self.task_description = task_description
+        self.data_source = data_source
+        self.task_schema = task_schema
+        self.output = output
+        self.evaluation_metric = evaluation_metric
+        self.task_example = task_example
+
+    def to_dict(self):
+        return {
+            'task description': self.task_description,
+            'data source': self.data_source,
+            'task schema': self.task_schema.to_dict(),
+            'output': self.output,
+            'evaluation metric': self.evaluation_metric,
+            'task example': self.task_example
+        }
+
+
+class ChatGPTSchema(object):
+
+    def __init__(self, context=None, metadata=''):
+        self.context = context
+        self.metadata = metadata
+
+    def to_dict(self):
+        return {'context': self.context, 'metadata': self.metadata}
+
+
+class ResultsForHumanSchema(object):
+
+    def __init__(self,
+                 index,
+                 problem_input,
+                 label,
+                 model_input='',
+                 model_output='',
+                 parse_result='',
+                 first_stage_output='',
+                 second_stage_input='',
+                 is_correct=False):
+        self.index = index
+        self.problem_input = problem_input
+        self.model_input = model_input
+        self.model_output = model_output
+        self.parse_result = parse_result
+        self.label = label
+        self.first_stage_output = first_stage_output
+        self.second_stage_input = second_stage_input
+        self.is_correct = is_correct
+
+    def to_dict(self):
+        return {
+            'index': self.index,
+            'problem_input': self.problem_input,
+            'model_input': self.model_input,
+            'model_output': self.model_output,
+            'parse_result': self.parse_result,
+            'label': self.label,
+            'is_correct': self.is_correct,
+            'first_stage_output': self.first_stage_output,
+            'second_stage_input': self.second_stage_input,
+        }
+
+    @staticmethod
+    def to_tsv(result_list, path):
+        result_json = [item.to_dict() for item in result_list]
+        table = pd.json_normalize(result_json)
+        table.to_excel(path, index=False)

opencompass-my-api/build/lib/opencompass/datasets/agieval/dataset_loader.py

@@ -0,0 +1,392 @@
+# flake8: noqa
+import ast
+import json
+import os
+
+import pandas as pd
+import tiktoken
+from tqdm import tqdm
+
+from .constructions import ChatGPTSchema, ResultsForHumanSchema
+from .utils import extract_answer, read_jsonl, save_jsonl
+
+# define the datasets
+english_qa_datasets = [
+    'lsat-ar', 'lsat-lr', 'lsat-rc', 'logiqa-en', 'sat-math', 'sat-en',
+    'aqua-rat', 'sat-en-without-passage', 'gaokao-english'
+]
+chinese_qa_datasets = [
+    'logiqa-zh', 'jec-qa-kd', 'jec-qa-ca', 'gaokao-chinese',
+    'gaokao-geography', 'gaokao-history', 'gaokao-biology', 'gaokao-chemistry',
+    'gaokao-physics', 'gaokao-mathqa'
+]
+english_cloze_datasets = ['math']
+chinese_cloze_datasets = ['gaokao-mathcloze']
+
+multi_choice_datasets = ['jec-qa-kd', 'jec-qa-ca', 'gaokao-physics']
+math_output_datasets = ['gaokao-mathcloze', 'math']
+
+
+def convert_zero_shot(line, dataset_name):
+    try:
+        passage = line['passage'] if line['passage'] is not None else ''
+        if dataset_name in english_qa_datasets:
+            option_string = 'ABCDEFG'
+            count = len(line['options'])
+            if count == 1:
+                count = 5
+            return passage + 'Q: '  + line['question'] + ' ' \
+                + 'Answer Choices: ' + ' '.join(line['options']) + '\n' + \
+                'A: Among A through {}, the answer is'.format(option_string[count - 1])
+
+        elif dataset_name in chinese_qa_datasets:
+            option_string = 'ABCDEFG'
+            count = len(line['options'])
+            if count == 1:
+                count = 4
+            return passage + '问题：' + line['question'] + ' ' \
+                + '选项：' + ' '.join(line['options']) + '\n' + \
+                '答案：从A到{}, 我们应选择'.format(option_string[count - 1])
+
+        elif dataset_name in english_cloze_datasets:
+            return passage + 'Q: ' + line['question'] + '\n' \
+                                              'A: The answer is'
+
+        elif dataset_name in chinese_cloze_datasets:
+            return passage + '问题：' + line['question'] + '\n' \
+                                                '答案：'
+    except NameError:
+        print('Dataset not defined.')
+
+
+prefix = '该问题为单选题，所有选项中必有一个正确答案，且只有一个正确答案。\n'
+
+
+def convert_zero_shot_CoT_stage1(line, dataset_name):
+    try:
+        passage = line['passage'] if line['passage'] is not None else ''
+        if dataset_name in english_qa_datasets:
+            return passage + 'Q: ' + line['question'] + ' ' \
+                + 'Answer Choices: ' + ' '.join(line['options']) + '\n' + \
+                "Let's think step by step."
+
+        elif dataset_name in chinese_qa_datasets:
+            option_string = 'ABCDEFG'
+            count = len(line['options'])
+            if count == 1:
+                count = 4
+            return passage + '问题：' + line['question'] + ' ' \
+                + '选项：' + ' '.join(line['options']) + '\n' + \
+                '从A到{}, 我们应选择什么？让我们逐步思考：'.format(option_string[count - 1])
+
+        elif dataset_name in english_cloze_datasets:
+            return passage + 'Q: ' + line['question'] + '\n' \
+                                              "A: Let's think step by step."
+
+        elif dataset_name in chinese_cloze_datasets:
+            return passage + '问题：' + line['question'] + '\n' \
+                                                '答案：让我们逐步思考：'
+    except NameError:
+        print('Dataset not defined.')
+
+
+# process few-shot raw_prompts
+def combine_prompt(prompt_path,
+                   dataset_name,
+                   load_explanation=True,
+                   chat_mode=False):
+    skip_passage = False
+    if dataset_name == 'sat-en-without-passage':
+        skip_passage = True
+        dataset_name = 'sat-en'
+    demostrations = []
+    # read the prompts by context and explanation
+    context_row = [0, 1, 3, 5, 7, 9]
+    explanation_row = [0, 2, 4, 6, 8, 10]
+    raw_prompts_context = pd.read_csv(prompt_path,
+                                      header=0,
+                                      skiprows=lambda x: x not in context_row,
+                                      keep_default_na=False)
+    raw_prompts_explanation = pd.read_csv(
+        prompt_path,
+        header=0,
+        skiprows=lambda x: x not in explanation_row,
+        keep_default_na=False).replace(r'\n\n', '\n', regex=True)
+    contexts = []
+    for line in list(raw_prompts_context[dataset_name]):
+        if line:
+            # print(line)
+            contexts.append(ast.literal_eval(line))
+    explanations = [
+        exp for exp in raw_prompts_explanation[dataset_name] if exp
+    ]
+
+    for idx, (con, exp) in enumerate(zip(contexts, explanations)):
+        passage = con['passage'] if con[
+            'passage'] is not None and not skip_passage else ''
+        question = con['question']
+        options = con['options'] if con['options'] is not None else ''
+        label = con['label'] if con['label'] is not None else ''
+        answer = con[
+            'answer'] if 'answer' in con and con['answer'] is not None else ''
+
+        if dataset_name in english_qa_datasets:
+            question_input = 'Problem {}.   '.format(idx + 1) + passage + ' ' + question + '\n' \
+                              + 'Choose from the following options:    ' + ' '.join(options) + '\n'
+            question_output = (('Explanation for Problem {}:   '.format(idx + 1) + exp + '\n') if load_explanation else '') \
+                              + 'The answer is therefore {}'.format(label)
+
+        elif dataset_name in chinese_qa_datasets:
+            question_input = '问题 {}.   '.format(idx + 1) + passage + ' ' + question + '\n' \
+                              + '从以下选项中选择:    ' + ' '.join(options) + '\n'
+            question_output = (('问题 {}的解析:   '.format(idx + 1) + exp + '\n') if load_explanation else '') \
+                              + '答案是 {}'.format(label)
+
+        elif dataset_name in english_cloze_datasets:
+            question_input = 'Problem {}.   '.format(idx + 1) + question + '\n'
+            question_output = (('Explanation for Problem {}:   '.format(idx + 1) + exp + '\n') if load_explanation else '') \
+                              + 'The answer is therefore {}'.format(answer)
+
+        elif dataset_name in chinese_cloze_datasets:
+            question_input = '问题 {}.   '.format(idx + 1) + question + '\n'
+            question_output = (('问题 {}的解析:   '.format(idx + 1) + exp + '\n') if load_explanation else '') \
+                              + '答案是 {}'.format(answer)
+        else:
+            raise ValueError(
+                f'During loading few-sot examples, found unknown dataset: {dataset_name}'
+            )
+        if chat_mode:
+            demostrations.append((question_input, question_output))
+        else:
+            demostrations.append(question_input + question_output + '\n')
+
+    return demostrations
+
+
+enc = None
+
+
+def _lazy_load_enc():
+    global enc
+    if enc is None:
+        enc = tiktoken.encoding_for_model('gpt-4')
+
+
+# cut prompt if reach max token length
+def concat_prompt(demos,
+                  dataset_name,
+                  max_tokens,
+                  end_of_example='\n',
+                  verbose=False):
+    _lazy_load_enc()
+    demostration_en = 'Here are the answers for the problems in the exam.\n'
+    demostration_zh = '以下是考试中各个问题的答案。\n'
+
+    for i in range(len(demos)):
+        # print(len(enc.encode(demostration_en)), len(enc.encode(demostration_zh)))
+        if dataset_name in english_qa_datasets:
+            demostration_en = demostration_en + demos[i] + end_of_example
+        elif dataset_name in chinese_qa_datasets:
+            demostration_zh = demostration_zh + demos[i] + end_of_example
+        elif dataset_name in english_cloze_datasets:
+            demostration_en = demostration_en + demos[i] + end_of_example
+        elif dataset_name in chinese_cloze_datasets:
+            demostration_zh = demostration_zh + demos[i] + end_of_example
+        # break if reach max token limit
+        if len(enc.encode(demostration_en)) < max_tokens and len(
+                enc.encode(demostration_zh)) < max_tokens:
+            output = demostration_en if len(demostration_en) > len(
+                demostration_zh) else demostration_zh
+            prompt_num = i + 1
+        else:
+            break
+    if verbose:
+        print('max_tokens set as ', max_tokens, 'actual_tokens is',
+              len(enc.encode(output)), 'num_shot is', prompt_num)
+    return output, prompt_num
+
+
+def concat_prompt_chat_mode(demos,
+                            dataset_name,
+                            max_tokens,
+                            end_of_example='\n',
+                            verbose=False):
+    _lazy_load_enc()
+    answers = []
+    sentences = ''
+    for i in range(len(demos)):
+        answers += [
+            {
+                'role': 'user',
+                'content': demos[i][0]
+            },
+            {
+                'role': 'assistant',
+                'content': demos[i][1]
+            },
+        ]
+        sentences += json.dumps(answers[-1])
+        # break if reach max token limit
+        if len(enc.encode(sentences)) > max_tokens:
+            answers.pop()
+            answers.pop()
+            break
+    if verbose:
+        print('max_tokens set as ', max_tokens, 'actual_tokens is',
+              len(enc.encode(sentences)), 'num_shot is',
+              len(answers) // 2)
+    return answers, len(answers) // 2
+
+
+def convert_few_shot(line, dataset_name, demo, n_shot, chat_mode=False):
+    passage = line['passage'] if line['passage'] is not None else ''
+    question = line['question']
+    options = line['options'] if line['options'] is not None else ''
+
+    if dataset_name in english_qa_datasets:
+        question_input = 'Problem {}.   '.format(n_shot + 1) + passage + ' ' + question + '\n' \
+            + 'Choose from the following options:    ' + ' '.join(options) + '\n'
+        # + "Explanation for Problem {}:   ".format(n_shot + 1)
+
+    if dataset_name in chinese_qa_datasets:
+        question_input = '问题 {}.   '.format(n_shot + 1) + passage + ' ' + question + '\n' \
+            + '从以下选项中选择:    ' + ' '.join(options) + '\n'
+        # + "问题 {}的解析:   ".format(n_shot + 1)
+
+    if dataset_name in english_cloze_datasets:
+        question_input = 'Problem {}.   '.format(n_shot + 1) + question + '\n'
+        # + "Explanation for Problem {}:   ".format(n_shot + 1)
+
+    if dataset_name in chinese_cloze_datasets:
+        question_input = '问题 {}.   '.format(n_shot + 1) + question + '\n'
+        # + "问题 {}的解析:   ".format(n_shot + 1)
+    if chat_mode:
+        return demo + [
+            {
+                'role': 'user',
+                'content': question_input
+            },
+        ]
+    else:
+        return demo + question_input
+
+
+def load_dataset(dataset_name,
+                 setting_name,
+                 parent_path,
+                 prompt_path=None,
+                 max_tokens=None,
+                 end_of_example='\n',
+                 chat_mode=False,
+                 verbose=False):
+    test_path = os.path.join(parent_path, dataset_name + '.jsonl')
+    loaded_jsonl = read_jsonl(test_path)
+    processed = []
+    if setting_name == 'few-shot-CoT' or setting_name == 'few-shot':
+        # process demo once if it is few-shot-CoT
+        processed_demos = combine_prompt(
+            prompt_path,
+            dataset_name,
+            load_explanation=setting_name == 'few-shot-CoT',
+            chat_mode=chat_mode)
+        if chat_mode:
+            chosen_prompt, n_shot = concat_prompt_chat_mode(processed_demos,
+                                                            dataset_name,
+                                                            max_tokens,
+                                                            end_of_example,
+                                                            verbose=verbose)
+        else:
+            chosen_prompt, n_shot = concat_prompt(processed_demos,
+                                                  dataset_name,
+                                                  max_tokens,
+                                                  end_of_example,
+                                                  verbose=verbose)
+    if verbose:
+        loaded_jsonl = tqdm(loaded_jsonl)
+    for meta_idx, line in enumerate(loaded_jsonl):
+        if setting_name == 'zero-shot':
+            ctxt = convert_zero_shot(line, dataset_name)
+        elif setting_name == 'zero-shot-CoT':
+            ctxt = convert_zero_shot_CoT_stage1(line, dataset_name)
+        elif setting_name == 'few-shot-CoT' or setting_name == 'few-shot':
+            ctxt = convert_few_shot(line, dataset_name, chosen_prompt, n_shot,
+                                    chat_mode)
+        try:
+            new_instance = ChatGPTSchema(context=ctxt, metadata=meta_idx)
+            processed.append(new_instance.to_dict())
+        except NameError:
+            print('Dataset not defined.')
+    return processed
+
+
+def generate_second_stage_input(dataset_name,
+                                input_list,
+                                output_list,
+                                with_format_prompt=False):
+    try:
+        english_format_prompt = 'Based on the previous results, your task is to extract the final answer and provide the output enclosed in brackets【】, such as 【0】 or 【A】.'
+        chinese_format_prompt = '根据以上内容，你的任务是把最终的答案提取出来并填在【】中，例如【0】或者【A】。'
+        if dataset_name in english_qa_datasets:
+            prompt_suffix = 'Therefore, among A through E, the answer is'
+            if with_format_prompt:
+                prompt_suffix = english_format_prompt + prompt_suffix
+        elif dataset_name in chinese_qa_datasets:
+            prompt_suffix = '因此，从A到D, 我们应选择'
+            if with_format_prompt:
+                prompt_suffix = chinese_format_prompt + prompt_suffix
+        elif dataset_name in english_cloze_datasets:
+            prompt_suffix = 'Therefore, the answer is'
+            if with_format_prompt:
+                prompt_suffix = english_format_prompt + prompt_suffix
+        elif dataset_name in chinese_cloze_datasets:
+            prompt_suffix = '因此，答案是'
+            if with_format_prompt:
+                prompt_suffix = chinese_format_prompt + prompt_suffix
+    except NameError:
+        print('Dataset not defined.')
+    processed = []
+    for i in range(len(input_list)):
+        ctxt = '{0}\n{1}\n{2}'.format(input_list[i]['context'],
+                                      extract_answer(output_list[i]),
+                                      prompt_suffix)
+        new_instance = ChatGPTSchema(context=ctxt,
+                                     metadata=input_list[i]['metadata'])
+        processed.append(new_instance.to_dict())
+    return processed
+
+
+def load_dataset_as_result_schema(dataset_name, parent_path):
+    test_path = os.path.join(parent_path, dataset_name + '.jsonl')
+    loaded_jsonl = read_jsonl(test_path)
+
+    processed = []
+    for i, line in enumerate(loaded_jsonl):
+        problem_input = convert_zero_shot(line, dataset_name)
+        processed.append(
+            ResultsForHumanSchema(
+                index=i,
+                problem_input=problem_input,
+                label=line['label'] if line['label'] else line['answer'],
+            ))
+    return processed
+
+
+if __name__ == '__main__':
+
+    # set variables
+    parent_dir = '../../data/V1_1/'
+    raw_prompt_path = '../data/few_shot_prompts.csv'
+
+    # set dataset name to process
+    setting_name = 'few-shot-CoT'  # setting_name can be chosen from ["zero-shot", "zero-shot-CoT", "few-shot-CoT"]
+    data_name = 'jec-qa-kd'
+    save_dir = '../../experiment_input/{}/'.format(setting_name)
+    if not os.path.exists(save_dir):
+        os.makedirs(save_dir)
+    processed_data = load_dataset(data_name,
+                                  setting_name,
+                                  parent_dir,
+                                  prompt_path=raw_prompt_path,
+                                  max_tokens=2048)
+    save_jsonl(processed_data,
+               os.path.join(save_dir, '{}.jsonl'.format(data_name)))

opencompass-my-api/build/lib/opencompass/datasets/agieval/evaluation.py

@@ -0,0 +1,43 @@
+# flake8: noqa
+from . import dataset_loader, utils
+from .math_equivalence import is_equiv
+
+
+def convert_to_set(item):
+    if isinstance(item, list):
+        return set(item)
+    if isinstance(item, str):
+        return {item}
+    if item is None:
+        return {}
+    raise ValueError("Input can't parse:", item)
+
+
+def evaluate_single_sample(dataset_name, prediction, label):
+    if dataset_name in dataset_loader.multi_choice_datasets:
+        p = convert_to_set(prediction)
+        l = convert_to_set(label)
+        return p == l
+    elif dataset_name in dataset_loader.math_output_datasets:
+        return is_equiv(prediction, label)
+    else:
+        return prediction == label
+
+
+# def evaluate(dataset_name, prediction_list, label_list):
+#     correct = 0
+#     if dataset_name in multi_choice_datasets:
+#         for prediction, label in zip(prediction_list, label_list):
+#             p = convert_to_set(prediction)
+#             l = convert_to_set(label)
+#             if p == l:
+#                 correct += 1
+#     elif dataset_name in math_output_datasets:
+#         for prediction, label in zip(prediction_list, label_list):
+#             if is_equiv(prediction, label):
+#                 correct += 1
+#     else:
+#         for prediction, label in zip(prediction_list, label_list):
+#             if prediction == label:
+#                 correct += 1
+#     return "{0:.2%}".format(correct / len(label_list))

opencompass-my-api/build/lib/opencompass/datasets/agieval/math_equivalence.py

@@ -0,0 +1,161 @@
+# flake8: noqa
+
+
+# code from https://github.com/hendrycks/math/blob/main/modeling/math_equivalence.py
+def _fix_fracs(string):
+    substrs = string.split('\\frac')
+    new_str = substrs[0]
+    if len(substrs) > 1:
+        substrs = substrs[1:]
+        for substr in substrs:
+            new_str += '\\frac'
+            if substr[0] == '{':
+                new_str += substr
+            else:
+                try:
+                    assert len(substr) >= 2
+                except:
+                    return string
+                a = substr[0]
+                b = substr[1]
+                if b != '{':
+                    if len(substr) > 2:
+                        post_substr = substr[2:]
+                        new_str += '{' + a + '}{' + b + '}' + post_substr
+                    else:
+                        new_str += '{' + a + '}{' + b + '}'
+                else:
+                    if len(substr) > 2:
+                        post_substr = substr[2:]
+                        new_str += '{' + a + '}' + b + post_substr
+                    else:
+                        new_str += '{' + a + '}' + b
+    string = new_str
+    return string
+
+
+def _fix_a_slash_b(string):
+    if len(string.split('/')) != 2:
+        return string
+    a = string.split('/')[0]
+    b = string.split('/')[1]
+    try:
+        a = int(a)
+        b = int(b)
+        assert string == '{}/{}'.format(a, b)
+        new_string = '\\frac{' + str(a) + '}{' + str(b) + '}'
+        return new_string
+    except:
+        return string
+
+
+def _remove_right_units(string):
+    # "\\text{ " only ever occurs (at least in the val set) when describing units
+    if '\\text{ ' in string:
+        splits = string.split('\\text{ ')
+        assert len(splits) == 2
+        return splits[0]
+    else:
+        return string
+
+
+def _fix_sqrt(string):
+    if '\\sqrt' not in string:
+        return string
+    splits = string.split('\\sqrt')
+    new_string = splits[0]
+    for split in splits[1:]:
+        if split[0] != '{':
+            a = split[0]
+            new_substr = '\\sqrt{' + a + '}' + split[1:]
+        else:
+            new_substr = '\\sqrt' + split
+        new_string += new_substr
+    return new_string
+
+
+def _strip_string(string):
+    # linebreaks
+    string = string.replace('\n', '')
+    # print(string)
+
+    # remove inverse spaces
+    string = string.replace('\\!', '')
+    # print(string)
+
+    # replace \\ with \
+    string = string.replace('\\\\', '\\')
+    # print(string)
+
+    # replace tfrac and dfrac with frac
+    string = string.replace('tfrac', 'frac')
+    string = string.replace('dfrac', 'frac')
+    # print(string)
+
+    # remove \left and \right
+    string = string.replace('\\left', '')
+    string = string.replace('\\right', '')
+    # print(string)
+
+    # Remove circ (degrees)
+    string = string.replace('^{\\circ}', '')
+    string = string.replace('^\\circ', '')
+
+    # remove dollar signs
+    string = string.replace('\\$', '')
+
+    # remove units (on the right)
+    string = _remove_right_units(string)
+
+    # remove percentage
+    string = string.replace('\\%', '')
+    string = string.replace('\%', '')
+
+    # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
+    string = string.replace(' .', ' 0.')
+    string = string.replace('{.', '{0.')
+    # if empty, return empty string
+    if len(string) == 0:
+        return string
+    if string[0] == '.':
+        string = '0' + string
+
+    # to consider: get rid of e.g. "k = " or "q = " at beginning
+    if len(string.split('=')) == 2:
+        if len(string.split('=')[0]) <= 2:
+            string = string.split('=')[1]
+
+    # fix sqrt3 --> sqrt{3}
+    string = _fix_sqrt(string)
+
+    # remove spaces
+    string = string.replace(' ', '')
+
+    # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b}
+    string = _fix_fracs(string)
+
+    # manually change 0.5 --> \frac{1}{2}
+    if string == '0.5':
+        string = '\\frac{1}{2}'
+
+    # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y
+    string = _fix_a_slash_b(string)
+
+    return string
+
+
+def is_equiv(str1, str2, verbose=False):
+    if str1 is None and str2 is None:
+        print('WARNING: Both None')
+        return True
+    if str1 is None or str2 is None:
+        return False
+
+    try:
+        ss1 = _strip_string(str1)
+        ss2 = _strip_string(str2)
+        if verbose:
+            print(ss1, ss2)
+        return ss1 == ss2
+    except:
+        return str1 == str2

opencompass-my-api/build/lib/opencompass/datasets/agieval/post_process.py

@@ -0,0 +1,198 @@
+# flake8: noqa
+import json
+import re
+
+from . import dataset_loader
+
+
+def extract_last_line(string):
+    lines = string.split('\n')
+    for item in lines[::-1]:
+        if item.strip() != '':
+            string = item
+            break
+    return string
+
+
+def remove_few_shot_prefix(string: str):
+    prefix_list = ['The answer is therefore', '答案是']
+    for prefix in prefix_list:
+        if string.startswith(prefix):
+            string = string[len(prefix):].strip()
+        elif prefix in string:
+            index = string.rfind(prefix)
+            if index >= 0:
+                string = string[index + len(prefix):].strip()
+    return string
+
+
+def try_parse_few_shot_qa_single_answer(string, setting_name, language='en'):
+    if setting_name == 'few-shot-CoT':
+        string = extract_last_line(string)
+    if language == 'en':
+        pattern = 'answer is .*?([A-G])'
+        match = re.search(pattern, string)
+    elif language == 'zh':
+        pattern = '答案是.*?([A-G])'
+        match = re.search(pattern, string)
+    else:
+        raise ValueError('Unknown language {0}'.format(language))
+    if match:
+        return match.group(1)
+    else:
+        return None
+
+
+def try_parse_few_shot_pattern(string: str, dataset_name, setting_name):
+    if setting_name == 'few-shot-CoT':
+        string = extract_last_line(string)
+    if dataset_name in dataset_loader.chinese_cloze_datasets:
+        return string.startswith('答案是')
+    elif dataset_name in dataset_loader.english_cloze_datasets:
+        return string.startswith('The answer is therefore')
+    elif dataset_name in dataset_loader.chinese_qa_datasets:
+        pattern = '答案是.*?([A-G])'
+        match = re.search(pattern, string)
+        return match is not None
+    elif dataset_name in dataset_loader.english_qa_datasets:
+        pattern = 'answer is .*?([A-G])'
+        match = re.search(pattern, string)
+        return match is not None
+    return False
+
+
+def parse_few_shot_qa_single_answer(string, setting_name, language='en'):
+    answer = try_parse_few_shot_qa_single_answer(string, setting_name,
+                                                 language)
+    if answer is None:
+        return find_first_capital_letter(string)
+    else:
+        return answer
+
+
+def find_first_capital_letter(answer):
+    letter_set = {'A', 'B', 'C', 'D', 'E', 'F'}
+    for c in answer:
+        if c in letter_set:
+            return c
+    # print("Can't find capital letter in:", answer)
+    return ''
+
+
+def extract_answer_in_bracket(answer, prefix='【', suffix='】'):
+    if prefix not in answer and suffix not in answer:
+        # print("doesn't found special tokens in:", answer)
+        return ''
+    s = answer.index(prefix) + len(prefix)
+    t = answer.index(suffix)
+    ret = answer[s:t]
+    return ret
+
+
+def parse_math_answer(setting_name, raw_string):
+    if setting_name == 'few-shot-CoT':
+        raw_string = extract_last_line(raw_string)
+    if setting_name == 'few-shot-CoT' or setting_name == 'few-shot':
+        raw_string = remove_few_shot_prefix(raw_string)
+        return raw_string
+
+    def remove_boxed(s):
+        left = '\\boxed{'
+        try:
+            assert s[:len(left)] == left
+            assert s[-1] == '}'
+            answer = s[len(left):-1]
+            if '=' in answer:
+                answer = answer.split('=')[-1].lstrip(' ')
+            return answer
+        except:
+            return None
+
+    def last_boxed_only_string(string):
+        idx = string.rfind('\\boxed')
+        if idx < 0:
+            idx = string.rfind('\\fbox')
+            if idx < 0:
+                return None
+        i = idx
+        right_brace_idx = None
+        num_left_braces_open = 0
+        while i < len(string):
+            if string[i] == '{':
+                num_left_braces_open += 1
+            if string[i] == '}':
+                num_left_braces_open -= 1
+                if num_left_braces_open == 0:
+                    right_brace_idx = i
+                    break
+            i += 1
+
+        if right_brace_idx == None:
+            retval = None
+        else:
+            retval = string[idx:right_brace_idx + 1]
+
+        return retval
+
+    def get_answer_with_dollar_sign(s):
+        first_pattern = '\$(.*)\$'
+        last_match = None
+        matches = re.findall(first_pattern, s)
+        if matches:
+            last_match = matches[-1]
+            if '=' in last_match:
+                last_match = last_match.split('=')[-1].lstrip(' ')
+        return last_match
+
+    def get_answer_without_dollar_sign(s):
+        last_match = None
+        if '=' in s:
+            last_match = s.split('=')[-1].lstrip(' ').rstrip('.')
+            if '\\n' in last_match:
+                last_match = last_match.split('\\n')[0]
+        else:
+            pattern = '(?:\\$)?\d+(?:\.\d+)?(?![\w\d])'
+            matches = re.findall(pattern, s)
+            if matches:
+                last_match = matches[-1]
+        return last_match
+
+    raw_string = remove_few_shot_prefix(raw_string)
+    if '\\boxed' in raw_string:
+        answer = remove_boxed(last_boxed_only_string(raw_string))
+    else:
+        answer = get_answer_with_dollar_sign(raw_string)
+        if not answer:
+            answer = get_answer_without_dollar_sign(raw_string)
+    return answer
+
+
+def parse_qa_multiple_answer(string, setting_name):
+    if setting_name == 'few-shot-CoT':
+        string = extract_last_line(string)
+    pattern = '\(*([A-Z])\)*'
+    match = re.findall(pattern, string)
+    if match:
+        return match
+    return []
+
+
+def post_process(dataset_name, setting_name, prediction):
+    if dataset_name in dataset_loader.english_cloze_datasets or dataset_name in dataset_loader.chinese_cloze_datasets:
+        return parse_math_answer(setting_name, prediction)
+
+    if dataset_name in ['jec-qa-kd', 'jec-qa-ca', 'gaokao-physics']:
+        return parse_qa_multiple_answer(prediction, setting_name)
+
+    # all other datasets are QA problems with single answer
+    if 'zero-shot' in setting_name:
+        answer = find_first_capital_letter(prediction)
+        return answer
+
+    # all other datasets are QA problems with single answer and setting_name are few-shot
+    language = 'en' if dataset_name in dataset_loader.english_qa_datasets else 'zh'
+    if dataset_name in dataset_loader.english_qa_datasets or dataset_name in dataset_loader.chinese_qa_datasets:
+        return parse_few_shot_qa_single_answer(prediction, setting_name,
+                                               language)
+    else:
+        raise ValueError(f'Unsupported dataset name {dataset_name}')

opencompass-my-api/build/lib/opencompass/datasets/agieval/utils.py

@@ -0,0 +1,43 @@
+# flake8: noqa
+import json
+
+
+def read_jsonl(path):
+    with open(path, encoding='utf8') as fh:
+        results = []
+        for line in fh:
+            if line is None:
+                continue
+            try:
+                results.append(json.loads(line) if line != 'null' else line)
+            except Exception as e:
+                print(e)
+                print(path)
+                print(line)
+                raise e
+    return results
+
+
+def save_jsonl(lines, directory):
+    with open(directory, 'w', encoding='utf8') as f:
+        for line in lines:
+            f.write(json.dumps(line, ensure_ascii=False) + '\n')
+
+
+def extract_answer(js):
+    try:
+        if js is None or js == 'null':
+            return ''
+        answer = ''
+        if isinstance(js, str):
+            answer = js
+        elif 'text' in js['choices'][0]:
+            answer = js['choices'][0]['text']
+        else:
+            answer = js['choices'][0]['message']['content']
+            # answer = js['']
+        return answer
+    except Exception as e:
+        # print(e)
+        # print(js)
+        return ''

opencompass-my-api/build/lib/opencompass/datasets/anli.py

@@ -0,0 +1,18 @@
+import json
+
+from datasets import Dataset
+
+from .base import BaseDataset
+
+
+class AnliDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        dataset = []
+        with open(path, 'r') as f:
+            for line in f:
+                line = json.loads(line)
+                line['label'] = {'c': 'A', 'e': 'B', 'n': 'C'}[line['label']]
+                dataset.append(line)
+        return Dataset.from_list(dataset)

opencompass-my-api/build/lib/opencompass/datasets/anthropics_evals.py

@@ -0,0 +1,63 @@
+from datasets import load_dataset
+
+from .base import BaseDataset
+
+
+class AiRiskDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        """Load dataset."""
+
+        dataset = load_dataset('json', data_files=path)
+
+        def choices_process(example):
+            # the original answer format is ` (A)`, etc.
+            for i in 'ABCDEFGH':
+                if i in example['answer_matching_behavior']:
+                    example['answer_matching_behavior'] = i
+                    break
+            return example
+
+        dataset = dataset.map(choices_process)
+        return dataset
+
+
+class PersonaDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        """Load dataset."""
+
+        dataset = load_dataset('json', data_files=path)
+
+        def choices_process(example):
+            # the original answer format is ` No` or ` Yes`.
+            if example['answer_matching_behavior'] == ' Yes':
+                example['answer_matching_behavior'] = 'A'
+            else:
+                example['answer_matching_behavior'] = 'B'
+            return example
+
+        dataset = dataset.map(choices_process)
+        return dataset
+
+
+class SycophancyDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        """Load dataset."""
+
+        dataset = load_dataset('json', data_files=path)
+
+        def choices_process(example):
+            # the original answer format is ` (A)`, etc.
+            for i in 'ABCDEFG':
+                if i in example['answer_matching_behavior']:
+                    example['answer_matching_behavior'] = i
+                    break
+            return example
+
+        dataset = dataset.map(choices_process)
+        return dataset

opencompass-my-api/build/lib/opencompass/datasets/arc.py

@@ -0,0 +1,84 @@
+import json
+import os.path as osp
+
+from datasets import Dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class ARCDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        with open(path, 'r', errors='ignore') as in_f:
+            rows = []
+            for line in in_f:
+                item = json.loads(line.strip())
+                question = item['question']
+                if len(question['choices']) != 4:
+                    continue
+                labels = [c['label'] for c in question['choices']]
+                answerKey = 'ABCD'[labels.index(item['answerKey'])]
+                rows.append({
+                    'question': question['stem'],
+                    'answerKey': answerKey,
+                    'textA': question['choices'][0]['text'],
+                    'textB': question['choices'][1]['text'],
+                    'textC': question['choices'][2]['text'],
+                    'textD': question['choices'][3]['text'],
+                })
+            return Dataset.from_list(rows)
+
+
+class ARCDatasetClean(BaseDataset):
+
+    # load the contamination annotations of CEval from
+    # https://github.com/liyucheng09/Contamination_Detector
+    @staticmethod
+    def load_contamination_annotations(path, split='val'):
+        import requests
+
+        assert split == 'test', 'We only have test set annotation for ARC'
+        annotation_cache_path = osp.join(
+            path, f'ARC_c_{split}_contamination_annotations.json')
+        if osp.exists(annotation_cache_path):
+            with open(annotation_cache_path, 'r') as f:
+                annotations = json.load(f)
+            return annotations
+        link_of_annotations = 'https://github.com/liyucheng09/Contamination_Detector/releases/download/v0.1.1rc/ARC_annotations.json'  # noqa
+        annotations = json.loads(requests.get(link_of_annotations).text)
+        with open(annotation_cache_path, 'w') as f:
+            json.dump(annotations, f)
+        return annotations
+
+    @staticmethod
+    def load(path: str):
+        annotations = ARCDatasetClean.load_contamination_annotations(
+            osp.dirname(path), 'test')
+        with open(path, 'r', errors='ignore') as in_f:
+            rows = []
+            for line in in_f:
+                item = json.loads(line.strip())
+                id_ = item['id']
+                question = item['question']
+                if id_ in annotations:
+                    is_clean = annotations[id_][0]
+                else:
+                    is_clean = 'not labeled'
+                if len(question['choices']) != 4:
+                    continue
+                labels = [c['label'] for c in question['choices']]
+                answerKey = 'ABCD'[labels.index(item['answerKey'])]
+                rows.append({
+                    'question': question['stem'],
+                    'answerKey': answerKey,
+                    'textA': question['choices'][0]['text'],
+                    'textB': question['choices'][1]['text'],
+                    'textC': question['choices'][2]['text'],
+                    'textD': question['choices'][3]['text'],
+                    'is_clean': is_clean,
+                })
+            return Dataset.from_list(rows)

opencompass-my-api/build/lib/opencompass/datasets/ax.py

@@ -0,0 +1,24 @@
+import json
+
+from datasets import Dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class AXDataset_V2(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        dataset = []
+        with open(path, 'r') as f:
+            for line in f:
+                line = json.loads(line)
+                line['label'] = {
+                    'entailment': 'A',
+                    'not_entailment': 'B'
+                }[line['label']]
+                dataset.append(line)
+        return Dataset.from_list(dataset)

opencompass-my-api/build/lib/opencompass/datasets/base.py

@@ -0,0 +1,28 @@
+from abc import abstractstaticmethod
+from typing import Dict, Optional, Union
+
+from datasets import Dataset, DatasetDict
+
+from opencompass.openicl import DatasetReader
+
+
+class BaseDataset:
+
+    def __init__(self, reader_cfg: Optional[Dict] = {}, **kwargs):
+        self.dataset = self.load(**kwargs)
+        self._init_reader(**reader_cfg)
+
+    def _init_reader(self, **kwargs):
+        self.reader = DatasetReader(self.dataset, **kwargs)
+
+    @property
+    def train(self):
+        return self.reader.dataset['train']
+
+    @property
+    def test(self):
+        return self.reader.dataset['test']
+
+    @abstractstaticmethod
+    def load(**kwargs) -> Union[Dataset, DatasetDict]:
+        pass

opencompass-my-api/build/lib/opencompass/datasets/bbh.py

@@ -0,0 +1,98 @@
+import json
+import os.path as osp
+import re
+
+from datasets import Dataset
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import (ICL_EVALUATORS, LOAD_DATASET,
+                                  TEXT_POSTPROCESSORS)
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class BBHDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str, name: str):
+        with open(osp.join(path, f'{name}.json'), 'r') as f:
+            data = json.load(f)['examples']
+        dataset = Dataset.from_list(data)
+        return dataset
+
+
+@TEXT_POSTPROCESSORS.register_module('bbh-mcq')
+def bbh_mcq_postprocess(text: str) -> str:
+    ans = text
+    ans_line = ans.split('answer is ')
+    if len(ans_line) != 1:
+        ans = ans_line[1].strip()
+    match = re.search(r'\(([A-Z])\)*', ans)
+    if match:
+        return match.group(1)
+    match = re.search(r'([A-Z])', ans)
+    if match:
+        return match.group(1)
+    return ans
+
+
+@TEXT_POSTPROCESSORS.register_module('bbh-freeform')
+def bbh_freeform_postprocess(text: str) -> str:
+    ans = text
+    ans_line = ans.split('answer is ')
+    if len(ans_line) != 1:
+        ans = ans_line[1].strip()
+    ans = ans.split('\n')[0]
+    if ans.endswith('.'):
+        ans = ans[:-1]
+    return ans
+
+
+@ICL_EVALUATORS.register_module()
+class BBHEvaluator(BaseEvaluator):
+
+    def score(self, predictions, references):
+        if len(predictions) != len(references):
+            return {
+                'error': 'predictions and references have different '
+                'length'
+            }
+
+        predictions = [bbh_freeform_postprocess(pred) for pred in predictions]
+
+        details = []
+        cnt = 0
+        for pred, ref in zip(predictions, references):
+            detail = {'pred': pred, 'answer': ref, 'correct': False}
+            if pred == ref:
+                cnt += 1
+                detail['correct'] = True
+            details.append(detail)
+
+        score = cnt / len(predictions) * 100
+
+        return {'score': score, 'details': details}
+
+
+@ICL_EVALUATORS.register_module()
+class BBHEvaluator_mcq(BaseEvaluator):
+
+    def score(self, predictions, references):
+        if len(predictions) != len(references):
+            return {
+                'error': 'predictions and references have different '
+                'length'
+            }
+        details = []
+        cnt = 0
+        for pred, ref in zip(predictions, references):
+            detail = {'pred': pred, 'answer': ref, 'correct': False}
+            if pred == ref:
+                cnt += 1
+                detail['correct'] = True
+            details.append(detail)
+
+        score = cnt / len(predictions) * 100
+
+        return {'score': score, 'details': details}

opencompass-my-api/build/lib/opencompass/datasets/boolq.py

@@ -0,0 +1,56 @@
+import json
+
+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class BoolQDataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+        dataset = load_dataset(**kwargs)
+
+        def preprocess(example):
+            if example['label'] == 'true':
+                example['answer'] = 1
+            else:
+                example['answer'] = 0
+            return example
+
+        dataset = dataset.map(preprocess)
+        return dataset
+
+
+@LOAD_DATASET.register_module()
+class BoolQDataset_V2(BaseDataset):
+
+    @staticmethod
+    def load(path):
+        dataset = []
+        with open(path, 'r') as f:
+            for line in f:
+                line = json.loads(line)
+                line['label'] = {'true': 'A', 'false': 'B'}[line['label']]
+                dataset.append(line)
+        return Dataset.from_list(dataset)
+
+
+@LOAD_DATASET.register_module()
+class BoolQDataset_V3(BaseDataset):
+
+    @staticmethod
+    def load(path):
+        dataset = []
+        with open(path, 'r') as f:
+            for line in f:
+                line = json.loads(line)
+                line['passage'] = ' -- '.join(
+                    line['passage'].split(' -- ')[1:])
+                line['question'] = line['question'][0].upper(
+                ) + line['question'][1:]
+                dataset.append(line)
+        return Dataset.from_list(dataset)

opencompass-my-api/build/lib/opencompass/datasets/bustum.py

@@ -0,0 +1,21 @@
+import json
+
+from datasets import Dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class bustumDataset_V2(BaseDataset):
+
+    @staticmethod
+    def load(path):
+        data = []
+        with open(path, 'r', encoding='utf-8') as f:
+            for line in f:
+                line = json.loads(line)
+                line['label'] = 'AB'[int(line['label'])]
+                data.append(line)
+        return Dataset.from_list(data)

opencompass-my-api/build/lib/opencompass/datasets/c3.py

@@ -0,0 +1,80 @@
+import json
+
+from datasets import Dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class C3Dataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+
+        with open(path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        rows = []
+        for _, row in enumerate(data):
+            content = row[0]
+            content_str = ' '.join(
+                [''.join(paragraph) for paragraph in content])
+
+            for question in row[1]:
+                label = question['choice'].index(question['answer'])
+                length = len(question['choice'])
+                if length < 4:
+                    fill_value = question['choice'][0]  # 以第一个值为填充值
+                    fill_count = 4 - length  # 需要填充的数量
+                    question['choice'] += [fill_value] * fill_count  # 填充
+
+                rows.append({
+                    'content': content_str,
+                    'question': question['question'],
+                    'choices': question['choice'],
+                    'choice0': question['choice'][0],
+                    'choice1': question['choice'][1],
+                    'choice2': question['choice'][2],
+                    'choice3': question['choice'][3],
+                    'label': label
+                })
+
+        dataset = Dataset.from_dict({
+            'content': [row['content'] for row in rows],
+            'question': [row['question'] for row in rows],
+            'choice0': [row['choice0'] for row in rows],
+            'choice1': [row['choice1'] for row in rows],
+            'choice2': [row['choice2'] for row in rows],
+            'choice3': [row['choice3'] for row in rows],
+            'choices': [row['choices'] for row in rows],
+            'label': [row['label'] for row in rows]
+        })
+        return dataset
+
+
+@LOAD_DATASET.register_module()
+class C3Dataset_V2(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        with open(path, 'r', encoding='utf-8') as f:
+            raw = json.load(f)
+        data = []
+        for line in raw:
+            content = ''.join([''.join(paragraph) for paragraph in line[0]])
+            for question in line[1]:
+                label = question['choice'].index(question['answer'])
+                label = 'ABCD'[label]
+                while len(question['choice']) < 4:
+                    question['choice'].append('[NULL]')
+                data.append({
+                    'content': content,
+                    'question': question['question'],
+                    'choice0': question['choice'][0],
+                    'choice1': question['choice'][1],
+                    'choice2': question['choice'][2],
+                    'choice3': question['choice'][3],
+                    'label': label
+                })
+        return Dataset.from_list(data)

opencompass-my-api/build/lib/opencompass/datasets/cb.py

@@ -0,0 +1,25 @@
+import json
+
+from datasets import Dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class CBDataset_V2(BaseDataset):
+
+    @staticmethod
+    def load(path):
+        dataset = []
+        with open(path, 'r') as f:
+            for line in f:
+                line = json.loads(line)
+                line['label'] = {
+                    'contradiction': 'A',
+                    'entailment': 'B',
+                    'neutral': 'C'
+                }[line['label']]
+                dataset.append(line)
+        return Dataset.from_list(dataset)

opencompass-my-api/build/lib/opencompass/datasets/ceval.py

@@ -0,0 +1,76 @@
+import csv
+import json
+import os.path as osp
+
+from datasets import Dataset, DatasetDict
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class CEvalDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str, name: str):
+        dataset = {}
+        for split in ['dev', 'val', 'test']:
+            filename = osp.join(path, split, f'{name}_{split}.csv')
+            with open(filename, encoding='utf-8') as f:
+                reader = csv.reader(f)
+                header = next(reader)
+                for row in reader:
+                    item = dict(zip(header, row))
+                    item.setdefault('explanation', '')
+                    item.setdefault('answer', '')
+                    dataset.setdefault(split, []).append(item)
+        dataset = {i: Dataset.from_list(dataset[i]) for i in dataset}
+        return DatasetDict(dataset)
+
+
+class CEvalDatasetClean(BaseDataset):
+
+    # load the contamination annotations of CEval from
+    # https://github.com/liyucheng09/Contamination_Detector
+    @staticmethod
+    def load_contamination_annotations(path, split='val'):
+        import requests
+
+        assert split == 'val', 'Now we only have annotations for val set'
+        annotation_cache_path = osp.join(
+            path, split, 'ceval_contamination_annotations.json')
+        if osp.exists(annotation_cache_path):
+            with open(annotation_cache_path, 'r') as f:
+                annotations = json.load(f)
+            return annotations
+        link_of_annotations = 'https://github.com/liyucheng09/Contamination_Detector/releases/download/v0.1.1rc/ceval_annotations.json'  # noqa
+        annotations = json.loads(requests.get(link_of_annotations).text)
+        with open(annotation_cache_path, 'w') as f:
+            json.dump(annotations, f)
+        return annotations
+
+    @staticmethod
+    def load(path: str, name: str):
+        dataset = {}
+        for split in ['dev', 'val', 'test']:
+            if split == 'val':
+                annotations = CEvalDatasetClean.load_contamination_annotations(
+                    path, split)
+            filename = osp.join(path, split, f'{name}_{split}.csv')
+            with open(filename, encoding='utf-8') as f:
+                reader = csv.reader(f)
+                header = next(reader)
+                for row_index, row in enumerate(reader):
+                    item = dict(zip(header, row))
+                    item.setdefault('explanation', '')
+                    item.setdefault('answer', '')
+                    if split == 'val':
+                        row_id = f'{name}-{row_index}'
+                        if row_id in annotations:
+                            item['is_clean'] = annotations[row_id][0]
+                        else:
+                            item['is_clean'] = 'not labeled'
+                    dataset.setdefault(split, []).append(item)
+        dataset = {i: Dataset.from_list(dataset[i]) for i in dataset}
+        return DatasetDict(dataset)

opencompass-my-api/build/lib/opencompass/datasets/chid.py

@@ -0,0 +1,43 @@
+import json
+
+from datasets import Dataset, load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class CHIDDataset(BaseDataset):
+
+    @staticmethod
+    def load(**kwargs):
+
+        dataset = load_dataset(**kwargs)
+
+        def preprocess(example):
+            content = example['content']
+            for i, c in enumerate(example['candidates']):
+                example[f'content{i}'] = content.replace('#idiom#', c)
+            return example
+
+        dataset = dataset.map(preprocess)
+        return dataset
+
+
+@LOAD_DATASET.register_module()
+class CHIDDataset_V2(BaseDataset):
+
+    @staticmethod
+    def load(path):
+        data = []
+        with open(path, 'r', encoding='utf-8') as f:
+            for line in f:
+                line = json.loads(line)
+                item = {}
+                item['content'] = line['content'].replace('#idiom#', '______')
+                for i, c in enumerate(line['candidates']):
+                    item[chr(ord('A') + i)] = c
+                item['answer'] = 'ABCDEFG'[line['answer']]
+                data.append(item)
+        return Dataset.from_list(data)

opencompass-my-api/build/lib/opencompass/datasets/cibench.py

@@ -0,0 +1,511 @@
+import json
+import os
+import os.path as osp
+import re
+import subprocess
+from collections import defaultdict
+from typing import List, Optional
+
+import numpy as np
+from datasets import Dataset
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
+
+from .base import BaseDataset
+
+
+def load_experiment(file: str) -> dict:
+    """Load single experiment file with solutions."""
+    with open(file, 'r') as f:
+        notebook = json.load(f)
+        example = notebook['cells']
+        metadata = notebook['metadata']
+        modules = metadata.get('modules', [])
+        if modules:
+            # these two annotations should be the same
+            assert len(modules) == len(metadata.get('step_types'))
+            # reformat annotations
+            modules = [[_m.strip() for _m in _modules.split('&')]
+                       for _modules in modules]
+        questions = []
+        source_codes = []
+        outputs = []
+        tags = []
+        for cell in example:
+            if cell['cell_type'] == 'markdown':
+                text = ''.join(cell['source']).strip()
+                if modules:
+                    _modules = modules.pop(0)
+                    text += f"Please use {' and '.join(_modules)} modules."
+                text = text.strip() + '\n'
+                # append the formatted text
+                questions.append(text)
+            elif cell['cell_type'] == 'code':
+                source_codes.append(''.join(cell['source']))
+                if cell['outputs'] and 'data' in cell['outputs'][-1]:
+                    if 'image/png' in cell['outputs'][-1]['data']:
+                        # skip vis temporarily due to lack of evaluation
+                        tags.append('vis')
+                        outputs.append(
+                            cell['outputs'][-1]['data']['image/png'])
+                    elif 'text/plain' in cell['outputs'][-1]['data']:
+                        tags.append('general')
+                        outputs.append(''.join(
+                            cell['outputs'][-1]['data']['text/plain']))
+                else:
+                    tags.append('exec')
+                    outputs.append(None)
+    return dict(
+        experiment=file,
+        questions=sum(([
+            dict(role='user', content=question),
+            dict(role='assistant', content=source_code)
+        ] for question, source_code in zip(questions, source_codes)), []),
+        references=dict(outputs=outputs,
+                        tags=tags,
+                        metadata=metadata,
+                        experiment=file),
+    )
+
+
+def load_experiment_template(file: str) -> dict:
+    """Load single experiment file with solutions for template experiment."""
+    with open(file, 'r') as f:
+        notebook = json.load(f)
+        example = notebook['cells']
+        metadata = notebook['metadata']
+        modules = metadata.get('modules', [])
+        if modules:
+            # these two annotations should be the same
+            assert len(modules) == len(metadata.get('step_types'))
+            # reformat annotations
+            modules = [[_m.strip() for _m in _modules.split('&')]
+                       for _modules in modules]
+        questions = []
+        source_codes = []
+        outputs = []
+        tags = []
+        for cell in example:
+            if cell['cell_type'] == 'markdown':
+                text = ''.join(cell['source']).strip()
+                if modules:
+                    _modules = modules.pop(0)
+                    if 'chinese' not in file:
+                        text += f"Please use {' and '.join(_modules)} modules."
+                    else:
+                        text += f"请用 {' 和 '.join(_modules)} 模块."
+                text = text.strip() + '\n'
+                # append the formatted text
+                questions.append(text)
+            elif cell['cell_type'] == 'code':
+                source_codes.append(''.join(cell['source']))
+                output_flag = False
+                if cell['outputs']:
+                    for _output in cell['outputs']:
+                        if _output['output_type'] == 'display_data':
+                            assert not output_flag
+                            if 'image/png' in _output['data']:
+                                output_flag = True
+                                tags.append('vis')
+                                outputs.append(_output['data']['image/png'])
+                    for _output in cell['outputs'][::-1]:
+                        if output_flag:
+                            break
+                        if _output['output_type'] == 'stream' and _output[
+                                'name'] == 'stdout':
+                            assert not output_flag
+                            output_flag = True
+                            tags.append('general')
+                            outputs.append(''.join(_output['text']))
+                        elif _output['output_type'] == 'execute_result':
+                            assert not output_flag
+                            output_flag = True
+                            tags.append('general')
+                            outputs.append(''.join(
+                                _output['data']['text/plain']))
+                if not output_flag:
+                    # no output fallback to exec
+                    tags.append('exec')
+                    outputs.append(None)
+    return dict(
+        experiment=file,
+        questions=sum(([
+            dict(role='user', content=question),
+            dict(role='assistant', content=source_code)
+        ] for question, source_code in zip(questions, source_codes)), []),
+        references=dict(outputs=outputs,
+                        tags=tags,
+                        metadata=metadata,
+                        experiment=file),
+    )
+
+
+def check_internet():
+    """A tricky way to check internet."""
+    import socket
+
+    import nltk
+    socket.setdefaulttimeout(10)
+    ret = nltk.download('stopwords', quiet=True)
+    socket.setdefaulttimeout(None)
+    if not ret:
+        raise ConnectionError('CIBench needs internet to get response. Please'
+                              'check your internet and proxy.')
+
+
+@LOAD_DATASET.register_module()
+class CIBenchDataset(BaseDataset):
+    """Code Interpreter dataset."""
+
+    @staticmethod
+    def load(path: str, internet_check: bool = False):
+        """Load whole dataset.
+
+        Args:
+            path(str): Path of cibench dataset.
+            internet_check(bool): Whether to check internet.
+                Defaults to False.
+        """
+        if internet_check:
+            check_internet()
+        assert os.path.exists(path), f'Path {path} does not exist.'
+        data_list = []
+        for cwd, dirs, files in os.walk(path):
+            dirs.sort()
+            files.sort()
+            for f in files:
+                if '.ipynb' in f:
+                    data = load_experiment(os.path.join(cwd, f))
+                    data_list.append(data)
+
+        dataset = Dataset.from_list(data_list)
+        return dataset
+
+
+@LOAD_DATASET.register_module()
+class CIBenchTemplateDataset(BaseDataset):
+    """Code Interpreter dataset for template dataset."""
+
+    @staticmethod
+    def load(path: str, internet_check: bool = False):
+        """Load whole dataset.
+
+        Args:
+            path(str): Path of cibench dataset.
+            internet_check(bool): Whether to check internet.
+                Defaults to False.
+        """
+        if internet_check:
+            check_internet()
+        assert os.path.exists(path), f'Path {path} does not exist.'
+        data_list = []
+        for cwd, dirs, files in os.walk(path):
+            dirs.sort()
+            files.sort()
+            for f in files:
+                if '.ipynb' in f:
+                    data = load_experiment_template(os.path.join(cwd, f))
+                    data_list.append(data)
+
+        dataset = Dataset.from_list(data_list)
+        return dataset
+
+
+class CIBenchEvaluator(BaseEvaluator):
+    """Evaluator for CI dataset.
+
+    Args:
+        text_evaluator (optional, dict): The text evaluator for text result
+            comparison[]. Defaults to None, which use Rouge as defaults.
+            Please notice that a extra key for `metric_name` should be set
+            to get the exact metric result, such as `rouge1`.
+        output_dir (optional, str): The directory to save experiment
+            files in a markdown or notebook format.
+        with_ipynb (bool): Generate ipynb correspondingly.
+            Defaults to False.
+        user_data_dir (str): The directory to load local files.
+            Defaults to 'ENV', which means use environment variable
+            `USER_DATA_DIR` to get the data dir.
+    """
+
+    def __init__(self,
+                 text_evaluator: Optional[dict] = None,
+                 output_dir: Optional[str] = None,
+                 with_ipynb: bool = False,
+                 user_data_dir: str = 'ENV') -> None:
+        if text_evaluator is None:
+            from opencompass.openicl.icl_evaluator import RougeEvaluator
+            self.text_evaluator = ICL_EVALUATORS.build(
+                dict(type=RougeEvaluator))
+            self.text_eval_metric = 'rouge1'
+        else:
+            self.text_eval_metric = text_evaluator.pop('metric_name')
+            self.text_evaluator = ICL_EVALUATORS.build(text_evaluator)
+        # TODO: should use work dir for this task.
+        self.output_dir = output_dir
+        self.user_data_dir = self.check_user_data_dir(user_data_dir)
+        self.with_ipynb = with_ipynb
+        self.TAG_MAPPING = {
+            'exec': ('executable', self.valid_step),
+            'general': ('general_correct', self.correct_step),
+            'num': ('numeric_correct', self.correct_step),
+            'text': ('text_score', self.text_step),
+            'vis': ('vis_sim', self.vis_similarity_step),
+        }
+
+    def check_user_data_dir(self, user_data_dir):
+        if user_data_dir == 'ENV':
+            default_path = osp.abspath('./data/cibench_dataset/datasources')
+            user_data_dir = os.environ.get('USER_DATA_DIR', default_path)
+        user_data_dir = user_data_dir.rstrip('/')
+        basename = osp.basename(user_data_dir)
+        if basename and basename != 'data':
+            user_data_dir = osp.join(user_data_dir, 'data')
+            assert osp.exists(user_data_dir), \
+                f'a subfolder named `data` should exist under {user_data_dir}.'
+        elif basename:
+            assert osp.exists(user_data_dir), \
+                f'{user_data_dir} does not exist.'
+        return user_data_dir
+
+    @staticmethod
+    def valid_step(step):
+        """Whether the step is executable and valid."""
+        # Found the latest code interpreter to determine valid
+        for action in step[::-1]:
+            if action['type'] == 'IPythonInterpreter':
+                if action['errmsg']:
+                    return False
+                else:
+                    return True
+        # No code interpreter for this step, reckon as False
+        return False
+
+    @staticmethod
+    def correct_step(step, target):
+        """Whether the step output is correct."""
+        # Found the latest code interpreter to determine correct
+        for action in step[::-1]:
+            if action['type'] == 'IPythonInterpreter':
+                if action['result']:
+                    try:
+                        pred = action['result']['text']
+                        match_exec = re.search(
+                            'execute_result:\n\n```\n(.*?)\n```', pred,
+                            re.DOTALL)
+                        match_stdout = re.search('stdout:\n\n```\n(.*?)\n```',
+                                                 pred, re.DOTALL)
+                        # get pred result from execute_result by default
+                        # else stdout
+                        if match_exec and match_stdout:
+                            match = match_exec
+                        elif match_exec:
+                            match = match_exec
+                        elif match_stdout:
+                            match = match_stdout
+                        else:
+                            match = None
+                        if match:
+                            out = match.group(1)
+                            score = (out.strip() == target.strip()
+                                     or target.strip() in out.strip())
+                            return score
+                    except Exception:
+                        return False
+        # Fall back to False
+        return False
+
+    def text_step(self, step, target):
+        """Whether the step output is correct."""
+        # Found the latest code interpreter to determine correct
+        for action in step[::-1]:
+            if action['type'] == 'IPythonInterpreter':
+                if action['result']:
+                    try:
+                        pred = action['result']['text']
+                        match = re.search('```\n(.*?)\n```', pred, re.DOTALL)
+                        if match:
+                            out = match.group(1)
+                            score = self.text_evaluator.score([out], [target])
+                            return score[self.text_eval_metric] / 100
+                    except Exception:
+                        return False
+        # Fall back to False
+        return False
+
+    @staticmethod
+    def vis_similarity_step(step, target):
+        """Whether the step output image has the same structure similarity with
+        the given images."""
+        # Found the latest code interpreter to determine correct
+        import base64
+
+        import skimage
+
+        for action in step[::-1]:
+            if action['type'] == 'IPythonInterpreter':
+                if action['result']:
+                    try:
+                        pred = action['result']['text']
+                        match = re.search(r'!\[fig-[0-9]*\]\((.*?)\)', pred,
+                                          re.DOTALL)
+                        if match:
+                            img_pred = match.group(1)
+                        img2 = base64.b64decode(target)
+                        img2 = skimage.io.imread(img2, plugin='imageio')
+                        img1 = skimage.io.imread(img_pred, plugin='imageio')
+                        img1 = skimage.transform.resize(img1, img2.shape[:2])
+                        img1 = 255 * img1
+                        # Convert to integer data type pixels.
+                        img1 = img1.astype(np.uint8)
+                        ssim = skimage.metrics.structural_similarity(
+                            img1, img2, channel_axis=-1)
+                        # mse = skimage.metrics.mean_squared_error(img1, img2)
+                        # ssim greater better
+                        # mse smaller better but has no upper bound
+                        return ssim
+                    except Exception:
+                        return 0
+        # Fall back to 0
+        return 0
+
+    def save_results(self, origin_prompt, steps):
+        """Save the prediction result in a markdown and notebook format."""
+
+        def check_jupytext():
+            """Check requirements existence."""
+            from shutil import which
+
+            assert which('jupytext'), (
+                "Please install jupytext use 'pip install jupytext' to ensure"
+                'the conversion processes.')
+
+        check_jupytext()
+        p_list = []
+        from opencompass.lagent.actions.ipython_interpreter import extract_code
+        for idx, (example_origin_prompt,
+                  example_steps) in enumerate(zip(origin_prompt, steps)):
+            markdown_lines = []
+            for prompt, step in zip(example_origin_prompt, example_steps):
+                for action in step[::-1]:
+                    if action['type'] == 'IPythonInterpreter':
+                        valid_action = action
+                        break
+                    # fall back to final action
+                    valid_action = step[-1]
+                markdown_lines.append(prompt)
+                markdown_lines.append('\n')
+                code_text = valid_action['args']['text']
+                code_text = extract_code(code_text)
+                code_text = '```python\n' + code_text + '\n```'
+                markdown_lines.append(code_text)
+                markdown_lines.append('\n')
+
+            md_file = f'experiment{idx}.md'
+            with open(md_file, 'w') as f:
+                f.writelines(markdown_lines)
+
+            # TODO: be careful for this
+            # The result might be different with infer process
+            # please check carefully
+            # convert markdown to ipynb and exectue with error tolerance
+            if self.with_ipynb:
+                p = subprocess.Popen(
+                    'jupytext --to ipynb --pipe-fmt ipynb '
+                    "--pipe 'jupyter nbconvert --to ipynb --execute "
+                    f"--allow-errors --stdin --stdout' {md_file}",
+                    shell=True)
+                p_list.append(p)
+        # TODO: async wait
+        for p in p_list:
+            p.wait()
+
+    def set_data_dir(self, work_dir):
+        """Set work directory and link data files for save notebook results."""
+        if self.user_data_dir:
+            basename = osp.basename(self.user_data_dir)
+
+            if not osp.exists(osp.join(self.output_dir, basename)):
+                os.symlink(self.user_data_dir,
+                           osp.join(self.output_dir, basename))
+        os.chdir(work_dir)
+
+    def unset_data_dir(self, work_dir):
+        """Change work directory and keep the symlink."""
+        os.chdir(work_dir)
+
+    def single_exp(self, gold, steps):
+        tags = gold['tags']
+        outputs = gold['outputs']
+        metadata = gold['metadata']
+        hard_tags = metadata.get('step_types', [])
+        if hard_tags:
+            tags = hard_tags
+
+        # executable: exec succeed
+        # general_correct: general correct
+        # numeric_correct: numerical correct
+        # text_score: text score
+        # vis_sim: visual similarity
+
+        # create empty results
+        result = dict()
+        if hard_tags:
+            check_tags = ['exec', 'num', 'text', 'vis']
+        else:
+            check_tags = ['exec', 'general', 'vis']
+        for tag in check_tags:
+            key = self.TAG_MAPPING[tag][0]
+            result[key] = []
+
+        for tag, step, output in zip(tags, steps, outputs):
+            # check whether this step is valid
+            result['executable'].append(self.valid_step(step))
+            if tag != 'exec':
+                key, func = self.TAG_MAPPING[tag]
+                result[key].append(func(step, output))
+
+        return result
+
+    def get_output_dir(self):
+        """Get output dir from eval task.
+
+        Notice: output dir should be in format xxx/data.
+        All the needed files should be
+        """
+        # hard hack for get output dir from eval task
+        if hasattr(self, '_out_dir') and self.output_dir is None:
+            self.output_dir = self._out_dir
+
+    def score(self, predictions: List, references: List, steps: List,
+              origin_prompt: List):
+        """Calculate accuracy."""
+        if len(steps) != len(references):
+            return {'error': 'steps and refrs have different length'}
+        cwd = os.getcwd()
+        self.get_output_dir()
+        if self.output_dir:
+            if not osp.exists(self.output_dir):
+                os.makedirs(self.output_dir)
+            self.set_data_dir(self.output_dir)
+            self.save_results(origin_prompt, steps)
+            self.unset_data_dir(cwd)
+
+        total_results = defaultdict(float)
+        total_scores = defaultdict(float)
+        total_nums = defaultdict(int)
+        for gold, single_steps in zip(references, steps):
+            result = self.single_exp(gold, single_steps)
+
+            for k, v in result.items():
+                total_scores[k] += sum(v)
+                total_nums[k] += len(v)
+
+        for k, v in total_scores.items():
+            if total_nums[k] > 0:
+                total_results[k] = total_scores[k] / total_nums[k] * 100
+            else:
+                total_results[k] = -1
+
+        return total_results

opencompass-my-api/build/lib/opencompass/datasets/circular.py

@@ -0,0 +1,373 @@
+import copy
+import itertools
+from typing import Callable, List, Optional, Union
+
+from datasets import Dataset, DatasetDict
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+
+from .arc import ARCDataset
+from .ceval import CEvalDataset
+from .cmmlu import CMMLUDataset
+from .commonsenseqa import commonsenseqaDataset
+from .hellaswag import hellaswagDataset_V2
+from .mmlu import MMLUDataset
+from .obqa import OBQADataset
+from .piqa import piqaDataset_V2
+from .race import RaceDataset
+from .siqa import siqaDataset_V3
+from .xiezhi import XiezhiDataset
+
+
+def get_origin_patterns(option_keys):
+    return [tuple(option_keys)]
+
+
+def get_circular_patterns(option_keys):
+    double_option_keys = option_keys + option_keys
+    circular_patterns = [
+        tuple(double_option_keys[i:i + len(option_keys)])
+        for i in range(len(option_keys))
+    ]
+    return circular_patterns
+
+
+def get_all_possible_patterns(option_keys):
+    circular_patterns = list(itertools.permutations(option_keys))
+    return circular_patterns
+
+
+class CircularDatasetMeta(type):
+    """This Meta Class is designed to transform a class that reads datasets
+    into one that supports reading datasets required for CircularEval. It
+    overloads an existing load method for the original class.
+
+    The Meta Class should possess the following attributes:
+
+    - `dataset_class` (class): The class for reading datasets, such as
+        `CEvalDataset`.
+    - `default_circular_splits` (list, optional): The default splits of the
+        dataset that need to undergo CircularEval, like ['val', 'test']. If a
+        `Dataset` is loaded originally, this field will be ignored.
+    - `default_option_keys` (list): The keys for options in the dataset, such
+        as ['A', 'B', 'C', 'D'].
+    - `default_answer_key` (str, optional): The key for answers in the dataset,
+        like 'answer'. This is an alternative to
+        `default_answer_key_switch_method`.
+    - `default_answer_key_switch_method` (function, optional): The method to
+        transform the key for answers in the dataset. This is an alternative to
+        `default_answer_key`.
+    """
+
+    @staticmethod
+    def make_circular_items(
+        origin_item,
+        circular_patterns,
+        option_keys,
+        answer_key,
+        answer_key_switch_method,
+        qid,
+    ):
+        items = []
+        for circular_pattern in circular_patterns:
+            item = copy.deepcopy(origin_item)
+            for i in range(len(option_keys)):
+                item[circular_pattern[i]] = origin_item[option_keys[i]]
+            if answer_key_switch_method is None:
+                if origin_item[answer_key] in option_keys:
+                    item[answer_key] = circular_pattern[option_keys.index(
+                        origin_item[answer_key])]
+                else:
+                    pass
+            else:
+                item = answer_key_switch_method(item, circular_pattern)
+            item['qid'] = qid
+            item['circular_pattern'] = tuple(circular_pattern)
+            items.append(item)
+        return items
+
+    @staticmethod
+    def make_circular_dataset(dataset, circular_patterns, option_keys,
+                              answer_key, answer_key_switch_method):
+        circulated_items = []
+        for i, item in enumerate(dataset):
+            item = CircularDatasetMeta.make_circular_items(
+                item,
+                circular_patterns,
+                option_keys,
+                answer_key,
+                answer_key_switch_method,
+                i,
+            )
+            circulated_items.extend(item)
+        return Dataset.from_list(circulated_items)
+
+    def make_circular(
+        dataset: Union[Dataset, DatasetDict],
+        circular_splits: Optional[List[str]] = ['test'],
+        circular_patterns: str = 'circular',
+        option_keys: List[str] = ['A', 'B', 'C', 'D'],
+        answer_key: Optional[str] = 'answer',
+        answer_key_switch_method: Optional[Callable] = None,
+    ):
+        """Transform the dataset into one that is compatible with CircularEval.
+        In CircularEval, the original multiple-choice questions with options
+        ABCD are augmented by shuffling the order of options, such as BCDA,
+        CDAB, DABC, etc. A model is considered correct only if it answers all
+        augmented questions correctly. This method effectively prevents models
+        from memorizing answers.
+
+        Args:
+        datasets: The dataset to be augmented.
+        circular_splits: List of splits to make circular. This is only
+            effective when the dataset is a DatasetDict.
+        circular_patterns: Method for circular processing, can be 'circular'
+            for single cycle or 'all_possible' for all permutations, default
+            is 'circular'.
+        option_keys: List of keys for options, default to ['A', 'B', 'C', 'D'].
+        answer_key: Key for the answer, default to 'answer'. When specified,
+            ensure that the content of answer_key is among the option_keys.
+            It is an alternative to specifying answer_key_switch_method.
+        answer_key_switch_method: Function to modify the answer_key. It is an
+            alternative to specifying answer_key.
+        """
+
+        if isinstance(circular_patterns, str):
+            if circular_patterns == 'circular':
+                circular_patterns = get_circular_patterns(option_keys)
+            elif circular_patterns == 'all_possible':
+                circular_patterns = get_all_possible_patterns(option_keys)
+            else:
+                raise ValueError(
+                    f'Unknown circular_patterns: {circular_patterns}')
+        else:
+            assert isinstance(circular_patterns, list)
+            assert all([isinstance(i, list) for i in circular_patterns])
+            # TODO: other necessary sanity checks
+            raise NotImplementedError(
+                'circular_patterns int list of list has not been tested yet')
+
+        if answer_key is None and answer_key_switch_method is None:
+            raise ValueError(
+                'answer_key and answer_key_switch_method cannot be both None')
+        if answer_key is not None and answer_key_switch_method is not None:
+            raise ValueError(
+                'either answer_key or answer_key_switch_method should be None')
+
+        if isinstance(dataset, Dataset):
+            dataset = CircularDatasetMeta.make_circular_dataset(
+                dataset,
+                circular_patterns,
+                option_keys,
+                answer_key,
+                answer_key_switch_method,
+            )
+        else:
+            assert isinstance(dataset, DatasetDict)
+            dataset_dict = {}
+            for split in dataset:
+                if circular_splits is not None and split in circular_splits:
+                    dataset_dict[
+                        split] = CircularDatasetMeta.make_circular_dataset(
+                            dataset[split],
+                            circular_patterns,
+                            option_keys,
+                            answer_key,
+                            answer_key_switch_method,
+                        )
+                else:
+                    dataset_dict[split] = dataset[split]
+            dataset = DatasetDict(dataset_dict)
+        return dataset
+
+    def __new__(cls, name, bases, dct):
+        new_cls = super().__new__(cls, name, bases, dct)
+
+        def load(cls, circular_patterns='circular', *args, **kwargs):
+            circular_splits = getattr(cls, 'default_circular_splits', None)
+            option_keys = getattr(cls, 'default_option_keys', None)
+            if 'option_keys' in kwargs:
+                option_keys = kwargs.pop('option_keys')
+            assert option_keys is not None, 'option_keys cannot be None'
+            answer_key = getattr(cls, 'default_answer_key', None)
+            if 'answer_key' in kwargs:
+                answer_key = kwargs.pop('answer_key')
+            answer_key_switch_method = getattr(
+                cls, 'default_answer_key_switch_method', None)
+            dataset = cls.dataset_class.load(*args, **kwargs)
+            return CircularDatasetMeta.make_circular(
+                dataset,
+                circular_splits,
+                circular_patterns,
+                option_keys,
+                answer_key,
+                answer_key_switch_method,
+            )
+
+        setattr(new_cls, 'load', classmethod(load))
+        return new_cls
+
+
+class CircularCEvalDataset(CEvalDataset, metaclass=CircularDatasetMeta):
+    dataset_class = CEvalDataset
+    default_circular_splits = ['val', 'test']
+    default_option_keys = ['A', 'B', 'C', 'D']
+    default_answer_key = 'answer'
+
+
+class CircularMMLUDataset(MMLUDataset, metaclass=CircularDatasetMeta):
+    dataset_class = MMLUDataset
+    default_circular_splits = ['test']
+    default_option_keys = ['A', 'B', 'C', 'D']
+    default_answer_key = 'target'
+
+
+class CircularCMMLUDataset(CMMLUDataset, metaclass=CircularDatasetMeta):
+    dataset_class = CMMLUDataset
+    default_circular_splits = ['test']
+    default_option_keys = ['A', 'B', 'C', 'D']
+    default_answer_key = 'answer'
+
+
+class CircularCSQADataset(commonsenseqaDataset, metaclass=CircularDatasetMeta):
+    dataset_class = commonsenseqaDataset
+    default_circular_splits = ['validation']
+    default_option_keys = ['A', 'B', 'C', 'D', 'E']
+    default_answer_key = 'answerKey'
+
+
+class CircularARCDataset(ARCDataset, metaclass=CircularDatasetMeta):
+    dataset_class = ARCDataset
+    default_circular_splits = None
+    default_option_keys = ['textA', 'textB', 'textC', 'textD']
+
+    def default_answer_key_switch_method(item, circular_pattern):
+        circular_pattern = tuple(i[-1] for i in circular_pattern)
+        item['answerKey'] = circular_pattern['ABCD'.index(item['answerKey'])]
+        return item
+
+
+class CircularHSWAGDataset(hellaswagDataset_V2, metaclass=CircularDatasetMeta):
+    dataset_class = hellaswagDataset_V2
+    default_circular_splits = None
+    default_option_keys = ['A', 'B', 'C', 'D']
+    default_answer_key = 'label'
+
+
+class CircularOBQADataset(OBQADataset, metaclass=CircularDatasetMeta):
+    dataset_class = OBQADataset
+    default_circular_splits = None
+    default_option_keys = ['A', 'B', 'C', 'D']
+    default_answer_key = 'answerKey'
+
+
+class CircularRaceDataset(RaceDataset, metaclass=CircularDatasetMeta):
+    dataset_class = RaceDataset
+    default_circular_splits = ['test']
+    default_option_keys = ['A', 'B', 'C', 'D']
+    default_answer_key = 'answer'
+
+
+class CircularXiezhiDataset(XiezhiDataset, metaclass=CircularDatasetMeta):
+    dataset_class = XiezhiDataset
+    default_circular_splits = None
+    default_option_keys = ['A', 'B', 'C', 'D']
+    default_answer_key = 'answer'
+
+
+class CircularsiqaDataset(siqaDataset_V3, metaclass=CircularDatasetMeta):
+    dataset_class = siqaDataset_V3
+    default_circular_splits = ['validation']
+    default_option_keys = ['A', 'B', 'C']
+    default_answer_key = 'answer'
+
+
+class CircularpiqaDataset(piqaDataset_V2, metaclass=CircularDatasetMeta):
+    dataset_class = piqaDataset_V2
+    default_circular_splits = ['validation']
+    default_option_keys = ['sol1', 'sol2']
+
+    def default_answer_key_switch_method(item, circular_pattern):
+        circular_pattern = tuple(int(i[-1]) - 1 for i in circular_pattern)
+        item['answer'] = 'AB'[circular_pattern['AB'.index(item['answer'])]]
+        return item
+
+
+class CircularEvaluator(BaseEvaluator):
+    """This Evaluator assesses datasets post-Circular processing, generating
+    the following evaluation metrics:
+
+    - `acc_{origin|circular|all_possible}`: Treats each question with shuffled
+        answer options as separate, calculating accuracy.
+    - `perf_{origin|circular|all_possible}`: According Circular logic, a
+        question is considered correct only if all its variations with shuffled
+        options are answered correctly, calculating accuracy. perf is short for
+        perfect.
+    - `more_{num}_{origin|circular|all_possible}`: According to Circular logic,
+        a question is considered correct only if the number of its variations
+        answered correctly is greater than or equal to `num`, calculating
+        accuracy.
+
+    Note that when the `all_possible` method is used to shuffle option order,
+        it naturally includes the Circular method, and its metrics will also be
+        output.
+
+    Args:
+        circular_pattern: The method of shuffling options, either 'circular' or
+            'all_possible', defaulting to 'circular'.
+    """
+
+    def __init__(self, circular_pattern='circular'):
+        super().__init__()
+        self.circular_pattern = circular_pattern
+
+    def score(self, predictions, references, test_set):
+        circular_patterns = {}
+        circular_patterns['origin'] = get_origin_patterns(
+            test_set[0]['circular_pattern'])
+        circular_patterns['circular'] = get_circular_patterns(
+            test_set[0]['circular_pattern'])
+        if self.circular_pattern == 'all_possible':
+            circular_patterns['all_possible'] = get_all_possible_patterns(
+                test_set[0]['circular_pattern'])
+
+        metrics = {}
+        tmp_metrics = {}
+        tmp_metrics.update({f'correct_{k}': 0 for k in circular_patterns})
+        tmp_metrics.update({f'count_{k}': 0 for k in circular_patterns})
+        # calculate the original accuracy
+        for pred, refr, origin_item in zip(predictions, references, test_set):
+            circular_pattern = origin_item['circular_pattern']
+            for k in circular_patterns:
+                if tuple(circular_pattern) in circular_patterns[k]:
+                    tmp_metrics[f'correct_{k}'] += 1 if pred == refr else 0
+                    tmp_metrics[f'count_{k}'] += 1
+
+        for k in circular_patterns:
+            metrics[f'acc_{k}'] = (tmp_metrics[f'correct_{k}'] /
+                                   tmp_metrics[f'count_{k}'] * 100)
+
+        # calculate the circular accuracy
+        _details = {k: {} for k in circular_patterns}
+        for pred, refr, origin_item in zip(predictions, references, test_set):
+            index = origin_item['qid']
+            circular_pattern = origin_item['circular_pattern']
+            for k in circular_patterns:
+                if tuple(circular_pattern) in circular_patterns[k]:
+                    _details[k].setdefault(
+                        index, []).append(True if pred == refr else False)
+        for k in _details:
+            _details[k] = {
+                index: sum(_details[k][index])
+                for index in _details[k]
+            }
+        for k in _details:
+            for j in range(1, len(circular_patterns[k]) + 1):
+                count = sum([_details[k][index] >= j for index in _details[k]])
+                total = len(_details[k])
+                if j != len(circular_patterns[k]):
+                    metrics[f'more_{j}_{k}'] = count / total * 100
+                else:
+                    metrics[f'perf_{k}'] = count / total * 100
+
+        return metrics