File size: 2,345 Bytes
256a159 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 |
from opencompass.openicl.icl_prompt_template import PromptTemplate
from opencompass.openicl.icl_retriever import ZeroRetriever
from opencompass.openicl.icl_inferencer import AgentInferencer
from opencompass.datasets import DS1000Dataset_Interperter, DS1000InterpreterEvaluator
ds1000_example = """
In the following task, you should generate code with one assertion to testify the correctness of your code.
Example:
<HUMAN>Problem:
How do I get the dimensions of an array? For instance, this is (2, 2):
a = np.array([[1,2],[3,4]])
<ASSISTANT>{thought} In Python, Numpy provides a method called `shape` which helps to get the dimensions of an array.
{action} PythonInterpreter
{action_input}
```python
import numpy as np
def solution(x):
# Convert to np.ndarray
x = np.array(x)
# Getting the dimensions of the array
dimensions = x.shape
return dimensions
assert solution([[1,2],[3,4]]) == (2, 2)
```
<SYSTEM>{response}True
<ASSISTANT> {thought} By running this code, you can get the dimensions of an array.
{finish}
```python
import numpy as np
def solution(x):
# Convert to np.ndarray
x = np.array(x)
# Getting the dimensions of the array
dimensions = x.shape
return dimensions
```
"""
ds1000_reader_cfg = dict(
input_columns=["prompt"],
output_column="test_column",
train_split="test",
test_split="test",
)
ds1000_infer_cfg = dict(
prompt_template=dict(
type=PromptTemplate,
template="""{prompt}""",
),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=AgentInferencer, example=ds1000_example),
)
ds1000_eval_cfg = dict(
evaluator=dict(type=DS1000InterpreterEvaluator),
pred_role="BOT",
)
# The DS-1000 dataset can be downloaded from
# https://github.com/HKUNLP/DS-1000/blob/main/ds1000_data.zip
# Matplotlib cannot fit this setting
ds1000_datasets = [
dict(
abbr=f"ds1000_{lib}",
type=DS1000Dataset_Interperter, # bustm share the same format with AFQMC
path="./data/ds1000_data/",
libs=f"{lib}",
reader_cfg=ds1000_reader_cfg,
infer_cfg=ds1000_infer_cfg,
eval_cfg=ds1000_eval_cfg,
)
for lib in [
"Pandas",
"Numpy",
# 'Tensorflow', # error using tensorflow, skipped temporarily
"Scipy",
"Sklearn",
"Pytorch",
]
]
|