region:us

Creation (Copied & adapted from https://github.com/stanford-crfm/helm/blob/0eaaa62a2263ddb94e9850ee629423b010f57e4a/src/helm/benchmark/scenarios/babi_qa_scenario.py):

```python
!wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz
!tar -xf tasks_1-20_v1-2.tar.gz
import json
from typing import List

tasks = list(range(1, 20))
splits = ["train", "valid", "test"]

def process_path(path: str) -> str:
    """Turn a path string (task 19) from the original format 's,w' to a verbal model-friendly format 'south west'"""
    steps: List[str] = path.split(",")
    directions = {"s": "south", "n": "north", "e": "east", "w": "west"}
    path = " ".join([directions[step] for step in steps])
    return path

for split in splits:
    with open(f"babi_{split}.jsonl", "w") as f_base:
        for task in tasks:
            split_path: str = f"./tasks_1-20_v1-2/en-valid/qa{task}_{split}.txt"
            with open(split_path, "r") as f:
                facts = list(f)
                story: List[str] = []
                for fact in facts:
                    fid = int(fact.split(" ")[0])
                    if fid == 1:
                        story = []
                    fact = " ".join(fact.split(" ")[1:])
                    is_question = "?" in fact
                    if is_question:
                        question, answer = fact.split("\t")[:2]
                        question, answer = question.strip(), answer.strip()
                        # All tasks except task 19 have a verbal single-word answer (e.g. kitchen, apple, yes).
                        # Task 19 (path finding) has a non verbal answer format (
                        if task == 19:
                            answer = process_path(answer)

                        f_base.write(json.dumps({
                            "passage": "".join(story),
                            "question": question,
                            "answer": answer,
                            "task": task,
                        }) + "\n")
                        if "?" in story:
                            print("STORY", "".join(story))
                    else:
                        story.append(fact)   
```