region:us
---
dataset_info:
  features:
  - name: task_name
    dtype: string
  - name: id
    dtype: string
  - name: definition
    dtype: string
  - name: inputs
    dtype: string
  - name: targets
    dtype: string
  - name: input_ids
    sequence: int32
  - name: attention_mask
    sequence: int8
  - name: labels
    sequence: int64
  splits:
  - name: train
    num_bytes: 30103878026
    num_examples: 6164188
  - name: validation
    num_bytes: 10504992
    num_examples: 5995
  - name: test
    num_bytes: 3937289042
    num_examples: 982285
  download_size: 10233494905
  dataset_size: 34051672060
---
# Dataset Card for "natural-instructions-tokenized"

Here is the script used to tokenize the dataset:

```python
import multiprocessing
from typing import Union

from datasets import DatasetDict, load_dataset

from transformers import LlamaTokenizer

# Find your available cores
num_cores = multiprocessing.cpu_count()

cutoff_len = 2048

tokenizer = LlamaTokenizer.from_pretrained("chainyo/alpaca-lora-7b")
tokenizer.padding_side = "left"
tokenizer.pad_token_id = (0)

prompt_template = {
    "prompt": "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:\n",
    "response": "### Response:"    
}

class Prompter(object):
    __slots__ = ("template", "_verbose")

    def __init__(self, verbose: bool = False):
        self._verbose = verbose

    def generate_prompt(
        self,
        definition: str,
        inputs: str,
        targets: Union[None, str] = None,
    ) -> str:
        """Generate a prompt from instruction and input."""
        res = prompt_template["prompt"].format(
            instruction=definition, input=inputs
        )

        if targets:
            res = f"{res}{targets}"

        return res

    def get_response(self, output: str) -> str:
        return output.split(self.template["response"])[1].strip()


prompter = Prompter()

def tokenize(prompt, add_eos_token=True):
    # there's probably a way to do this with the tokenizer settings
    # but again, gotta move fast
    result = tokenizer(
        prompt,
        truncation=True,
        max_length=cutoff_len,
        padding=False,
        return_tensors=None,
    )
    if (
        result["input_ids"][-1] != tokenizer.eos_token_id
        and len(result["input_ids"]) < cutoff_len
        and add_eos_token
    ):
        result["input_ids"].append(tokenizer.eos_token_id)
        result["attention_mask"].append(1)

    result["labels"] = result["input_ids"].copy()

    return result

def generate_and_tokenize_prompt(data_point):
    full_prompt = prompter.generate_prompt(
        data_point["definition"],
        data_point["inputs"],
        data_point["targets"],
    )
    tokenized_full_prompt = tokenize(full_prompt)

    return tokenized_full_prompt


data_path = "Muennighoff/natural-instructions"
data = load_dataset(data_path)

train_data = data["train"].shuffle().map(generate_and_tokenize_prompt, num_proc=num_cores)
val_data = data["validation"].map(generate_and_tokenize_prompt, num_proc=num_cores)
test_data = data["test"].map(generate_and_tokenize_prompt, num_proc=num_cores)

tokenized_dataset = DatasetDict({"train": train_data, "validation": val_data, "test": test_data})
tokenized_dataset.save_to_disk("natural-instructions-tokenized")
```