region:us
---
configs:
- config_name: default
  data_files:
  - split: memories.deduped.12b
    path: data/memories.deduped.12b-*
  - split: memories.duped.12b
    path: data/memories.duped.12b-*
  - split: memories.duped.6.9b
    path: data/memories.duped.6.9b-*
  - split: pile.duped.6.9b
    path: data/pile.duped.6.9b-*
  - split: memories.duped.70m
    path: data/memories.duped.70m-*
  - split: memories.duped.160m
    path: data/memories.duped.160m-*
  - split: memories.duped.410m
    path: data/memories.duped.410m-*
  - split: pile.duped.70m
    path: data/pile.duped.70m-*
  - split: pile.duped.160m
    path: data/pile.duped.160m-*
  - split: pile.duped.410m
    path: data/pile.duped.410m-*
  - split: memories.duped.1.4b
    path: data/memories.duped.1.4b-*
  - split: memories.duped.1b
    path: data/memories.duped.1b-*
  - split: memories.duped.2.8b
    path: data/memories.duped.2.8b-*
  - split: pile.duped.1.4b
    path: data/pile.duped.1.4b-*
  - split: pile.duped.1b
    path: data/pile.duped.1b-*
  - split: pile.duped.2.8b
    path: data/pile.duped.2.8b-*
  - split: pile.duped.12b
    path: data/pile.duped.12b-*
  - split: memories.deduped.70m
    path: data/memories.deduped.70m-*
  - split: memories.deduped.160m
    path: data/memories.deduped.160m-*
  - split: memories.deduped.410m
    path: data/memories.deduped.410m-*
  - split: pile.deduped.70m
    path: data/pile.deduped.70m-*
  - split: pile.deduped.160m
    path: data/pile.deduped.160m-*
  - split: pile.deduped.410m
    path: data/pile.deduped.410m-*
  - split: memories.deduped.6.9b
    path: data/memories.deduped.6.9b-*
  - split: pile.deduped.6.9b
    path: data/pile.deduped.6.9b-*
  - split: pile.deduped.12b
    path: data/pile.deduped.12b-*
  - split: memories.deduped.2.8b
    path: data/memories.deduped.2.8b-*
  - split: pile.deduped.2.8b
    path: data/pile.deduped.2.8b-*
  - split: memories.deduped.1.4b
    path: data/memories.deduped.1.4b-*
  - split: memories.deduped.1b
    path: data/memories.deduped.1b-*
  - split: pile.deduped.1.4b
    path: data/pile.deduped.1.4b-*
  - split: pile.deduped.1b
    path: data/pile.deduped.1b-*
dataset_info:
  features:
  - name: index
    dtype: int32
  - name: prompt_perplexity
    dtype: float32
  - name: generation_perplexity
    dtype: float32
  - name: sequence_perplexity
    dtype: float32
  splits:
  - name: memories.deduped.12b
    num_bytes: 29939456
    num_examples: 1871216
  - name: memories.duped.12b
    num_bytes: 38117248
    num_examples: 2382328
  - name: memories.duped.6.9b
    num_bytes: 33935616
    num_examples: 2120976
  - name: pile.duped.6.9b
    num_bytes: 80000000
    num_examples: 5000000
  - name: memories.duped.70m
    num_bytes: 7423248
    num_examples: 463953
  - name: memories.duped.160m
    num_bytes: 11034768
    num_examples: 689673
  - name: memories.duped.410m
    num_bytes: 15525456
    num_examples: 970341
  - name: pile.duped.70m
    num_bytes: 80000000
    num_examples: 5000000
  - name: pile.duped.160m
    num_bytes: 80000000
    num_examples: 5000000
  - name: pile.duped.410m
    num_bytes: 80000000
    num_examples: 5000000
  - name: memories.duped.1.4b
    num_bytes: 21979552
    num_examples: 1373722
  - name: memories.duped.1b
    num_bytes: 20098256
    num_examples: 1256141
  - name: memories.duped.2.8b
    num_bytes: 26801232
    num_examples: 1675077
  - name: pile.duped.1.4b
    num_bytes: 80000000
    num_examples: 5000000
  - name: pile.duped.1b
    num_bytes: 80000000
    num_examples: 5000000
  - name: pile.duped.2.8b
    num_bytes: 80000000
    num_examples: 5000000
  - name: pile.duped.12b
    num_bytes: 80000000
    num_examples: 5000000
  - name: memories.deduped.70m
    num_bytes: 6583168
    num_examples: 411448
  - name: memories.deduped.160m
    num_bytes: 9299120
    num_examples: 581195
  - name: memories.deduped.410m
    num_bytes: 12976624
    num_examples: 811039
  - name: pile.deduped.70m
    num_bytes: 80000000
    num_examples: 5000000
  - name: pile.deduped.160m
    num_bytes: 80000000
    num_examples: 5000000
  - name: pile.deduped.410m
    num_bytes: 80000000
    num_examples: 5000000
  - name: memories.deduped.6.9b
    num_bytes: 26884704
    num_examples: 1680294
  - name: pile.deduped.6.9b
    num_bytes: 80000000
    num_examples: 5000000
  - name: pile.deduped.12b
    num_bytes: 80000000
    num_examples: 5000000
  - name: memories.deduped.2.8b
    num_bytes: 21683376
    num_examples: 1355211
  - name: pile.deduped.2.8b
    num_bytes: 80000000
    num_examples: 5000000
  - name: memories.deduped.1.4b
    num_bytes: 16769552
    num_examples: 1048097
  - name: memories.deduped.1b
    num_bytes: 16525840
    num_examples: 1032865
  - name: pile.deduped.1.4b
    num_bytes: 80000000
    num_examples: 5000000
  - name: pile.deduped.1b
    num_bytes: 80000000
    num_examples: 5000000
  download_size: 1891778367
  dataset_size: 1595577216
---
# Dataset Card for "pythia-semantic-memorization-perplexities"

[More Information needed](https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)