license:mit region:us
---
license: mit
configs:
- config_name: default
  data_files:
  - split: pile_deduped_70m
    path: data/pile_deduped_70m-*
  - split: memories_deduped_70m
    path: data/memories_deduped_70m-*
  - split: pile_deduped_160m
    path: data/pile_deduped_160m-*
  - split: memories_deduped_160m
    path: data/memories_deduped_160m-*
  - split: pile_deduped_410m
    path: data/pile_deduped_410m-*
  - split: memories_deduped_410m
    path: data/memories_deduped_410m-*
  - split: pile_deduped_1b
    path: data/pile_deduped_1b-*
  - split: memories_deduped_1b
    path: data/memories_deduped_1b-*
  - split: pile_deduped_1.4b
    path: data/pile_deduped_1.4b-*
  - split: memories_deduped_1.4b
    path: data/memories_deduped_1.4b-*
  - split: pile_deduped_2.8b
    path: data/pile_deduped_2.8b-*
  - split: memories_deduped_2.8b
    path: data/memories_deduped_2.8b-*
  - split: pile_deduped_6.9b
    path: data/pile_deduped_6.9b-*
  - split: memories_deduped_6.9b
    path: data/memories_deduped_6.9b-*
  - split: pile_deduped_12b
    path: data/pile_deduped_12b-*
  - split: memories_deduped_12b
    path: data/memories_deduped_12b-*
  - split: pile_duped_70m
    path: data/pile_duped_70m-*
  - split: memories_duped_70m
    path: data/memories_duped_70m-*
  - split: pile_duped_160m
    path: data/pile_duped_160m-*
  - split: memories_duped_160m
    path: data/memories_duped_160m-*
  - split: pile_duped_410m
    path: data/pile_duped_410m-*
  - split: memories_duped_410m
    path: data/memories_duped_410m-*
  - split: pile_duped_1b
    path: data/pile_duped_1b-*
  - split: memories_duped_1b
    path: data/memories_duped_1b-*
  - split: pile_duped_1.4b
    path: data/pile_duped_1.4b-*
  - split: memories_duped_1.4b
    path: data/memories_duped_1.4b-*
  - split: pile_duped_2.8b
    path: data/pile_duped_2.8b-*
  - split: memories_duped_2.8b
    path: data/memories_duped_2.8b-*
  - split: pile_duped_6.9b
    path: data/pile_duped_6.9b-*
  - split: memories_duped_6.9b
    path: data/memories_duped_6.9b-*
  - split: pile_duped_12b
    path: data/pile_duped_12b-*
  - split: memories_duped_12b
    path: data/memories_duped_12b-*
dataset_info:
  features:
  - name: sequence_id
    dtype: int64
  - name: tokens
    sequence: int64
  - name: memorized_frequencies
    sequence: int64
  - name: non_memorized_frequencies
    sequence: int64
  - name: memorization_score
    dtype: float64
  - name: sequence_frequency
    dtype: int64
  splits:
  - name: pile_deduped_70m
    num_bytes: 7860000000
    num_examples: 5000000
  - name: memories_deduped_70m
    num_bytes: 646796256
    num_examples: 411448
  - name: pile_deduped_160m
    num_bytes: 7860000000
    num_examples: 5000000
  - name: memories_deduped_160m
    num_bytes: 913638540
    num_examples: 581195
  - name: pile_deduped_410m
    num_bytes: 7860000000
    num_examples: 5000000
  - name: memories_deduped_410m
    num_bytes: 1274953308
    num_examples: 811039
  - name: pile_deduped_1b
    num_bytes: 7860000000
    num_examples: 5000000
  - name: memories_deduped_1b
    num_bytes: 1623663780
    num_examples: 1032865
  - name: pile_deduped_1.4b
    num_bytes: 7860000000
    num_examples: 5000000
  - name: memories_deduped_1.4b
    num_bytes: 1647608484
    num_examples: 1048097
  - name: pile_deduped_2.8b
    num_bytes: 7860000000
    num_examples: 5000000
  - name: memories_deduped_2.8b
    num_bytes: 2130391692
    num_examples: 1355211
  - name: pile_deduped_6.9b
    num_bytes: 7860000000
    num_examples: 5000000
  - name: memories_deduped_6.9b
    num_bytes: 2641422168
    num_examples: 1680294
  - name: pile_deduped_12b
    num_bytes: 7860000000
    num_examples: 5000000
  - name: memories_deduped_12b
    num_bytes: 2941549980
    num_examples: 1871215
  - name: pile_duped_70m
    num_bytes: 7860000000
    num_examples: 5000000
  - name: memories_duped_70m
    num_bytes: 729334116
    num_examples: 463953
  - name: pile_duped_160m
    num_bytes: 7860000000
    num_examples: 5000000
  - name: memories_duped_160m
    num_bytes: 1084165956
    num_examples: 689673
  - name: pile_duped_410m
    num_bytes: 7860000000
    num_examples: 5000000
  - name: memories_duped_410m
    num_bytes: 1525376052
    num_examples: 970341
  - name: pile_duped_1b
    num_bytes: 7860000000
    num_examples: 5000000
  - name: memories_duped_1b
    num_bytes: 1974653652
    num_examples: 1256141
  - name: pile_duped_1.4b
    num_bytes: 7860000000
    num_examples: 5000000
  - name: memories_duped_1.4b
    num_bytes: 2159490984
    num_examples: 1373722
  - name: pile_duped_2.8b
    num_bytes: 7860000000
    num_examples: 5000000
  - name: memories_duped_2.8b
    num_bytes: 2633221044
    num_examples: 1675077
  - name: pile_duped_6.9b
    num_bytes: 7860000000
    num_examples: 5000000
  - name: memories_duped_6.9b
    num_bytes: 3334163268
    num_examples: 2120969
  - name: pile_duped_12b
    num_bytes: 7860000000
    num_examples: 5000000
  - name: memories_duped_12b
    num_bytes: 3745016472
    num_examples: 2382326
  download_size: 11256676441
  dataset_size: 156765445752
---

This dataset is a partial computation of metrics (memorized token frequencies, non-memorized token frequencies, sequence frequencies) needed for [research](https://github.com/EleutherAI/semantic-memorization).