region:us
---
dataset_info:
  features:
  - name: sequence_id
    dtype: int64
  - name: text
    dtype: string
  - name: sequence_duplicates
    dtype: int64
  - name: max_frequency
    dtype: int64
  - name: avg_frequency
    dtype: float64
  - name: min_frequency
    dtype: int64
  - name: median_frequency
    dtype: float64
  - name: p25_frequency
    dtype: int64
  - name: p75_frequency
    dtype: int64
  - name: frequencies
    sequence: int64
  - name: is_incrementing
    dtype: bool
  - name: tokens
    sequence: int64
  - name: repeating_offset
    dtype: int32
  - name: num_repeating
    dtype: int32
  - name: smallest_repeating_chunk
    sequence: int64
  - name: memorization_score
    dtype: float64
  - name: templating_frequency_0.9
    dtype: int64
  - name: templating_frequency_0.8
    dtype: int64
  - name: prompt_perplexity
    dtype: float32
  - name: generation_perplexity
    dtype: float32
  - name: sequence_perplexity
    dtype: float32
  splits:
  - name: memories.duped.70m
    num_bytes: 648141277
    num_examples: 463953
  - name: memories.duped.160m
    num_bytes: 955903849
    num_examples: 689673
  - name: memories.duped.410m
    num_bytes: 1337555782
    num_examples: 970341
  - name: memories.duped.1b
    num_bytes: 1725540452
    num_examples: 1256141
  - name: memories.duped.1.4b
    num_bytes: 1884519155
    num_examples: 1373722
  - name: memories.duped.2.8b
    num_bytes: 2292743123
    num_examples: 1675077
  - name: memories.duped.6.9b
    num_bytes: 2898035658
    num_examples: 2120976
  - name: memories.duped.12b
    num_bytes: 3252649684
    num_examples: 2382328
  - name: memories.deduped.70m
    num_bytes: 576211560
    num_examples: 411448
  - name: memories.deduped.160m
    num_bytes: 809545073
    num_examples: 581195
  - name: memories.deduped.410m
    num_bytes: 1126006111
    num_examples: 811039
  - name: memories.deduped.1b
    num_bytes: 1430399436
    num_examples: 1032865
  - name: memories.deduped.1.4b
    num_bytes: 1450336662
    num_examples: 1048097
  - name: memories.deduped.2.8b
    num_bytes: 1871907415
    num_examples: 1355211
  - name: memories.deduped.6.9b
    num_bytes: 2319039796
    num_examples: 1680294
  - name: memories.deduped.12b
    num_bytes: 2581349436
    num_examples: 1871216
  download_size: 9223426756
  dataset_size: 27159884469
---
# Dataset Card for "memories-semantic-memorization-filter-results"

[More Information needed](https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)