region:us
---
dataset_info:
  features:
  - name: sequence_id
    dtype: int64
  - name: text
    dtype: string
  - name: sequence_duplicates
    dtype: int64
  - name: max_frequency
    dtype: int64
  - name: avg_frequency
    dtype: float64
  - name: min_frequency
    dtype: int64
  - name: median_frequency
    dtype: float64
  - name: p25_frequency
    dtype: int64
  - name: p75_frequency
    dtype: int64
  - name: frequencies
    sequence: int64
  - name: is_incrementing
    dtype: bool
  - name: tokens
    sequence: int64
  - name: repeating_offset
    dtype: int32
  - name: num_repeating
    dtype: int32
  - name: smallest_repeating_chunk
    sequence: int64
  - name: memorization_score
    dtype: float64
  - name: templating_frequency_0.9
    dtype: int64
  - name: templating_frequency_0.8
    dtype: int64
  - name: prompt_perplexity
    dtype: float32
  - name: generation_perplexity
    dtype: float32
  - name: sequence_perplexity
    dtype: float32
  splits:
  - name: pile.duped.70m
    num_bytes: 7003348430
    num_examples: 5000000
  - name: pile.duped.160m
    num_bytes: 7003348430
    num_examples: 5000000
  - name: pile.duped.410m
    num_bytes: 7003348430
    num_examples: 5000000
  - name: pile.duped.1b
    num_bytes: 7003348430
    num_examples: 5000000
  - name: pile.duped.1.4b
    num_bytes: 7003348430
    num_examples: 5000000
  - name: pile.duped.2.8b
    num_bytes: 7003348430
    num_examples: 5000000
  - name: pile.duped.6.9b
    num_bytes: 7003348430
    num_examples: 5000000
  - name: pile.duped.12b
    num_bytes: 7003348430
    num_examples: 5000000
  - name: pile.deduped.70m
    num_bytes: 7013409756
    num_examples: 5000000
  - name: pile.deduped.160m
    num_bytes: 7013409756
    num_examples: 5000000
  - name: pile.deduped.410m
    num_bytes: 7013409756
    num_examples: 5000000
  - name: pile.deduped.1b
    num_bytes: 7013409756
    num_examples: 5000000
  - name: pile.deduped.1.4b
    num_bytes: 7013409756
    num_examples: 5000000
  - name: pile.deduped.2.8b
    num_bytes: 7013409756
    num_examples: 5000000
  - name: pile.deduped.6.9b
    num_bytes: 7013409756
    num_examples: 5000000
  - name: pile.deduped.12b
    num_bytes: 7013409756
    num_examples: 5000000
  download_size: 48107269588
  dataset_size: 112134065488
---
# Dataset Card for "pile-semantic-memorization-filter-results"

[More Information needed](https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)