region:us
---
dataset_info:
  features:
  - name: id
    dtype: int64
  - name: text
    dtype: string
  - name: meta
    struct:
    - name: warc_headers
      struct:
      - name: warc-record-id
        dtype: string
      - name: warc-date
        dtype: string
      - name: content-type
        dtype: string
      - name: content-length
        dtype: int32
      - name: warc-type
        dtype: string
      - name: warc-identified-content-language
        dtype: string
      - name: warc-refers-to
        dtype: string
      - name: warc-target-uri
        dtype: string
      - name: warc-block-digest
        dtype: string
    - name: identification
      struct:
      - name: label
        dtype: string
      - name: prob
        dtype: float32
    - name: annotations
      sequence: string
    - name: line_identifications
      list:
      - name: label
        dtype: string
      - name: prob
        dtype: float32
  - name: perplexity_score
    dtype: float64
  - name: text_length
    dtype: int64
  - name: url
    dtype: string
  - name: domain
    dtype: string
  - name: dup_ratio
    dtype: float64
  - name: pairs
    sequence:
      sequence: int64
  - name: repetitions
    sequence: binary
  - name: included_in_dedup
    dtype: bool
  - name: cluster
    sequence: int64
  splits:
  - name: train
    num_bytes: 3188486875748
    num_examples: 431992659
  download_size: 419397499659
  dataset_size: 3188486875748
---

this is the one where we build the suffix array for 25% Oscar and only deduplicate that part - by deduplication I mean removing any document which has an at least 100-char span overlapping with another document in the 25% chunk. This is very strict and preserves only about 20 million documents, so less then 5% of the full Oscar.