task_categories:text-generation task_ids:dialogue-modeling task_ids:language-modeling annotations_creators:no-annotation language_creators:machine-generated multilinguality:monolingual size_categories:1B<n<10B language:en reddit social-media arxiv:2001.08435 region:us
---
dataset_info:
  features:
  - name: allow_live_comments
    dtype: string
  - name: archived
    dtype: string
  - name: author
    dtype: string
  - name: author_fullname
    dtype: string
  - name: banned_by
    dtype: string
  - name: category
    dtype: string
  - name: content_categories
    dtype: string
  - name: contest_mode
    dtype: string
  - name: created_utc
    dtype: string
  - name: discussion_type
    dtype: string
  - name: distinguished
    dtype: string
  - name: domain
    dtype: string
  - name: edited
    dtype: string
  - name: gilded
    dtype: string
  - name: hidden
    dtype: string
  - name: hide_score
    dtype: string
  - name: id
    dtype: string
  - name: is_created_from_ads_ui
    dtype: string
  - name: is_crosspostable
    dtype: string
  - name: is_meta
    dtype: string
  - name: is_original_content
    dtype: string
  - name: is_reddit_media_domain
    dtype: string
  - name: is_robot_indexable
    dtype: string
  - name: is_self
    dtype: string
  - name: is_video
    dtype: string
  - name: locked
    dtype: string
  - name: media
    dtype: string
  - name: media_embed
    dtype: string
  - name: media_only
    dtype: string
  - name: name
    dtype: string
  - name: no_follow
    dtype: string
  - name: num_comments
    dtype: string
  - name: num_crossposts
    dtype: string
  - name: over_18
    dtype: string
  - name: parent_whitelist_status
    dtype: string
  - name: permalink
    dtype: string
  - name: pinned
    dtype: string
  - name: post_hint
    dtype: string
  - name: pwls
    dtype: string
  - name: quarantine
    dtype: string
  - name: removed_by
    dtype: string
  - name: removed_by_category
    dtype: string
  - name: retrieved_on
    dtype: string
  - name: score
    dtype: string
  - name: secure_media
    dtype: string
  - name: secure_media_embed
    dtype: string
  - name: selftext
    dtype: string
  - name: send_replies
    dtype: string
  - name: spoiler
    dtype: string
  - name: stickied
    dtype: string
  - name: subreddit_id
    dtype: string
  - name: subreddit_name_prefixed
    dtype: string
  - name: subreddit_subscribers
    dtype: string
  - name: subreddit_type
    dtype: string
  - name: suggested_sort
    dtype: string
  - name: title
    dtype: string
  - name: top_awarded_type
    dtype: string
  - name: total_awards_received
    dtype: string
  - name: treatment_tags
    dtype: string
  - name: upvote_ratio
    dtype: string
  - name: url
    dtype: string
  - name: url_overridden_by_dest
    dtype: string
  - name: view_count
    dtype: string
  - name: whitelist_status
    dtype: string
  - name: wls
    dtype: string
  splits:
  - name: tifu
    num_bytes: 711926746
    num_examples: 526283
  - name: explainlikeimfive
    num_bytes: 1407570925
    num_examples: 1811324
  - name: WritingPrompts
    num_bytes: 883683696
    num_examples: 1001358
  - name: changemyview
    num_bytes: 366049867
    num_examples: 257332
  - name: LifeProTips
    num_bytes: 596724168
    num_examples: 715494
  - name: todayilearned
    num_bytes: 1882122179
    num_examples: 2153849
  - name: science
    num_bytes: 675817380
    num_examples: 872768
  - name: askscience
    num_bytes: 1180347707
    num_examples: 1562708
  - name: ifyoulikeblank
    num_bytes: 248876237
    num_examples: 221368
  - name: Foodforthought
    num_bytes: 56817554
    num_examples: 70647
  - name: IWantToLearn
    num_bytes: 97666128
    num_examples: 103347
  - name: bestof
    num_bytes: 230879506
    num_examples: 341029
  - name: IAmA
    num_bytes: 375534116
    num_examples: 436003
  - name: socialskills
    num_bytes: 327412682
    num_examples: 260354
  - name: relationship_advice
    num_bytes: 5050087947
    num_examples: 3284961
  - name: philosophy
    num_bytes: 230221165
    num_examples: 212792
  - name: YouShouldKnow
    num_bytes: 87706881
    num_examples: 94635
  - name: history
    num_bytes: 295389153
    num_examples: 284318
  - name: books
    num_bytes: 635450859
    num_examples: 692807
  - name: Showerthoughts
    num_bytes: 4859309870
    num_examples: 6358205
  - name: personalfinance
    num_bytes: 1813984142
    num_examples: 1347837
  - name: buildapc
    num_bytes: 4754190700
    num_examples: 3030207
  - name: EatCheapAndHealthy
    num_bytes: 95544413
    num_examples: 79694
  - name: boardgames
    num_bytes: 379980593
    num_examples: 287493
  - name: malefashionadvice
    num_bytes: 523741819
    num_examples: 548587
  - name: femalefashionadvice
    num_bytes: 131338068
    num_examples: 131110
  - name: scifi
    num_bytes: 148283250
    num_examples: 134568
  - name: Fantasy
    num_bytes: 265612464
    num_examples: 175866
  - name: Games
    num_bytes: 1112497898
    num_examples: 830997
  - name: bodyweightfitness
    num_bytes: 154845910
    num_examples: 144829
  - name: SkincareAddiction
    num_bytes: 908265410
    num_examples: 890421
  - name: podcasts
    num_bytes: 114495922
    num_examples: 113707
  - name: suggestmeabook
    num_bytes: 307022597
    num_examples: 300601
  - name: AskHistorians
    num_bytes: 586939915
    num_examples: 592242
  - name: gaming
    num_bytes: 7306865977
    num_examples: 6418305
  - name: DIY
    num_bytes: 612049815
    num_examples: 505769
  - name: mildlyinteresting
    num_bytes: 1497282377
    num_examples: 1971187
  - name: sports
    num_bytes: 866461524
    num_examples: 783890
  - name: space
    num_bytes: 413125181
    num_examples: 415629
  - name: gadgets
    num_bytes: 242359652
    num_examples: 284487
  - name: Documentaries
    num_bytes: 658519015
    num_examples: 300935
  - name: GetMotivated
    num_bytes: 458864553
    num_examples: 395894
  - name: UpliftingNews
    num_bytes: 294091853
    num_examples: 285339
  - name: technology
    num_bytes: 1562501874
    num_examples: 2112572
  - name: Fitness
    num_bytes: 939461866
    num_examples: 1035109
  - name: travel
    num_bytes: 988622317
    num_examples: 1012452
  - name: lifehacks
    num_bytes: 124628404
    num_examples: 116871
  - name: Damnthatsinteresting
    num_bytes: 536680874
    num_examples: 397143
  - name: gardening
    num_bytes: 652169745
    num_examples: 723267
  - name: programming
    num_bytes: 455470198
    num_examples: 571221
  download_size: 15928530968
  dataset_size: 49105493092
annotations_creators:
- no-annotation
language:
- en
language_creators:
- machine-generated
license: []
multilinguality:
- monolingual
pretty_name: Reddit submissions
size_categories:
- 1B<n<10B
source_datasets: []
tags:
- reddit
- social-media
task_categories:
- text-generation
task_ids:
- dialogue-modeling
- language-modeling
---

# Dataset Card for "REDDIT_submissions"
## Dataset Description

- **Homepage:**
- **Paper: https://arxiv.org/abs/2001.08435**

### Dataset Summary
Submissions of 50 high-quality subreddits, extracted from the REDDIT PushShift data dumps (from 2006 to Jan 2023).

### Supported Tasks
These submissions can be used for text generation and language modeling, as well as dialogue modeling.

## Dataset Structure
### Data Splits
Each split corresponds to a specific subreddit in the following list: "tifu", "explainlikeimfive", "WritingPrompts", "changemyview", "LifeProTips", "todayilearned", "science", "askscience", "ifyoulikeblank", "Foodforthought", "IWantToLearn", "bestof", "IAmA", "socialskills", "relationship_advice", "philosophy", "YouShouldKnow", "history", "books", "Showerthoughts", "personalfinance", "buildapc", "EatCheapAndHealthy", "boardgames", "malefashionadvice", "femalefashionadvice", "scifi", "Fantasy", "Games", "bodyweightfitness", "SkincareAddiction", "podcasts", "suggestmeabook", "AskHistorians", "gaming", "DIY", "mildlyinteresting", "sports", "space", "gadgets", "Documentaries", "GetMotivated", "UpliftingNews", "technology", "Fitness", "travel", "lifehacks", "Damnthatsinteresting", "gardening", "programming"

## Dataset Creation

### Curation Rationale
All the information fields have been cast to string, as their format change through time from one dump to the following. A reduced number of keys have been kept:     "allow_live_comments", "archived", "author", "author_fullname", "banned_by", "category", "content_categories", "contest_mode", "created_utc", "discussion_type", "distinguished", "domain", "edited", "gilded", "hidden", "hide_score", "id", "is_created_from_ads_ui", "is_crosspostable", "is_meta", "is_original_content", "is_reddit_media_domain", "is_robot_indexable", "is_self", "is_video", "locked", "media", "media_embed", "media_only", "name", "no_follow", "num_comments", "num_crossposts", "over_18", "parent_whitelist_status", "permalink", "pinned", "post_hint", "pwls", "quarantine", "removed_by", "removed_by_category", "retrieved_on", "score", "secure_media", "secure_media_embed", "selftext", "send_replies", "spoiler", "stickied", "subreddit", "subreddit_id", "subreddit_name_prefixed", "subreddit_subscribers", "subreddit_type", "suggested_sort", "title", "top_awarded_type", "total_awards_received", "treatment_tags", "upvote_ratio", "url", "url_overridden_by_dest", "view_count", "whitelist_status", "wls".

### Source Data
The [Reddit PushShift data dumps](https://files.pushshift.io/reddit/) are part of a data collection effort which crawls Reddit at regular intervals, to extract and keep all its data.

#### Initial Data Collection and Normalization
See the paper.

#### Who are the source language producers?
Redditors are mostly young (65% below 30), male (70%), and American (50% of the site).

### Personal and Sensitive Information
The data contains Redditor's usernames associated to their content.

## Considerations for Using the Data

This dataset should be anonymized before any processing.
Though the subreddits selected are considered as being of higher quality, they can still reflect what you can find on the internet in terms of expressions of biases and toxicity.

### Contributions

Thanks to [@clefourrier](https://github.com/clefourrier) for adding this dataset.