region:us
Filtered with the following code:
```python
from datasets import load_dataset, load_from_disk
import time

# Categories from https://dsi.ut-capitole.fr/blacklists/index_en.php
blocked_categories = set([
    "adult",                    # Some adult site from erotic to hard pornography
    "aggressif",                # Sites that are aggressive or violent
    "malware",                  # Any website which delivers malware
    "phishing",                 # Same as above
    "cryptojacking",            # Mining site by hijacking
    "dangerous_material",       # Sites which describe how to make bomb and some dangerous material
])

# Blocked quality filters
blocked_quality_warnings = set([
    "tiny",                     # The document has a low (≤ 5) number of lines
    "short sentences",          # The document has a high number (≥ 50%) of short lines
    "header",                   # Indicates that low-quality content could be present at the start of the document
    "footer",                   # Indicates that low-quality content could be present at the tail of the document
    "noisy",                    # Indicates that the document is noisy
])

harmful_ppl_threshold = 500        # Determines the threshold for harmful ppl (lower is more harmful) TODO
language_prob_threshold = 0.9    # Determines the threshold for language identification (higher is more likely) TODO


blocked_urls = set([
    "de.wikipedia.org",         # Wikipedia (because we already have it)
    "tagesschau.de",            # Tagesschau (because we already have it)
])

def filter_content(example):
    has_blocked_category = False
    if "categories" in example["meta"] and example["meta"]["categories"] is not None:
        has_blocked_category = len(set(example["meta"]["categories"]).intersection(blocked_categories)) > 0
    has_blocked_quality_warnings = False
    if "quality_warnings" in example["meta"] and example["meta"]["quality_warnings"] is not None:
        has_blocked_quality_warnings = len(set(example["meta"]["quality_warnings"]).intersection(blocked_quality_warnings)) > 0
    has_blocked_url = False
    if "warc_headers" in example["meta"] and "warc-target-uri" in example["meta"]["warc_headers"] and example["meta"]["warc_headers"]["warc-target-uri"] is not None:
        has_blocked_url = any([url in example["meta"]["warc_headers"]["warc-target-uri"] for url in blocked_urls])
    has_harmful_ppl = example["meta"]["harmful_pp"] < harmful_ppl_threshold if "harmful_pp" in example["meta"] else False
    has_bad_german_identification = example["meta"]["identification"]["prob"] < language_prob_threshold if "identification" in example["meta"] else True
    return not (has_blocked_category or has_blocked_quality_warnings or has_blocked_url or has_harmful_ppl or has_bad_german_identification)
        

t_start = time.time()
ds = load_dataset("bjoernp/oscar2023_de_deduped", split="train", num_proc=128)
print(f"Loading took {time.time() - t_start}s")
print(f"Dataset size before filtering: {len(ds)}")
t_start = time.time()
ds = ds.filter(filter_content, num_proc=128)
print(f"Filtering took {time.time() - t_start}s")
print(f"Dataset size after filtering: {len(ds)}")
```