<commit_before>sizes = [1024] #[10240]

DACIT_URL = "dacit.cs.uni-dortmund.de/datasets"

download_and_extract("large", sizes, [
    DACIT_URL + "/commoncrawl_10240.ascii",

    # http://acube.di.unipi.it/rlzap-dataset/
    DACIT_URL + "/rlzap_dlcp_female",

    # http://www.internationalgenome.org/data-portal/sample/NA19000
    DACIT_URL + "/1000Genome_JPT_NA19000_SRR099528_1.fastq.filtered4_no_N.dna",

    # https://panthema.net/2012/1119-eSAIS-Inducing-Suffix-and-LCP-Arrays-in-External-Memory/
    # Gutenberg Concatenation from September 2012
    DACIT_URL + "/gutenberg-201209.24090588160.xz",
])
<commit_msg>actually generate 10Gb texts<commit_after>sizes = [1, 10240]

DACIT_URL = "dacit.cs.uni-dortmund.de/datasets"

download_and_extract("large", sizes, [
    DACIT_URL + "/commoncrawl_10240.ascii",

    # http://acube.di.unipi.it/rlzap-dataset/
    # (array of 32bit values)
    DACIT_URL + "/rlzap_dlcp_female",

    # http://www.internationalgenome.org/data-portal/sample/NA19000
    DACIT_URL + "/1000Genome_JPT_NA19000_SRR099528_1.fastq.filtered4_no_N.dna",

    # https://panthema.net/2012/1119-eSAIS-Inducing-Suffix-and-LCP-Arrays-in-External-Memory/
    # Gutenberg Concatenation from September 2012
    DACIT_URL + "/gutenberg-201209.24090588160.xz",
])

{'fraction_non_alphanumeric': 0.14184952978056425, 'fraction_numerical': 0.10266457680250783, 'mean_word_length': 4.087649402390438, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6246778', 'n_tokens_mistral': 634, 'n_tokens_neox': 525, 'n_words': 53}