Update curated.py

curated.py

@@ -450,9 +450,6 @@ filtering_process = Div(
         P("The Wikimedia dataset was downloaded from the official snapshot on Huggingface: ", A("https://huggingface.co/datasets/wikimedia/wikipedia/tree/main", href="https://huggingface.co/datasets/wikimedia/wikipedia/tree/main"), ". The", D_code("huggingface dataset.to_json", language="python"), " function was used to convert the original parqet format to the jsonl format."),
         H4("Filtering"),
         P("Manual inspection of the dataset demostrated high quality content. Only one filter was used to remove articles with few words. Based normal sentence constructs, the article was kept if it contained 10 or more words. Any article with fewer than 10 words was removed."),
-        H4("Local Deduplication Process"),
-        Ol(
-            Li("Whole wikipedia was deduped using minhash generation following Slim pajama code"),
         ),
         table_div_wikipedia,
         ), 
@@ -470,10 +467,6 @@ filtering_process = Div(
             Li("Unigram Log Probablity Filter: Documents were kept if they their average unigram log probability was higher than -20. To calculate the average log word probability, we use word frequencies extracted from the", A("1T Web-gram corpus", href= "https://catalog.ldc.upenn.edu/LDC2006T13"),". Specifically, we use the list available created by", A("Rachel Tatman", href="https://www.kaggle.com/datasets/rtatman/english-word-frequency"),"."),
             Li("Note: The Frequency Filter was calculated but not applied. The most frequent word in the paper consists of alpha characters only, and it appears in less than 7.5% of the document. Words are obtained by splitting the text on whitespace."),
         ),
-        H4("Local Deduplication Process"),
-        Ol(
-            Li("Local dedup was done with all papers combined."),
-        ),
         table_div_arx,
         ),
     ),
@@ -554,10 +547,6 @@ filtering_process = Div(
         Ol(
             Li("Many filters were used to clean the phil papers like double whitespaces, new lines etc. All filter details are here: https://github.com/thoppe/The-Pile-PhilPapers/blob/master/pdf_filter.py"),
         ),
-        H4("Local Deduplication Process"),
-        Ol(
-            Li("Local dedup was done with all papers combined."),
-        ),
         table_div_phil,
         ),
     ),
@@ -568,10 +557,6 @@ filtering_process = Div(
         P("Original dataset was downloaded from", A("http://www.statmt.org/europarl/v7/europarl.tgz", href="http://www.statmt.org/europarl/v7/europarl.tgz"),". The files were converted to jsonl lines for filtering."),
         H4("Filtering"),
         P("EuroParl was initially filtered during the download process. Documents with fewer than 200 characters were removed. The documents also contained 'TAGS' which were removed."),
-        H4("Local Deduplication Process"),
-        Ol(
-            Li("Local dedup was done within europarl itself"),
-        ),
         table_div_up,
     ),
     ),
@@ -587,10 +572,6 @@ filtering_process = Div(
             Li("Minimum Word Count Filter: 10"),
             Li("Unigram Log Probability"),
         ),
-        H4("Local Deduplication Process"),
-        Ol(
-            Li("Local dedup was done within hackernews itself"),
-        ),
         table_div_hn,
         ),
     ),
@@ -605,10 +586,6 @@ filtering_process = Div(
             Li("Minimum Word Count Filter: 50"),
             Li("Unigram Log Probability"),
         ),
-        H4("Local Deduplication Process"),
-        Ol(
-            Li("Local dedup was done within USPTO itself"),
-        ),
         table_div_uspto,
         ),
     ),
@@ -660,10 +637,6 @@ filtering_process = Div(
         Ol(
             Li("Minimum Word Count Filter: 10"),
         ),
-        H4("Local Deduplication Process"),
-        Ol(
-            Li("Local dedup was done within stackexchange itself"),
-        ),
         table_div_se,
         ),
     ),
@@ -679,7 +652,8 @@ filtering_process = Div(
 
         def exclude_select_system(x):
             return '\n'.join(line for line in x.split('\n') if not (line.startswith('===') 
-                    and any(term in line for term in ['has joined #', 'has left #', 'Topic for #', "Topic (#", "is now known as"]) ))
+                and any(term in line for term in 
+                ['has joined #', 'has left #', 'Topic for #', "Topic (#", "is now known as"]) ))
         
         def clean(x):
             return '\n'.join('* ' + line[4:] if line.startswith('===') else line[8:] for line in x.split('\n'))
@@ -690,10 +664,6 @@ filtering_process = Div(
             Li("Minimum Word Count Filter: 10"),
             Li("Unigram Log Probability"),
         ),
-        H4("Local Deduplication Process"),
-        Ol(
-            Li("Local dedup was done within Ubuntu IRC itself"),
-        ),
         table_div_uirc,
         ),
     ),
@@ -729,10 +699,6 @@ filtering_process = Div(
             Li("Minimum Word Count Filter: 20"),
             Li("Unigram Log Probability"),
         ),
-        H4("Local Deduplication Process"),
-        Ol(
-            Li("Local dedup was done within PG19 itself"),
-        ),
         table_div_pg19,
         ),
     ),