victormiller commited on
Commit
0dbc84f
1 Parent(s): 775d041

Update curated.py

Browse files
Files changed (1) hide show
  1. curated.py +2 -1
curated.py CHANGED
@@ -445,8 +445,9 @@ filtering_process = Div(
445
  Section(
446
  Div(
447
  H3("Wikipedia"),
 
 
448
  H4("Download and Extraction"),
449
-
450
  P("The Wikimedia dataset was downloaded from the official snapshot on Huggingface: ", A("https://huggingface.co/datasets/wikimedia/wikipedia/tree/main", href="https://huggingface.co/datasets/wikimedia/wikipedia/tree/main"), ". The", D_code("huggingface dataset.to_json", language="python"), " function was used to convert the original parqet format to the jsonl format."),
451
  H4("Filtering"),
452
  P("Manual inspection of the dataset demostrated high quality content. Only one filter was used to remove articles with few words. Based normal sentence constructs, the article was kept if it contained 10 or more words. Any article with fewer than 10 words was removed."),
 
445
  Section(
446
  Div(
447
  H3("Wikipedia"),
448
+ H4("What is Wikipedia:")
449
+ P("Wikipedia is an encyclopedia form of high-quality text data used for language modeling. We have included filtered and deduplicated versions of complete Wikipedia data directly provided by the Wikipedia Foundation for more than 350 languages."),
450
  H4("Download and Extraction"),
 
451
  P("The Wikimedia dataset was downloaded from the official snapshot on Huggingface: ", A("https://huggingface.co/datasets/wikimedia/wikipedia/tree/main", href="https://huggingface.co/datasets/wikimedia/wikipedia/tree/main"), ". The", D_code("huggingface dataset.to_json", language="python"), " function was used to convert the original parqet format to the jsonl format."),
452
  H4("Filtering"),
453
  P("Manual inspection of the dataset demostrated high quality content. Only one filter was used to remove articles with few words. Based normal sentence constructs, the article was kept if it contained 10 or more words. Any article with fewer than 10 words was removed."),