victormiller commited on
Commit
32c6d51
1 Parent(s): 8a3e0e7

Update web.py

Browse files
Files changed (1) hide show
  1. web.py +6 -6
web.py CHANGED
@@ -431,11 +431,11 @@ def web_data():
431
  """),
432
  P("We directly read WARC files instead of WET files and extracted text using Trafilatura. Similar to RefinedWeb, we avoid using Machine Learning (ML)-based metrics for filtering documents to prevent bias introduced by ML models. Importantly, we apply global deduplication across the entire dataset, whereas previous works only use local deduplication. Note that although The Pile also employed global deduplication on its web data (Pile-CC), this accounted for just 0.6\% of 74 snapshots."),
433
 
434
- #Details(
435
- # Summary("Text Extraction Examples"),
436
- # DV2("data/sample_wet.json", "data/sample_warc.json", 3),
437
- # ),
438
- DV2("data/sample_wet.json", "data/sample_warc.json", 3),
439
 
440
  H4("1.2 Language Identification"),
441
  P("""
@@ -443,7 +443,7 @@ def web_data():
443
  This step removes over 60% of the whole data.
444
  """),
445
 
446
- Details(
447
  Summary("Sample documents that are classified as non-English"),
448
  DV("data/sample_non_en.json", 3),
449
  ),
 
431
  """),
432
  P("We directly read WARC files instead of WET files and extracted text using Trafilatura. Similar to RefinedWeb, we avoid using Machine Learning (ML)-based metrics for filtering documents to prevent bias introduced by ML models. Importantly, we apply global deduplication across the entire dataset, whereas previous works only use local deduplication. Note that although The Pile also employed global deduplication on its web data (Pile-CC), this accounted for just 0.6\% of 74 snapshots."),
433
 
434
+ # Details(
435
+ # Summary("Text Extraction Examples"),
436
+ # DV2("data/sample_wet.json", "data/sample_warc.json", 3),
437
+ # ),
438
+ #DV2("data/sample_wet.json", "data/sample_warc.json", 3),
439
 
440
  H4("1.2 Language Identification"),
441
  P("""
 
443
  This step removes over 60% of the whole data.
444
  """),
445
 
446
+ Details(
447
  Summary("Sample documents that are classified as non-English"),
448
  DV("data/sample_non_en.json", 3),
449
  ),