Update web.py

web.py

@@ -9,6 +9,152 @@ from data.url_blocklist import urls_high_matches, urls_false_positives
 from data.non_web_urls import non_web_urls
 from fasthtml.components import D_code
 
+data_filtering_table_data = pd.DataFrame(
+        {
+            "Dataset": [
+                "TxT360",
+                "FineWeb",
+                "RefinedWeb",
+                "RedPajamaV2",
+                "C4",
+                "Dolma",
+                "RedPajamaV1",
+                "The Pile",
+            ],
+            "Data Reading": [
+                "warc",
+                "warc",
+                "warc",
+                "wet",
+                "wet",
+                "warc",
+                "wet",
+                "warc",
+            ],
+            "Text Extraction": [
+                "trafilatura",
+                "trafilatura",
+                "trafilatura",
+                "n/a",
+                "n/a",
+                "?",
+                "n/a",
+                "jusText",
+            ],
+            "URL Filtering": [
+                "Yes",
+                "Yes",
+                "Yes",
+                "Yes",
+                "No",
+                "No",
+                "No",
+                "No",
+            ],
+            "Language Identification": [
+                "fastText",
+                "fastText",
+                "fastText",
+                "fastText",
+                "langdetect",
+                "fastText",
+                "fastText",
+                "pycld2",
+            ],
+            "Line Removal": [
+                "Yes",
+                "Yes",
+                "Yes",
+                "Yes",
+                "Yes",
+                "Yes",
+                "No",
+                "No",
+            ],
+            "QF: ML-based": [
+                "No",
+                "No",
+                "No",
+                "Yes",
+                "No",
+                "No",
+                "Yes",
+                "Yes",
+            ],
+            "QF: Repition-based": [
+                "Yes",
+                "Yes",
+                "Yes",
+                "Yes",
+                "No",
+                "Yes",
+                "No",
+                "No",
+            ],
+            "QF: Correction-based": [
+                "Yes",
+                "Yes",
+                "Yes",
+                "No",
+                "No",
+                "No",
+                "No",
+                "No",
+            ],
+            "QF: Gopher Rules": [
+                "Yes",
+                "Yes",
+                "Yes",
+                "Yes",
+                "No",
+                "Yes",
+                "No",
+                "No",
+            ],
+            "QF: C4 Rules": [
+                "Yes",
+                "Yes",
+                "Yes",
+                "Yes",
+                "Yes",
+                "Yes",
+                "No",
+                "No",
+            ],
+            "PI Filtering": [
+                "Yes",
+                "Yes",
+                "No",
+                "No",
+                "No",
+                "Yes",
+                "No",
+                "No",
+            ],
+            "Exact Deduplication": [
+                "Bloom Filter",
+                "n/a",
+                "ExactSubStr",
+                "Bloom Filter",
+                "n/a",
+                "Bloom Filter",
+                "n/a",
+                "n/a",
+            ],
+            "Fuzzy Deduplication": [
+                "Global",
+                "Local",
+                "Local",
+                "Local",
+                "Local",
+                "Local",
+                "Local",
+                "Global",
+            ],
+        }
+)
+table_html_filter_data = data_filtering_table_data.to_html(index=False, border=0)
+table_div_filter_data = Div(NotStr(table_html_filter_data), style="margin: 40px;")
 
 def DVS(
     left,
@@ -216,6 +362,12 @@ def web_data():
             ),
             style="margin-top: 20px;",
         ),
+        H2("Web Data Processing Overview"),
+        P("The following section provides explicit details covering the reasoning and decisions behind each of the filters we applied. The table below provides a high-level comparison of TxT360's filtering compared to other commonly used pretraining datasets."),
+        table_div_filter_data,
+        P("Our filtering rate is illustrated below. Before deduplication, our filtering rate is comparable to RefinedWeb. During global deduplication, we removed approximately 85.89% of the data, significantly higher than previous works, indicating a large number of duplicates across dumps. "),
+        Img(src="images/filter_rate.jpg", height = "300", width = "600" ),
+        P("Note: All percentages are based on the number of documents. The gray bars represent the relative percentages of removed documents at each step, while the colorful bars represent the percentages of retained documents relative to the total number of documents in the raw Common Crawl."),
         H3("1. Document Preparation"),
         
         H4("1.1 Text Extraction"),
@@ -226,6 +378,9 @@ def web_data():
         we found WET files to include boilerplate content like navigation menus, ads, and other irrelevant texts. 
         Accordingly, our pipeline starts from raw WARC files, reads with the warcio library, and extracts texts using trafilatura.
         """),
+        P("We directly read WARC files instead of WET files and extracted text using Trafilatura. Similar to RefinedWeb, we avoid using Machine Learning (ML)-based metrics for filtering documents to prevent bias introduced by ML models. Importantly, we apply global deduplication across the entire dataset, whereas previous works only use local deduplication. Note that although The Pile also employed global deduplication on its web data (Pile-CC), this accounted for just 0.6\% of 74 snapshots."),
+
+
         DV2("data/sample_wet.json", "data/sample_warc.json", 3),
         
         H4("1.2 Language Identification"),