text fix

common.py

@@ -288,7 +288,7 @@ global_div = Div(
             "This section discusses all details related to deduplication and filterings steps that were uniformly applied to all data. The section is split into the following topic areas: "
         ),
         Ul(
-            Li("Motivation Behind Global Deduplication", style="margin-bottom: 5px"),
+            Li("Why Global Deduplication", style="margin-bottom: 5px"),
             Li(
                 "TxT360 Deduplication Process and Implementation",
                 style="margin-bottom: 5px",
@@ -302,7 +302,7 @@ global_div = Div(
         id="section41",
     ),
     Section(
-        H2("Motivation Behind Global Deduplication"),
+        H2("Why Global Deduplication"),
         P(
             "Deduplication is beneficial for LM pretraining in several ways, with the most important being controllable upsampling. With unique data, teams gain fine-grained control over the training data. Other benefits of deduplication include avoiding train-test overlap which prevents evaluation contamination."
         ),
@@ -408,7 +408,7 @@ global_div = Div(
     ),
     Section(
         H2("Personally Identifiable Information Removal"),
-        H3("Motivation Behind Personally Identifiable Information Removal"),
+        H3("Why Personally Identifiable Information Removal"),
         P(
             "Personally Identifiable Information (PII) refers to any information that can be used to identify an individual, such as names, addresses, phone numbers, email addresses, and social security numbers. PII removal is essential for data privacy and security, as well as for compliance with global regulations. By removing PII from the training data, we can reduce the risk of data breaches and unauthorized access to sensitive information. Additionally, removing PII from training data prevents the models generating that specific PII during inference time."
         ),

main.py

@@ -834,7 +834,7 @@ def intro():
         Section(
             H2("About TxT360"),
             P(  "TL;DR ", 
-                B("We introduce TxT360 (Trillion eXtracted Text), the first dataset to globally deduplicate 99 CommonCrawl snapshots and 14 high-quality data sources from diverse domains (e.g., FreeLaw, PG-19, etc.). Our large-scale deduplication process enables precise control over data weighting. In addition to document selection, TxT360, along with its rich metadata, allows for the assignment of optimal data weights. We demonstrate a simple but effective upsampling recipe that creates a 15+ trillion-token corpus, outperforming FineWeb 15T. Furthermore, TxT360 empowers pre-trainers to explore more advanced weighting techniques, a capability not commonly available in previous pre-training datasets."
+                B("We introduce TxT360 (Trillion eXtracted Text), the first dataset to globally deduplicate 99 CommonCrawl snapshots and 14 high-quality data sources from diverse domains (e.g., FreeLaw, PG-19, etc.). The large-scale deduplication process and rich metadata stored enables precise control over data distribution. In addition to document selection, TxT360, along with its rich metadata, allows for the assignment of optimal data weights. We demonstrate a simple but effective upsampling recipe that creates a 15+ trillion-token corpus, outperforming FineWeb 15T. Furthermore, TxT360 empowers pre-trainers to explore more advanced weighting techniques, a capability not commonly available in previous pre-training datasets."
                 )
             ),
             P(
@@ -843,22 +843,23 @@ def intro():
                 D_cite(bibtex_key="fineweb"),
                 D_cite(bibtex_key="c4"),
                 D_cite(bibtex_key="muennighoff2023scaling"),
+                D_cite(bibtex_key="dolma"),
                 ", TxT360 carefully implements data processing steps including extraction, filtering, deduplication, personally identifiable information removal, and other steps.",
             ),
             P(
-                "Metadata is stored to recover the raw distribution for each dataset, enabling fine-grained control to create data distributions and corpus of desired size. As an example, we present one simple upsampling scheme that takes into account the duplication counts, resulting in a 15~16 trillion token corpus, outperforming FineWeb and our non-upsampling baselines, on diverse evaluations. Unlike DCLM",
+                "Metadata is stored along the processing stpes, enabling fine-grained control to create data distributions and corpus of desired size. As an example, we present one simple upsampling scheme that takes into account the duplication counts, resulting in a 15~16 trillion token corpus, outperforming FineWeb and our non-upsampling baselines, on diverse evaluations. Unlike DCLM",
                 D_cite(bibtex_key="dclm"),
                 "and RedPajama V2,",
                 D_cite(bibtex_key="redpajama-v2"),
                 "we present the final deduplicated dataset that is ready to go.",
             ),
             P(
-                "We documented all implementation details in this blog post and are open sourcing the code. Examples of each filter and rationale supporting each decision are included."
+                "In line with our 360° open-source initiative, we’ve documented all implementation details in this blog post and will be open-sourcing the code soon (stay tuned!). We also provide examples of each filter along with the rationale behind every decision, with the goal of informing and inspiring future work."
             ),
             id="section11",
         ),
         Section(
-            H2("Motivation Behind TxT360"),
+            H2("Why TxT360"),
             H3(
                 "TxT360 is the first dataset to combine both web and curated data sources commonly used in pretraining."
             ),

overview.py

@@ -272,12 +272,12 @@ overview_div = Div(
             H2("Overview"),
             H3("What This Section Contains"),
             Ul(
-                Li("Motivation Behind TxT360", style = "margin-bottom: 5px"),
+                Li("Why TxT360", style = "margin-bottom: 5px"),
                 Li("The Highlevel Data Process Approach", style = "margin-bottom: 5px"),
                 Li("Introduction to Global Deduplication", style = "margin-bottom: 5px"),
             ),
-            H2("Motivation Behind TxT360"),
-            H3("TxT360 is the first dataset to combine both web and curated data sources commonly used in pretraining."),
+            H2("Why TxT360"),
+            H3("TxT360 is the first dataset to combine both crawled web pages and high quality curated data sources commonly used in pretraining."),
             P("The quality and size of a pre-training dataset play a crucial role in the performance of large language models (LLMs). Data is often referred as low quality if it has not been filtered to review unwanted text. The community has introduced a variety of filtered datasets including purely web-based datasets. Commonly used pretraining datasets are:"),
             Ul(
                 Li("RefinedWeb - cite", style = "margin-bottom: 5px"),