Update results.py

results.py

@@ -614,6 +614,62 @@ fig.update_layout(
 # Show the figure
 llama_graph6 = fig
 
+##txt360 vs fineweb comparison table
+dataset_comparison = pd.DataFrame(
+        {
+            "Metric": [
+                "BoolQ",
+                "PIQA",
+                "HellaSwag",
+                "Winogrande",
+                "MMLU",
+                "Natural Questions",
+                "TriviaQA",
+                "GSM8K",
+                "MATH",
+                "MedQA",
+            ],
+            "TxT360 - Upsampling": [
+                "70.31",
+                "80.36",
+                "73.54",
+                "68.43",
+                "30.26",
+                "22.22",
+                "58.52",
+                "3.41",
+                "28.04",
+                "25.61",
+            ],
+            "FineWeb-1.5T": [
+                "71.5",
+                "82.1",
+                "79.46",
+                "70.96",
+                "28.5",
+                "19.03",
+                "36.65",
+                "5.31",
+                "29.65",
+                "27.26",
+            ],
+            "TxT360 Difference": [
+                "(1.19)",
+                "(1.74)",
+                "(5.92)",
+                "(2.53)",
+                "1.76",
+                "3.19",
+                "21.87",
+                "(1.9)",
+                "(1.61)",
+                "(1.65)",
+            ],
+        }
+    )
+
+table_html = dataset_comparison.to_html(index=False, border=0)
+table_div_1 = Div(NotStr(table_html), style="margin: 40px;")
 
 intro_div = Div(
         H2("Perplexity Evaluation on Duplicate Data"),
@@ -621,13 +677,16 @@ intro_div = Div(
         P("We took one of the model-based data quality evaluation strategies adopted by [DataComp-LM](https://arxiv.org/abs/2406.11794), which used perplexity filtering as a candidate for quality filtering. DataComp-LM followed [CCNet’s](https://arxiv.org/abs/1911.00359) practice to use a 5-gram Kneser-Ney model as implemented in the [KenLM](https://github.com/kpu/kenlm) library for efficient perplexity calculation. Following this practice, we estimated data quality by taking a KenLM model (from [edugp/kenlm](https://huggingface.co/edugp/kenlm)) trained on English Wikipedia data to compute perplexity on data with different duplication patterns. Lower perplexity is regarded as a signal of higher quality."),
         H3("Sampling Strategy"),
         P("We started from a processed Common Crawl (CC) ablation dataset divided by the number of duplicates of each document. For each CC dump, we have different buckets each holding chunks of document with different duplicate count ranges (1-1, 2-5, 6-10, 11-100, 101-1000, 1001-30000000). We sampled the first 10k documents from each chunk with their meta data."),
-
 )
 
 upsampling_exp = Div(
         H2("Upsampling Experiment: TxT360 vs FineWeb"),
         H3("Experiment Setup"),
         P("We performed a comparison of 1.5T tokens from FineWeb and 1.5T tokens of TxT360 across 10 diverse evaluations. Our FineWeb evaluation is based on a random sample 1.5T tokens from FineWeb (base). For TxT360, we also random sample 1.5T tokens by upsampling data instances with more duplicates. Concretely, the upsampling weight is set to 3 for data points with duplicates in the range from 2 to 5,  5 for the range from 5 to 100, 8 for that from 101 to 1000, and 10 for more than 1000 duplicates."),
+        table_div_1,
+        P("To account for differing dataset sizes, the evaluation scores represent the final evaluation score after the entire dataset has been processed."),
+        H3("Training Evaluations"),
+        P("We also conducted full scale training using TxT360 and FineWeb-1.5T. Below are plots of the training and validation loss curves for each dataset. We can see that TxT360 achieves a lower training and validation loss compared to FineWeb-1.5T. "),
 )
 
 perp1_div = Div(
@@ -707,6 +766,7 @@ def results():
     return Div(
                 Section(
                     intro_div,
+                    upsampling_exp,
                     perp1_div,
                     llama_div,
                     P("test plotly"),