Update results.py

results.py

@@ -50,9 +50,9 @@ fig_val.add_trace(go.Scatter(x=steps, y=txt360, mode='lines', name='TxT360'))
 
 # Update layout
 fig_val.update_layout(
-    title='Perplexity Across Steps',
+    title='Validation Loss Over Steps: TxT360 vs FineWeb',
     xaxis_title='Steps',
-    yaxis_title='Perplexity',
+    yaxis_title='Loss',
     legend_title='Models'
 )
 
@@ -76,11 +76,10 @@ fig_loss.add_trace(go.Scatter(x=data['Step'], y=data['FineWeb'], mode='lines', n
 
 # Update layout
 fig_loss.update_layout(
-    title="Loss over Steps: TxT360 vs FineWeb",
+    title="LM Loss Over Steps: TxT360 vs FineWeb",
     xaxis_title="Steps",
     yaxis_title="Loss",
     legend_title="Models",
-    template="plotly_dark"
 )
 
 # Display the graph
@@ -750,16 +749,20 @@ dataset_comparison = pd.DataFrame(
     )
 
 table_html = dataset_comparison.to_html(index=False, border=0)
-table_div_1 = Div(NotStr(table_html), style="margin: 40px;")
+table_div_1 = Div(Center(NotStr(table_html), style="margin: 40px;"))
 
 intro_div = Div(
-        H2("Perplexity Evaluation on Duplicate Data"),
-        H3("Model based Quality Estimation"),
-        P("We took one of the model-based data quality evaluation strategies adopted by [DataComp-LM](https://arxiv.org/abs/2406.11794), which used perplexity filtering as a candidate for quality filtering. DataComp-LM followed [CCNet’s](https://arxiv.org/abs/1911.00359) practice to use a 5-gram Kneser-Ney model as implemented in the [KenLM](https://github.com/kpu/kenlm) library for efficient perplexity calculation. Following this practice, we estimated data quality by taking a KenLM model (from [edugp/kenlm](https://huggingface.co/edugp/kenlm)) trained on English Wikipedia data to compute perplexity on data with different duplication patterns. Lower perplexity is regarded as a signal of higher quality."),
-        H3("Sampling Strategy"),
-        P("We started from a processed Common Crawl (CC) ablation dataset divided by the number of duplicates of each document. For each CC dump, we have different buckets each holding chunks of document with different duplicate count ranges (1-1, 2-5, 6-10, 11-100, 101-1000, 1001-30000000). We sampled the first 10k documents from each chunk with their meta data."),
+    H2("TxT360 Results")
+    H3("What This Section Contains"),
+            P("This section provides brief upsampling ablation examples and detailed perplexity analysis of CommonCrawl snapshots. The section is split into the following topic areas: "),
+            Ul(
+                Li("Upsampling Experiment: TxT360 vs FineWeb", style = "margin-bottom: 5px"),
+                Li("Perplexity Evaluation on Duplicate Data", style = "margin-bottom: 5px"),
+            ),
+        ),
 )
 
+
 upsampling_exp = Div(
         H2("Upsampling Experiment: TxT360 vs FineWeb"),
         H3("Experiment Setup"),
@@ -772,11 +775,20 @@ upsampling_exp = Div(
         plotly2fasthtml(lm_loss_graph),
 )
 
+preplexity_intro_div = Div(
+        H2("Perplexity Evaluation on Duplicate Data"),
+        H3("Model based Quality Estimation"),
+        P("We took one of the model-based data quality evaluation strategies adopted by [DataComp-LM](https://arxiv.org/abs/2406.11794), which used perplexity filtering as a candidate for quality filtering. DataComp-LM followed [CCNet’s](https://arxiv.org/abs/1911.00359) practice to use a 5-gram Kneser-Ney model as implemented in the [KenLM](https://github.com/kpu/kenlm) library for efficient perplexity calculation. Following this practice, we estimated data quality by taking a KenLM model (from [edugp/kenlm](https://huggingface.co/edugp/kenlm)) trained on English Wikipedia data to compute perplexity on data with different duplication patterns. Lower perplexity is regarded as a signal of higher quality."),
+        H3("Sampling Strategy"),
+        P("We started from a processed Common Crawl (CC) ablation dataset divided by the number of duplicates of each document. For each CC dump, we have different buckets each holding chunks of document with different duplicate count ranges (1-1, 2-5, 6-10, 11-100, 101-1000, 1001-30000000). We sampled the first 10k documents from each chunk with their meta data."),
+)
+
+
 perp1_div = Div(
         Section(
             H3("Perplexity vs Buckets"),
             P("For each bucket, we aggregated all the chunks that belong to a single year and calculated the average perplexity for each (bucket, year) data point."),
-            Img(src="images/prep-diff-buckets-global.png", height = "300", width = "600" ),
+            #Img(src="images/prep-diff-buckets-global.png", height = "300", width = "600" ),
             plotly2fasthtml(Perplexity_Across_Different_Buckets_global_graph),
         ),
         Section(
@@ -794,19 +806,19 @@ perp1_div = Div(
     Section(
             H3("Perplexity vs Dump Duplication"),
             P("We are also interested in how the number of dumps a document is in affect data quality. From the graph below we can see that documents that are duplicated across around 40 - 60 dumps usually have lower perplexity."),
-            Img(src="images/prep-across-diff-dump-dup-counts-global.png", height = "300", width = "600" ),
+            #Img(src="images/prep-across-diff-dump-dup-counts-global.png", height = "300", width = "600" ),
             plotly2fasthtml(graph4),
         ),
     Section(
             H3("Perplexity vs Local Buckets"),
             P("Previously we have seen that documents in recent dumps tend to have lower perplexity. This might be related to the way how global deduplication was implemented. During global deduplication, we only keep copy in the latest dump. Hence documents that are duplicated across multiple dumps only appear in the latest one. To avoid bias brought by this strategy, we tried to recover the states before the global deduplication by reading the metadata attached with each document."),
-            Img(src="images/prep-across-diff-buckets-local.png", height = "300", width = "600" ),
+            #Img(src="images/prep-across-diff-buckets-local.png", height = "300", width = "600" ),
             plotly2fasthtml(graph5),
         ),
     Section(
             H3("Perplexity vs Local Dump Duplication"),
             P("Following the same practice, we can plot the local version of the graph of average perplexity with respect to dump duplication."),
-            Img(src="images/prep-diff-dump-dump-counts-local.png", height = "300", width = "600" ),
+            #Img(src="images/prep-diff-dump-dump-counts-local.png", height = "300", width = "600" ),
             plotly2fasthtml(graph6),
         ),
 )
@@ -819,27 +831,27 @@ llama_div = Div(
         ),
         Section(
             H3("Perplexity vs Buckets"),
-            Img(src="images/perp-across-diff-buckets-global.png", height = "300", width = "600" ),
+            #Img(src="images/perp-across-diff-buckets-global.png", height = "300", width = "600" ),
             plotly2fasthtml(llama_graph1),
         ),
         Section(
             H3("Perplexity vs Years"),
-            Img(src="images/prep-across-diff-years-global.png", height = "300", width = "600" ),
+            #Img(src="images/prep-across-diff-years-global.png", height = "300", width = "600" ),
             plotly2fasthtml(llama_graph2),
         ),
     Section(
             H3("Perplexity vs Dump Duplication"),
-            Img(src="images/prep-vs-dump-dup-global.png", height = "300", width = "600" ),
+            #Img(src="images/prep-vs-dump-dup-global.png", height = "300", width = "600" ),
             plotly2fasthtml(llama_graph4),
         ),
     Section(
             H3("Perplexity vs Local Buckets"),
-            Img(src="images/prep-diff-buckets-local.png", height = "300", width = "600" ),
+            #Img(src="images/prep-diff-buckets-local.png", height = "300", width = "600" ),
             plotly2fasthtml(llama_graph5),
         ),
     Section(
             H3("Perplexity vs Local Dump Duplication"),
-            Img(src="images/prep-vs-dump-dup-global.png", height = "300", width = "600" ),
+            #Img(src="images/prep-vs-dump-dup-global.png", height = "300", width = "600" ),
             plotly2fasthtml(llama_graph6),
         ),
 )
@@ -850,6 +862,7 @@ def results():
                 Section(
                     intro_div,
                     upsampling_exp,
+                    preplexity_intro_div,
                     perp1_div,
                     llama_div,
                     P("test plotly"),