Update results.py

results.py

@@ -16,62 +16,68 @@ perp1_div = Div(
             P("For each bucket, we aggregated all the chunks that belong to a single year and calculated the average perplexity for each (bucket, year) data point."),
             Img(src="images/prep-diff-buckets-global.png", height = "300", width = "600" ),
         ),
+        Section(
+            H3("Perplexity vs Years"),
+            P("Taking the same data, we can convert it into a graph indicating the yearly trend. For most buckets, the average perplexity of dumps from more recent years seem to be lower than that of former years."),
+            Img(src="images/prep-across-diff-year-global-dup-buckets.png", height = "300", width = "600" ),
+        ),
+    Section(
+            H3("Perplexity vs Document Duplication"),
+            P("We can also break each bucket into distinct document counts. The graph becomes a bit noisy at the end because of insufficient samples with larger duplication counts."),
+            Img(src="images/prep-across-diff-docs-dup-count-global.png", height = "300", width = "600" ),
+        ),
+    Section(
+            H3("Perplexity vs Dump Duplication"),
+            P("We are also interested in how the number of dumps a document is in affect data quality. From the graph below we can see that documents that are duplicated across around 40 - 60 dumps usually have lower perplexity."),
+            Img(src="images/prep-across-diff-dump-dup-counts-global.png", height = "300", width = "600" ),
+        ),
+    Section(
+            H3("Perplexity vs Local Buckets"),
+            P("Previously we have seen that documents in recent dumps tend to have lower perplexity. This might be related to the way how global deduplication was implemented. During global deduplication, we only keep copy in the latest dump. Hence documents that are duplicated across multiple dumps only appear in the latest one. To avoid bias brought by this strategy, we tried to recover the states before the global deduplication by reading the metadata attached with each document."),
+            Img(src="images/prep-across-diff-buckets-local.png", height = "300", width = "600" ),
+        ),
+    Section(
+            H3("Perplexity vs Local Dump Duplication"),
+            P("Following the same practice, we can plot the local version of the graph of average perplexity with respect to dump duplication."),
+            Img(src="images/prep-diff-dump-dump-counts-local.png", height = "300", width = "600" ),
+        ),
 )
 
 
+llama_div = Div(
+        Section(
+            H2("Llama 3.1 8B"),
+            P("For comparison purpose, we run the same perplexity evaluation with llama 3.1 8B model.")
+        )
+        Section(
+            H3("Perplexity vs Buckets"),
+            Img(src="images/perp-across-diff-buckets-global.png", height = "300", width = "600" ),
+        ),
+        Section(
+            H3("Perplexity vs Years"),
+            Img(src="images/prep-across-diff-years-global.png", height = "300", width = "600" ),
+        ),
+    Section(
+            H3("Perplexity vs Dump Duplication"),
+            Img(src="images/prep-vs-dump-dup-global.png", height = "300", width = "600" ),
+        ),
+    Section(
+            H3("Perplexity vs Local Buckets"),
+            Img(src="images/prep-diff-buckets-local.png", height = "300", width = "600" ),
+        ),
+    Section(
+            H3("Perplexity vs Local Dump Duplication"),
+            Img(src="images/prep-vs-dump-dup-global.png", height = "300", width = "600" ),
+        ),
+)
 
 
-perp_v_years = P("Taking the same data, we can convert it into a graph indicating the yearly trend. For most buckets, the average perplexity of dumps from more recent years seem to be lower than that of former years.")
-perp_v_years_img = Img(src="images/prep-across-diff-year-global-dup-buckets.png")
-perp_v_docdup = P("We can also break each bucket into distinct document counts. The graph becomes a bit noisy at the end because of insufficient samples with larger duplication counts.")
-perp_v_docdup_img = Img(src="images/prep-across-diff-docs-dup-count-global.png")
-perp_v_dumpdup = P("We are also interested in how the number of dumps a document is in affect data quality. From the graph below we can see that documents that are duplicated across around 40 - 60 dumps usually have lower perplexity.")
-perp_v_dumpdup_img = Img(src="images/prep-across-diff-dump-dup-counts-global.png")
-perp_v_localbuck = P("Previously we have seen that documents in recent dumps tend to have lower perplexity. This might be related to the way how global deduplication was implemented. During global deduplication, we only keep copy in the latest dump. Hence documents that are duplicated across multiple dumps only appear in the latest one. To avoid bias brought by this strategy, we tried to recover the states before the global deduplication by reading the metadata attached with each document.")
-perp_v_localbuck_img = Img(src="images/prep-across-diff-buckets-local.png")
-perp_v_localdump = P("Following the same practice, we can plot the local version of the graph of average perplexity with respect to dump duplication.")
-perp_v_localdump_img = Img(src="images/prep-diff-dump-dump-counts-local.png")
-
-llama_explan = P("For comparison purpose, we run the same perplexity evaluation with llama 3.1 8B model.")
-llama_perp_v_buckets_img = Img(src="images/perp-across-diff-buckets-global.png")
-llama_perp_v_years_img = Img(src="images/prep-across-diff-years-global.png")
-llama_perp_v_dumpdup_img = Img(src="images/prep-vs-dump-dup-global.png")
-llama_perp_v_localbuck_img = Img(src="images/prep-diff-buckets-local.png")
-llama_perp_v_localdump_img = Img(src="images/prep-vs-dump-dup-global.png")
-
 def results():
     return Div(
                 Section(
                     intro_div,
                     perp1_div,
-                    # redo
-                    #H3("Perplexity vs Years"),
-                 #   perp_v_years,
-                 #   perp_v_years_img,
-                 #   H3("Perplexity vs Document Duplication"),
-                 #   perp_v_docdup,
-                 #   perp_v_docdup_img,
-                 #   H3("Perplexity vs Dump Duplication"),
-                  #  perp_v_dumpdup,
-                  #  perp_v_dumpdup_img,
-                  #  H3("Perplexity vs Local Buckets"),
-                  #  perp_v_localbuck,
-                  #  perp_v_localbuck_img,
-                  #  H3("Perplexity vs Local Dump Duplication"),
-                  #  perp_v_localdump,
-                  #  perp_v_localdump_img,
-                  #  H2("Llama 3.1 8B"),
-                  #  llama_explan,
-                  #  H3("Perplexity vs Buckets"),
-                  #  llama_perp_v_buckets_img,
-                  #  H3("Perplexity vs Years"),
-                  #  llama_perp_v_years_img,
-                  #  H3("Perplexity vs Dump Duplication"),
-                  #  llama_perp_v_dumpdup_img,
-                  #  H3("Perplexity vs Local Buckets"),
-                  #  llama_perp_v_localbuck_img,
-                  #  H3("Perplexity vs Local Dump Duplication"),
-                   # llama_perp_v_localdump_img,
+                    llama_div,
                     id="inner-text"
                 )
     )