Update common.py

common.py

@@ -29,7 +29,109 @@ fig = px.bar(
 
 dup_cluster_graph = fig.update_layout(showlegend=False)
 
+dup_docs_count = {
+    "80": 382164413,
+    "90": 660766607,
+    "94": 2004544307,
+    "49": 1249363963,
+    "96": 6378899946,
+    "91": 796400555,
+    "13": 170737436,
+    "34": 390565222,
+    "37": 243097281,
+    "78": 333786871,
+    "40": 331019592,
+    "47": 633983288,
+    "74": 443143441,
+    "12": 115630971,
+    "82": 491144800,
+    "63": 335567006,
+    "60": 361001039,
+    "42": 369986102,
+    "43": 344094214,
+    "95": 3297371929,
+    "56": 450449769,
+    "58": 394889638,
+    "48": 821491815,
+    "18": 192658724,
+    "86": 621122463,
+    "50": 917219351,
+    "83": 468165632,
+    "38": 281883697,
+    "51": 244891366,
+    "93": 1236979939,
+    "65": 396080116,
+    "71": 403250107,
+    "11": 101639319,
+    "81": 367154215,
+    "72": 458795954,
+    "32": 218765954,
+    "92": 943046601,
+    "85": 507967375,
+    "66": 279985567,
+    "54": 291611429,
+    "87": 657754973,
+    "39": 296672084,
+    "89": 747973994,
+    "26": 179628225,
+    "45": 441047510,
+    "64": 319547728,
+    "76": 337730046,
+    "57": 415519600,
+    "53": 346555733,
+    "75": 319730996,
+    "21": 239475626,
+    "67": 277544884,
+    "10": 102493868,
+    "68": 348155455,
+    "59": 344897755,
+    "62": 326551051,
+    "22": 223000489,
+    "88": 722070344,
+    "52": 295881819,
+    "84": 613535675,
+    "55": 487356947,
+    "17": 226423150,
+    "69": 349626770,
+    "20": 163869592,
+    "16": 452282480,
+    "70": 390584359,
+    "73": 394778904,
+    "28": 197047765,
+    "36": 230817595,
+    "44": 618669127,
+    "29": 180518021,
+    "77": 429496570,
+    "25": 140344588,
+    "14": 212064682,
+    "41": 428759750,
+    "15": 147268059,
+    "00": 136048949,
+    "31": 325178167,
+    "35": 213448884,
+    "79": 394056890,
+    "24": 359444850,
+    "30": 178934263,
+    "61": 336060420,
+    "23": 378045294,
+    "46": 417319179,
+    "33": 239167872,
+    "27": 111503187,
+    "19": 125085842,
+}
 
+dup_docs_count_for_graph = pd.DataFrame(
+    sorted(dup_docs_count.items()), columns=["CC_dump", "num_duplicates"]
+)
+
+fig = px.bar(
+    dup_docs_count_for_graph,
+    x="CC_dump",
+    y="num_duplicates",
+    labels={"CC_dump": "CommonCrawl Dump", "num_duplicates": "Number of duplicates"},
+)
+
+dup_docs_count_graph = fig
 
 nfc_examples = pd.DataFrame(
         {
@@ -131,7 +233,7 @@ global_div = Div(
             P("The purpose of this step is to create a set of clusters of matching pairs. For example, a list of pairs (A, B), (B, C), (D, E) is merged into a list of components (A, B, C) and (D, E). Using a third-party library like NetworkX to find connected components would require all pairs to fit into the memory of a single machine, which is not feasible. Instead, we implemented a distributed connected component finder [4] using the Dask framework, which can scale across multiple machines. The algorithm works by mapping edges by both the source and destination of pairs and reducing only edges where the source is greater than the destination. It performs successive iterations of this MapReduce computation until convergence, meaning the number of new edges produced becomes zero. In the end, every document in a cluster points to the smallest document within the cluster. Later, we compile a list of duplicate documents that need deletion and gather statistics about each component."),
             P("We needed to partition the duplicate pairs generated in the third stage into three groups to reduce memory pressure on the final stage. We observed that the second stage itself generates partial components which have some overlap. These overlapping clusters cause some documents to appear in the delete set multiple times. However, our deletion code handled this overlap."),
             P("Below is the distribution of duplicate documents found across different dumps of CommonCrawl. The distribution is skewed to the right because the documents are bucketed by the dump ID of the document we retain, and we prefer documents from higher dump IDs."),
-            P("THIS IS A PLACEHOLDER FOR A GRAPH"),
+            plotly2fasthtml(dup_docs_count_graph),
         ),
     Section(
             H3("Analysis of Near-Duplicate Clusters"),