move to spa, smooth srolling

common.py

@@ -299,7 +299,7 @@ global_div = Div(
             ),
             Li("Normalization Form C Discussion", style="margin-bottom: 5px"),
         ),
-        id="section1",
+        id="section41",
     ),
     Section(
         H2("Motivation Behind Global Deduplication"),
@@ -331,7 +331,7 @@ global_div = Div(
         P(
             "Additionally, we maintained statistics about each matching clusters as they were formed during the final stage of deduplication. Below are the details of all four stages of our deduplication pipeline. We use Dask extensively throughout all stages of the deduplication. We have included the size of results of each stage on disk to give an idea about the scale:"
         ),
-        id="section2",
+        id="section42",
     ),
     Section(
         H3("MinHash Generation"),
@@ -339,7 +339,7 @@ global_div = Div(
             "We use the datasketch library to generate MinHash signatures with the number of permutations to 128. Each signature is signature represented as a MinHash object for each document. Before calculating the signature, the text is cleaned by stripping whitespace, converting to lowercase, and removing punctuation, consecutive spaces, newlines, and tabs. Next, a list of 13-grams is generated to use as features for creating a document signature. The globally-unique document IDs and signatures are then saved to disk. The documented ID is designed by an encoding scheme which converts file names and line numbers (there is one document per line) to unique document IDs. This also helped a lot in saving disk and memory for this stage."
         ),
         P(B("This step produced 20 TB of hashes.")),
-        id="section3",
+        id="section43",
     ),
     Section(
         H3("Matching Pairs Generation"),
@@ -351,7 +351,7 @@ global_div = Div(
         ),
         D_code(dask_algo, block="block", language="python"),
         P(B("This step produced 9.2 TB of matching pairs from all bands.")),
-        id="section4",
+        id="section44",
     ),
     Section(
         H3("Finding Duplicate Pairs"),
@@ -369,7 +369,7 @@ global_div = Div(
             "The resulting unique pairs are then used to identify clusters of near-duplicates by finding connected components in a graph, where the vertices represent documents and the edges represent matches."
         ),
         P(B("This step produced 1.9 TB of unique pairs.")),
-        id="section5",
+        id="section45",
     ),
     Section(
         H3("Finding Connected Components using MapReduce"),
@@ -389,7 +389,7 @@ global_div = Div(
             "Below is the distribution of duplicate documents found across different snapshots of CommonCrawl. The distribution is skewed to the right because the documents are bucketed by the dump ID of the document we retain, and we prefer documents from higher dump IDs."
         ),
         plotly2fasthtml(dup_docs_count_graph()),
-        id="section6",
+        id="section46",
     ),
     Section(
         H3("Analysis of Near-Duplicate Clusters"),
@@ -434,7 +434,7 @@ global_div = Div(
                 style="list-style-type: none",
             ),
         ),
-        id="section7",
+        id="section47",
     ),
     Section(
         H2("Normalization Form C"),
@@ -454,7 +454,7 @@ global_div = Div(
                 style="list-style-type: none",
             )
         ),  # "background-color= gray" "color= blue" maybe add this later
-        id="section8",
+        id="section48",
     ),
     Section(
         H3("NFC Examples"),

curated.py

@@ -1554,27 +1554,7 @@ table_html_data_pipe = data_pipeline_table.to_html(index=False, border=0)
 table_div_data_pipe = Div(NotStr(table_html_data_pipe), style="margin: 40px;")
 
 
-def update(target: str, request):
-    params = request.query_params
-    if data_source := params.get(f"data_source_{target}"):
-        return get_data(data_source, params.get(f"doc_id_{target}", 3), target)
-    if doc_id := params.get(f"doc_id_{target}"):
-        return get_data(params.get(f"data_source_{target}"), doc_id, target)
-
-
-def curated(request):
-    # Partial Updates
-    params = dict(request.query_params)
-    if target := params.get("target"):
-        if data_source := params.get(f"data_source_{target}"):
-            return get_data(
-                data_source, params.get(f"doc_id_{target}", 3), params.get("target")
-            )
-        if doc_id := params.get(f"doc_id_{target}"):
-            return get_data(
-                params.get(f"data_source_{target}"), doc_id, params.get("target")
-            )
-
+def curated():
     data_preparation_steps = pd.DataFrame(
         {
             "Method": [
@@ -1623,15 +1603,15 @@ def curated(request):
         Section(
             curated_sources_intro,
             plotly2fasthtml(treemap_chart),
-            id="section1",
+            id="section31",
         ),
         Section(
             data_preprocessing_div,
-            id="section2",
+            id="section32",
         ),
         Section(
             filtering_process,
-            id="section3",
+            id="section33",
         ),
         id="inner-text",
     )

main.py

@@ -175,9 +175,7 @@ def main():
                     Div(
                         A(
                             "TxT360",
-                            href="/intro#section1",
-                            hx_get="/intro#section1",
-                            hx_target="#inner-text",
+                            href="#section1",
                         )
                     ),
                     Div(
@@ -185,25 +183,19 @@ def main():
                             Li(
                                 A(
                                     "About TxT360",
-                                    href="/intro#section1",
-                                    hx_get="/intro#section1",
-                                    hx_target="#inner-text",
+                                    href="#section11",
                                 )
                             ),
                             Li(
                                 A(
                                     "Motivation Behind TxT360",
-                                    href="/intro#section2",
-                                    hx_get="/intro#section2",
-                                    hx_target="#inner-text",
+                                    href="#section12",
                                 )
                             ),
                             Li(
                                 A(
                                     "Generalizable Approach to Data Processing",
-                                    href="/intro#section3",
-                                    hx_get="/intro#section3",
-                                    hx_target="#inner-text",
+                                    href="#section13",
                                 )
                             ),
                         ),
@@ -211,9 +203,7 @@ def main():
                     Div(
                         A(
                             "Web Data Processing",
-                            href="/webdata#section1",
-                            hx_get="/webdata#section1",
-                            hx_target="#inner-text",
+                            href="#section21",
                         )
                     ),
                     Div(
@@ -221,41 +211,31 @@ def main():
                             Li(
                                 A(
                                     "Common Crawl Snapshot Processing",
-                                    href="/webdata#section1",
-                                    hx_get="/webdata#section1",
-                                    hx_target="#inner-text",
+                                    href="#section21",
                                 )
                             ),
                             Li(
                                 A(
                                     "Common Crawl Data Processing Summary",
-                                    href="/webdata#section2",
-                                    hx_get="/webdata#section2",
-                                    hx_target="#inner-text",
+                                    href="#section22",
                                 )
                             ),
                             Li(
                                 A(
                                     "Document Preparation",
-                                    href="/webdata#section3",
-                                    hx_get="/webdata#section3",
-                                    hx_target="#inner-text",
+                                    href="#section23",
                                 )
                             ),
                             Li(
                                 A(
                                     "Line-Level Removal",
-                                    href="/webdata#section4",
-                                    hx_get="/webdata#section4",
-                                    hx_target="#inner-text",
+                                    href="#section24",
                                 )
                             ),
                             Li(
                                 A(
                                     "Document-Level Filtering",
-                                    href="/webdata#section5",
-                                    hx_get="/webdata#section5",
-                                    hx_target="#inner-text",
+                                    href="#section25",
                                 )
                             ),
                         ),
@@ -263,9 +243,7 @@ def main():
                     Div(
                         A(
                             "Curated Sources Processing",
-                            href="/curated#section1",
-                            hx_get="/curated#section1",
-                            hx_target="#inner-text",
+                            href="#section31",
                         )
                     ),
                     Div(
@@ -273,25 +251,19 @@ def main():
                             Li(
                                 A(
                                     "Curated Sources in TxT360",
-                                    href="/curated#section1",
-                                    hx_get="/curated#section1",
-                                    hx_target="#inner-text",
+                                    href="#section31",
                                 )
                             ),
                             Li(
                                 A(
                                     "Filtering Steps and Definitions",
-                                    href="/curated#section2",
-                                    hx_get="/curated#section2",
-                                    hx_target="#inner-text",
+                                    href="#section32",
                                 )
                             ),
                             Li(
                                 A(
                                     "Filtering Discussion on All Curated Sources",
-                                    href="/curated#section3",
-                                    hx_get="/curated#section3",
-                                    hx_target="#inner-text",
+                                    href="#section33",
                                 )
                             ),
                         ),
@@ -299,9 +271,7 @@ def main():
                     Div(
                         A(
                             "Shared Processing Steps",
-                            href="/common#section1",
-                            hx_get="/common#section1",
-                            hx_target="#inner-text",
+                            href="#section41",
                         )
                     ),
                     Div(
@@ -309,65 +279,49 @@ def main():
                             Li(
                                 A(
                                     "Overview",
-                                    href="/common#section1",
-                                    hx_get="/common#section1",
-                                    hx_target="#inner-text",
+                                    href="#section41",
                                 )
                             ),
                             Li(
                                 A(
                                     "Motivation Behind Global Deduplication",
-                                    href="/common#section2",
-                                    hx_get="/common#section2",
-                                    hx_target="#inner-text",
+                                    href="#section42",
                                 )
                             ),
                             Li(
                                 A(
                                     "MinHash Generation",
-                                    href="/common#section3",
-                                    hx_get="/common#section3",
-                                    hx_target="#inner-text",
+                                    href="#section43",
                                 )
                             ),
                             Li(
                                 A(
                                     "Matching Pairs Generation",
-                                    href="/common#section4",
-                                    hx_get="/common#section4",
-                                    hx_target="#inner-text",
+                                    href="#section44",
                                 )
                             ),
                             Li(
                                 A(
                                     "Finding Duplicate Pairs",
-                                    href="/common#section5",
-                                    hx_get="/common#section5",
-                                    hx_target="#inner-text",
+                                    href="#section45",
                                 )
                             ),
                             Li(
                                 A(
                                     "Finding Connected Components using MapReduce",
-                                    href="/common#section6",
-                                    hx_get="/common#section6",
-                                    hx_target="#inner-text",
+                                    href="#section46",
                                 )
                             ),
                             Li(
                                 A(
                                     "Personally Identifiable Information Removal",
-                                    href="/common#section7",
-                                    hx_get="/common#section7",
-                                    hx_target="#inner-text",
+                                    href="#section47",
                                 )
                             ),
                             Li(
                                 A(
                                     "Normalization Form C",
-                                    href="/common#section8",
-                                    hx_get="/common#section8",
-                                    hx_target="#inner-text",
+                                    href="#section48",
                                 )
                             ),
                         ),
@@ -375,9 +329,7 @@ def main():
                     Div(
                         A(
                             "TxT360 Studies",
-                            href="/results#section1",
-                            hx_get="/results#section1",
-                            hx_target="#inner-text",
+                            href="#section51",
                         ),
                     ),
                     Div(
@@ -385,25 +337,19 @@ def main():
                             Li(
                                 A(
                                     "Overview",
-                                    href="/results#section1",
-                                    hx_get="/results#section1",
-                                    hx_target="#inner-text",
+                                    href="#section51",
                                 )
                             ),
                             Li(
                                 A(
                                     "Upsampling Experiment",
-                                    href="/results#section2",
-                                    hx_get="/results#section2",
-                                    hx_target="#inner-text",
+                                    href="#section52",
                                 )
                             ),
                             Li(
                                 A(
                                     "Perplexity Analysis",
-                                    href="/results#section3",
-                                    hx_get="/results#section3",
-                                    hx_target="#inner-text",
+                                    href="#section53",
                                 )
                             ),
                         ),
@@ -413,6 +359,10 @@ def main():
                 ),
             ),
             intro(),
+            curated.curated(),
+            web.web_data(),
+            common.common_steps(),
+            results.results(),
         ),
         D_appendix(
             D_bibliography(src="bibliography.bib"),
@@ -905,7 +855,7 @@ def intro():
             P(
                 "We documented all implementation details in this blog post and are open sourcing the code. Examples of each filter and rationale supporting each decision are included."
             ),
-            id="section1",
+            id="section11",
         ),
         Section(
             H2("Motivation Behind TxT360"),
@@ -923,7 +873,7 @@ def intro():
             ),
             # P("Table 2: Basic TxT360 Statistics."),
             # table_div_data,
-            id="section2",
+            id="section12",
         ),
         Section(
             H2("Our Generalizable Approach to Data Processing"),
@@ -944,7 +894,7 @@ def intro():
             # P(
             #    "Figure 1: Data processing pipeline. All the steps are adopted for processing web data while the yellow blocks are adopted for processing curated sources."
             # ),
-            id="section3",
+            id="section13",
         ),
         id="inner-text",
     )
@@ -952,12 +902,4 @@ def intro():
 
 rt("/update/{target}")(data_viewer.update)
 
-rt("/curated")(curated.curated)
-
-rt("/webdata")(web.web_data)
-
-rt("/common")(common.common_steps)
-
-rt("/results")(results.results)
-
 serve()

results.py

@@ -979,19 +979,19 @@ def results():
     return Div(
                 Section(
                     intro_div,
-                    id="section1"
+                    id="section51"
                 ),
                 Section(
                     upsampling_exp,
-                    id="section2"
+                    id="section52"
                 ),
                 Section(
                     preplexity_intro_div,
-                    id="section3"
+                    id="section53"
                 ),
                 Section(
                     perp1_div,
-                    id="section4"
+                    id="section54"
                 ),
                 Section(
                     llama_div,

style.css

@@ -288,3 +288,7 @@ d-appendix .citation {
     white-space: pre-wrap;
     word-wrap: break-word;
 }
+
+html {
+  scroll-behavior: smooth;
+}

web.py

@@ -390,7 +390,7 @@ def web_data():
         ),
         P("To generate a high-quality dataset from large-scale webpages, we have investigated the processing steps used by the community and made our choices based on careful manual inspection. Below is a comprehensive list of datasets we reviewed the comparison of filters we have applied."), 
         ),
-        id="section1",),
+        id="section21"),
         Section(
         H3("TxT360 CommonCrawl Filtering vs Other Pretraining Datasets"),
         P("The following section provides explicit details covering the reasoning and decisions behind each of the filters we applied. The table below provides a high-level comparison of TxT360's filtering compared to other commonly used pretraining datasets."),
@@ -402,7 +402,7 @@ def web_data():
         # The sankey diagram of the filtering percentage
         plotly2fasthtml(filtering_sankey_fig),
         P("A significant portion of the documents is filtered after the whole process. This figure illustrates the percentage of documents filtered at each step. The grey bars represent the filtered documents. The statistics are largely consistent with prior work (e.g., RefinedWeb) across most steps, though we have incorporated some custom filtering steps."),
-        id="section2",),
+        id="section22",),
         Section(
         H2("Document Preparation"),
         
@@ -563,7 +563,7 @@ def web_data():
             """, 
         ),
          
-        id="section3",),
+        id="section23",),
         Section(
         H2("Line-Level Removal"),
         P("""
@@ -677,7 +677,7 @@ def web_data():
             margin-bottom: 15px
             """, 
         ),
-        id="section4",),
+        id="section24",),
         Section(
         H2("Document-Level Filtering"),
         P("""
@@ -1748,5 +1748,5 @@ def web_data():
             margin-bottom: 15px
             """, 
         ),
-        id="section5",),
+        id="section25",),
     )