topic-analysis

common.py

@@ -298,6 +298,7 @@ global_div = Div(
                 style="margin-bottom: 5px",
             ),
             Li("Normalization Form C Discussion", style="margin-bottom: 5px"),
+            Li(B("Estimated Reading Time: 10 minutes"), style="margin-bottom: 5px"),
         ),
         id="section41",
     ),

curated.py

@@ -24,6 +24,7 @@ overview = (
                 "Individual Filtering Discussion for Each Source",
                 style="margin-bottom: 5px",
             ),
+            Li(B("Estimated Reading Time: 25 minutes"),style="margin-bottom: 5px", ),
         ),
     ),
 )
@@ -33,7 +34,9 @@ curated_sources_intro = Div(
     P(
         "Curated sources comprise high-quality datasets that contain domain-specificity.",
         B(
-            " TxT360 was strongly influenced by The Pile",  D_cite(bibtex_key="thepile"), " regarding both inclusion of the dataset and filtering techniques."
+            " TxT360 was strongly influenced by The Pile",
+            D_cite(bibtex_key="thepile"),
+            " regarding both inclusion of the dataset and filtering techniques.",
         ),
         " These sources, such as Arxiv, Wikipedia, and Stack Exchange, provide valuable data that is excluded from the web dataset mentioned above. Analyzing and processing non-web data can yield insights and opportunities for various applications. Details about each of the sources are provided below. ",
     ),
@@ -129,16 +132,16 @@ wikipedia_filter = pd.DataFrame(
             "0.00%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.31%",
         ],
         "Total Percentage Remaining": [
-            "",
+            "97.84%",
         ],
     }
 )
 
 table_html_wikipedia = wikipedia_filter.to_html(index=False, border=0)
-table_div_wikipedia = Div(NotStr(table_html_wikipedia), style="margin-left: auto; width: 80%; align: center;")
+table_div_wikipedia = Div(NotStr(table_html_wikipedia))
 
 freelaw_filter = pd.DataFrame(
     {
@@ -167,7 +170,7 @@ freelaw_filter = pd.DataFrame(
 )
 
 table_html_freelaw = freelaw_filter.to_html(index=False, border=0)
-table_div_freelaw = Div(NotStr(table_html_freelaw), style="margin-left: auto; width: 80%; align: center;")
+table_div_freelaw = Div(NotStr(table_html_freelaw))
 
 dmm_filter = pd.DataFrame(
     {
@@ -187,16 +190,16 @@ dmm_filter = pd.DataFrame(
             "0.00%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "100.00%",
         ],
     }
 )
 
 table_html_dmm = dmm_filter.to_html(index=False, border=0)
-table_div_dmm = Div(NotStr(table_html_dmm), style="margin-left: auto; width: 80%; align: center;")
+table_div_dmm = Div(NotStr(table_html_dmm))
 
 
 uspto_filter = pd.DataFrame(
@@ -217,16 +220,16 @@ uspto_filter = pd.DataFrame(
             "0.01%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "22.94%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "75.15%",
         ],
     }
 )
 
 table_html_uspto = uspto_filter.to_html(index=False, border=0)
-table_div_uspto = Div(NotStr(table_html_uspto), style="margin-left: auto; width: 80%; align: center;")
+table_div_uspto = Div(NotStr(table_html_uspto))
 
 pg19_filter = pd.DataFrame(
     {
@@ -246,16 +249,16 @@ pg19_filter = pd.DataFrame(
             "0.17%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.80%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "98.78%",
         ],
     }
 )
 
 table_html_pg19 = pg19_filter.to_html(index=False, border=0)
-table_div_pg19 = Div(NotStr(table_html_pg19), style="margin-left: auto; width: 80%; align: center;")
+table_div_pg19 = Div(NotStr(table_html_pg19))
 
 
 hn_filter = pd.DataFrame(
@@ -267,7 +270,7 @@ hn_filter = pd.DataFrame(
             "2064931",
         ],
         "Percent Removed After Language Filter": [
-            "2.62%%",
+            "2.62%",
         ],
         "Percent Removed After Min Word Count Filter": [
             "0.02%",
@@ -276,16 +279,16 @@ hn_filter = pd.DataFrame(
             "0.34%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "61.84%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "35.18%",
         ],
     }
 )
 
 table_html_hn = hn_filter.to_html(index=False, border=0)
-table_div_hn = Div(NotStr(table_html_hn), style="margin-left: auto; width: 80%; align: center;")
+table_div_hn = Div(NotStr(table_html_hn))
 
 
 uirc_filter = pd.DataFrame(
@@ -306,16 +309,16 @@ uirc_filter = pd.DataFrame(
             "1.12%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.66%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "59.98%",
         ],
     }
 )
 
 table_html_uirc = uirc_filter.to_html(index=False, border=0)
-table_div_uirc = Div(NotStr(table_html_uirc), style="margin-left: auto; width: 80%; align: center;")
+table_div_uirc = Div(NotStr(table_html_uirc))
 
 up_filter = pd.DataFrame(
     {
@@ -335,16 +338,16 @@ up_filter = pd.DataFrame(
             "0.00%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "1.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "99.00%",
         ],
     }
 )
 
 table_html_up = up_filter.to_html(index=False, border=0)
-table_div_up = Div(NotStr(table_html_up), style="margin-left: auto; width: 80%; align: center;")
+table_div_up = Div(NotStr(table_html_up))
 
 se_filter = pd.DataFrame(
     {
@@ -364,16 +367,16 @@ se_filter = pd.DataFrame(
             "0.00%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "100.00%",
         ],
     }
 )
 
 table_html_se = se_filter.to_html(index=False, border=0)
-table_div_se = Div(NotStr(table_html_se), style="margin-left: auto; width: 80%; align: center;")
+table_div_se = Div(NotStr(table_html_se))
 
 arx_filter = pd.DataFrame(
     {
@@ -393,16 +396,16 @@ arx_filter = pd.DataFrame(
             "0.07%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "92.06%",
         ],
     }
 )
 
 table_html_arx = arx_filter.to_html(index=False, border=0)
-table_div_arx = Div(NotStr(table_html_arx), style="margin-left: auto; width: 80%; align: center;")
+table_div_arx = Div(NotStr(table_html_arx))
 
 s2o_filter = pd.DataFrame(
     {
@@ -422,16 +425,16 @@ s2o_filter = pd.DataFrame(
             "0.00%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "100.00%",
         ],
     }
 )
 
 table_html_s2o = s2o_filter.to_html(index=False, border=0)
-table_div_s2o = Div(NotStr(table_html_s2o), style="margin-left: auto; width: 80%; align: center;")
+table_div_s2o = Div(NotStr(table_html_s2o))
 
 med_filter = pd.DataFrame(
     {
@@ -451,16 +454,16 @@ med_filter = pd.DataFrame(
             "0.02%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "91.03%",
         ],
     }
 )
 
 table_html_med = med_filter.to_html(index=False, border=0)
-table_div_med = Div(NotStr(table_html_med), style="margin-left: auto; width: 80%; align: center;")
+table_div_med = Div(NotStr(table_html_med))
 
 phil_filter = pd.DataFrame(
     {
@@ -480,16 +483,16 @@ phil_filter = pd.DataFrame(
             "0.12%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "79.20%",
         ],
     }
 )
 
 table_html_phil = phil_filter.to_html(index=False, border=0)
-table_div_phil = Div(NotStr(table_html_phil), style="margin-left: auto; width: 80%; align: center;")
+table_div_phil = Div(NotStr(table_html_phil))
 ## end individual tables showing filterin
 
 
@@ -681,24 +684,51 @@ filtering_process = Div(
             P(
                 B("Download and Extraction: "),
                 "All the data was downloaded in original latex format from ArXiv official S3 repo: ",
-                A("s3://arxic/src", href="s3://arxic/src"),
-                ". We try to encode the downloaded data into utf-8 or guess encoding using chardet library. After that pandoc was used to extract information from the latex files and saved as markdown format",
+                A("s3://arxiv/src", href="s3://arxiv/src"),
+                ". We aim to encode the downloaded data in UTF-8 format, and when necessary, utilize the chardet library to infer the appropriate encoding. After that, we use ",
+                A("Pandoc", href="https://pandoc.org/"),
+                " to extract information from the latex files into markdown format. The command we use is",
                 D_code(
-                    "pandoc -s {tex} -o out/{out_name}.md --wrap=none",
-                    language="python",
+                    "pandoc <raw_tex_path> -s -o <output_markdown_path> -f latex+raw_tex -t markdown_mmd [--lua-filter <lua_filter_path>]",
+                    language="bash",
                 ),
-                ". All markdowns were combined to create jsonl files.",
+                ". Finally, all markdowns were combined to create jsonl files.",
             ),
             P(B("Unique Data Preparation Challenges: ")),
+            P(
+                "When converting LaTeX files into Markdown using Pandoc, it is crucial to account for different data formats to minimize information loss while also filtering out noisy content in LaTeX. Below, we outline our considerations and methods for handling various data types during this conversion process:"
+            ),
             Ul(
                 Li(
-                    "Due to large amounts of meaningful data being contained in table formats, special consideration was taken to extract the data and proper metadata.",
+                    B("Tables: "),
+                    "The process for handling tables follows three main approaches. First, tables compatible with Pandoc’s built-in formats are directly converted into standard Markdown tables. Notably, LaTeX’s '\\multicolumn' and '\\multirow' commands can be successfully translated into valid Markdown tables. Second, tables unsupported by Pandoc’s native functionality, such as deluxetable or other complex LaTeX types, are preserved in their original LaTeX format to maintain the integrity of complex structures. Third, only a few remaining tables have been converted to HTML web tables.",
+                    style="margin-bottom: -3px",
+                ),
+                Li(
+                    B("Mathematical Expressions: "),
+                    "Inline mathematical expressions are rendered in Markdown. More complex equations remain unchanged, e.g., presented as '\\begin{aligned}' blocks, to ensure accuracy and readability.",
+                    style="margin-bottom: -3px",
+                ),
+                Li(
+                    B("Figures: "),
+                    "All figures are removed during the conversion process. Placeholder figures might not contribute to the paper’s data quality and, as such, have been omitted to streamline the output.",
+                    style="margin-bottom: -3px",
+                ),
+                Li(
+                    B("Section Headers: "),
+                    "Section headers are converted into markdown format, using leading '#' symbols to represent the heading levels.",
+                    style="margin-bottom: -3px",
+                ),
+                Li(
+                    B("References: "),
+                    "References are removed. Although they may be informative, references often introduce formatting inconsistencies or add little value compared to the core content of the paper.",
                     style="margin-bottom: -3px",
                 ),
             ),
             P(
                 B(" Filters Applied: "),
-                "multiple filters are used here after manually verifying output of all the filters as suggested by peS2o dataset",  D_cite(bibtex_key="peS2o"),
+                "multiple filters are used here after manually verifying output of all the filters as suggested by peS2o dataset",
+                D_cite(bibtex_key="peS2o"),
             ),
             Ul(
                 Li(
@@ -773,19 +803,19 @@ filtering_process = Div(
                 ),
             ),
             table_div_s2o,
-            Details(
-                Summary("S2ORC Filtering Examples -- need to update"),
-                Div(
-                    P("examples are missing"),
-                    style="background-color: white; padding: 15px; margin-top: 10px; margin-bottom: 10px; border-radius: 8px; border: none; ",  # Styling for the DV2 part
-                ),
-                style="""
-            background-color: #FFFAEA; /* Light yellow background */
-            padding: 15px;
-            border-radius: 12px;
-            margin-bottom: 15px
-            """,
-            ),
+           # Details(
+           #     Summary("S2ORC Filtering Examples -- need to update"),
+           #     Div(
+           #         P("examples are missing"),
+           #         style="background-color: white; padding: 15px; margin-top: 10px; margin-bottom: 10px; border-radius: 8px; border: none; ",  # Styling for the DV2 part
+           #     ),
+           #     style="""
+           # background-color: #FFFAEA; /* Light yellow background */
+           # padding: 15px;
+           # border-radius: 12px;
+           # margin-bottom: 15px
+           # """,
+           # ),
         ),
     ),
     Section(
@@ -825,19 +855,19 @@ filtering_process = Div(
                     style="margin-bottom: -3px",
                 ),
             ),
-            Details(
-                Summary("S2ORC Abstract Filtering Examples "),
-                Div(
-                    P("examples are missing"),
-                    style="background-color: white; padding: 15px; margin-top: 10px; margin-bottom: 10px; border-radius: 8px; border: none; ",  # Styling for the DV2 part
-                ),
-                style="""
-            background-color: #FFFAEA; /* Light yellow background */
-            padding: 15px;
-            border-radius: 12px;
-            margin-bottom: 15px
-            """,
-            ),
+            #Details(
+            #    Summary("S2ORC Abstract Filtering Examples "),
+           #     Div(
+           #         P("examples are missing"),
+           #         style="background-color: white; padding: 15px; margin-top: 10px; margin-bottom: 10px; border-radius: 8px; border: none; ",  # Styling for the DV2 part
+           #     ),
+           #     style="""
+           # background-color: #FFFAEA; /* Light yellow background */
+           # padding: 15px;
+           # border-radius: 12px;
+           # margin-bottom: 15px
+           # """,
+           # ),
         )
     ),
     Section(
@@ -851,13 +881,16 @@ filtering_process = Div(
                     href="ttps://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/",
                 ),
                 ". PubMed Central (PMC) files are downloaded in an xml.tar format. The tar files are opened and converted to markdown format using pandoc",
-                D_code("pandoc -f jats {nxml} -o {pmcid}.md", language="bash"),
-                ". The markdown files are combined to create jsonl files. PubMed Abstract (PMA) files  were downloaded in xml. The BeautifulSoup library was used to extract the abstract, title, and PMID. All files were stored in jsonl format.",
+                D_code(
+                    "pandoc <raw_xml_path> -s -o <output_markdown_path> -f jats -t markdown_mmd [--lua-filter <lua_filter_path>]",
+                    language="bash",
+                ),
+                ". The markdown files are combined to create jsonl files. PubMed Abstract (PMA) files were downloaded in xml. The BeautifulSoup library was used to extract the abstract, title, and PMID. All files were stored in jsonl format.",
             ),
             P(B("Unique Data Preparation Challenges: ")),
             Ul(
                 Li(
-                    "Due to large amounts of meaningful data being contained in table formats, speical consideration was taken to extract the data and proper metadata.",
+                    "We tried similar attempts on PMC as we did on ArXiv. The resulted markdown might have slight difference due to the different structure of the XML files.",
                     style="margin-bottom: -3px",
                 ),
             ),
@@ -1584,7 +1617,8 @@ def curated():
     table_html = data_preparation_steps.to_html(index=False, border=0)
     table_div = Div(NotStr(table_html), style="margin: 40px;")
 
-    text = P("""This initial stage serves as the foundation for the entire
+    text = P(
+        """This initial stage serves as the foundation for the entire
     process. Here, we focus on acquiring and extracting the raw data, which can
     come from various sources such as crawling websites, using HTTP/FTP dumps,
     or working with archive dumps.  For instance, to download and prepare a
@@ -1594,7 +1628,8 @@ def curated():
     preparation process: It is worth noting that some pipelines might require
     invoking additional functions or scripts to handle specific data sources or
     formats.  These helper scripts can be located within specific directories
-    or modules dedicated to the dataset.""")
+    or modules dedicated to the dataset."""
+    )
 
     return Div(
         Section(

main.py

@@ -175,7 +175,7 @@ def main():
                     Div(
                         A(
                             "TxT360",
-                            href="#section1",
+                            href="#section11",
                         )
                     ),
                     Div(
@@ -352,6 +352,12 @@ def main():
                                     href="#section53",
                                 )
                             ),
+                            Li(
+                                A(
+                                    "Topic Analysis",
+                                    href="#section55",
+                                )
+                            )
                         ),
                     ),
                     role="navigation",
@@ -359,8 +365,8 @@ def main():
                 ),
             ),
             intro(),
-            curated.curated(),
             web.web_data(),
+            curated.curated(),
             common.common_steps(),
             results.results(),
         ),
@@ -757,7 +763,7 @@ dataset_sources = pd.DataFrame(
             "StackExchange",
         ],
         "Raw Data Size": [
-            "11 TB",
+            "9.2 TB",
             "712 GB",
             "210 GB",
             "23 GB",
@@ -770,7 +776,7 @@ dataset_sources = pd.DataFrame(
             "45 GB",
         ],
         "Token Count": [
-            "5.71T",
+            "4.83T",
             "154.96B",
             "4.75B",
             "7.34B",

results.py

@@ -157,81 +157,69 @@ fig_loss.update_layout(
 lm_loss_graph = fig_loss
 
 
-#Perplexity Across Different Buckets (global)
-# The data you provided
-DATA = [
-    ["2014", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.410227605477868, 16.11176217183986, 15.632757662414805, 15.446116676532212, 16.716943171826703, 18.156821563322765]]],
-    ["2015", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.446573602753478, 16.14852530113782, 15.627408549576069, 15.0055028132117, 15.565430373421485, 17.314701050452452]]],
-    ["2016", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.307221780905284, 16.297702171159543, 15.948641884223639, 14.799690714225637, 14.935989931859659, 16.09585768919658]]],
-    ["2017", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.338525603992114, 15.960924352297502, 15.912187993988933, 14.822102470001267, 14.778913482337416, 15.428145290012955]]],
-    ["2018", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.08551151136689, 16.187802102106698, 14.935072408852303, 14.832038213200583, 14.508674264491997, 14.800605964649103]]],
-    ["2019", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [16.818363305107052, 16.474269837858706, 14.944741674400241, 14.568394784374943, 14.690158822673334, 15.990949424635108]]],
-    ["2020", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [16.98821894111693, 15.936494557783181, 14.79960386342691, 14.435682562274105, 14.58651834886038, 15.869365567783806]]],
-    ["2021", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.125795647512877, 15.780419457145868, 14.631430892394002, 14.276477514399625, 14.337146941773641, 15.872474774329305]]],
-    ["2022", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [16.573462144306383, 15.283018703313582, 14.378277745163881, 14.0611924390084, 13.9886330091318, 15.769421394877273]]],
-    ["2023", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [15.4293630385597, 14.608379914730168, 14.118271697056592, 13.880215644749589, 13.767106666731275, 15.05749135510839]]]
-]
-
-# Extract ranges (buckets) and years
-ranges = DATA[0][1][0]
-years = [year_data[0] for year_data in DATA]
-all_values = [year_data[1][1] for year_data in DATA]
+data = {
+    "1-1": [17.410227605477868, 17.446573602753478,17.307221780905284,17.338525603992114,17.08551151136689,16.818363305107052,16.98821894111693, 17.125795647512877,16.573462144306383, 15.4293630385597],
+    "2-5": [16.11176217183986,16.14852530113782,16.297702171159543,15.960924352297502,16.187802102106698,16.474269837858706,15.936494557783181, 15.780419457145868,15.283018703313582, 14.608379914730168],
+    "6-10": [15.632757662414805,15.627408549576069,15.948641884223639,15.912187993988933,14.935072408852303,14.944741674400241,14.79960386342691,14.631430892394002, 14.378277745163881,14.118271697056592],
+    "11-100": [15.446116676532212,15.0055028132117,14.799690714225637,14.822102470001267,14.832038213200583,14.568394784374943,14.435682562274105,14.276477514399625, 14.0611924390084,13.880215644749589],
+    "101-1000": [16.716943171826703,15.565430373421485,14.935989931859659,14.778913482337416,14.508674264491997,14.690158822673334,14.58651834886038,14.337146941773641,13.9886330091318,13.767106666731275 ],
+    "1001-30000000": [18.156821563322765,17.314701050452452,16.09585768919658,15.428145290012955, 14.800605964649103, 15.990949424635108,15.869365567783806,15.872474774329305, 15.769421394877273, 15.05749135510839],
+}
 
-# Create the figure
-fig = go.Figure()
+# Years for the x-axis
+years = ["2014", "2015", "2016", "2017", "2018", "2019", "2020", "2021", "2022", "2023"]
 
-# Add a trace for each year
-for i, year in enumerate(years):
-    values = all_values[i]
-    fig.add_trace(go.Scatter(x=ranges, y=values, mode='lines+markers', name=year))
+# Create a plotly figure
+fig11 = go.Figure()
 
-# Update layout
-fig.update_layout(
-    title="Perplexity Versus Buckets for Different Years",
-    xaxis_title="Buckets",
-    yaxis_title="Perplexity",
-    legend_title="Years",
-    hovermode="x unified"
+# Add a trace for each bucket
+for bucket, values in data.items():
+    fig11.add_trace(go.Scatter(x=years, y=values, mode='lines', name=bucket))
+
+# Update layout for better presentation
+fig11.update_layout(
+    title='Perplexity Versus Buckets for Different Years',
+    xaxis_title='Year',
+    yaxis_title='Perplexity',
+    xaxis_tickangle=-45,
+    legend_title="Buckets",
 )
 
-Perplexity_Across_Different_Buckets_global_graph = fig
+Perplexity_Across_Different_Buckets_global_graph = fig11
 
 ##graph 2
 
 # Data
-years = ["2014", "2015", "2016", "2017", "2018", "2019", "2020", "2021", "2022", "2023"]
-buckets = ["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"]
 data = {
-    "2014": [17.410227605477868, 16.11176217183986, 15.632757662414805, 15.446116676532212, 16.716943171826703, 18.156821563322765],
-    "2015": [17.446573602753478, 16.14852530113782, 15.627408549576069, 15.0055028132117, 15.565430373421485, 17.314701050452452],
-    "2016": [17.307221780905284, 16.297702171159543, 15.948641884223639, 14.799690714225637, 14.935989931859659, 16.09585768919658],
-    "2017": [17.338525603992114, 15.960924352297502, 15.912187993988933, 14.822102470001267, 14.778913482337416, 15.428145290012955],
-    "2018": [17.08551151136689, 16.187802102106698, 14.935072408852303, 14.832038213200583, 14.508674264491997, 14.800605964649103],
-    "2019": [16.818363305107052, 16.474269837858706, 14.944741674400241, 14.568394784374943, 14.690158822673334, 15.990949424635108],
-    "2020": [16.98821894111693, 15.936494557783181, 14.79960386342691, 14.435682562274105, 14.58651834886038, 15.869365567783806],
-    "2021": [17.125795647512877, 15.780419457145868, 14.631430892394002, 14.276477514399625, 14.337146941773641, 15.872474774329305],
-    "2022": [16.573462144306383, 15.283018703313582, 14.378277745163881, 14.0611924390084, 13.9886330091318, 15.769421394877273],
-    "2023": [15.4293630385597, 14.608379914730168, 14.118271697056592, 13.880215644749589, 13.767106666731275, 15.05749135510839]
+    "1-1": [17.410227605477868, 17.446573602753478,17.307221780905284,17.338525603992114,17.08551151136689,16.818363305107052,16.98821894111693, 17.125795647512877,16.573462144306383, 15.4293630385597],
+    "2-5": [16.11176217183986,16.14852530113782,16.297702171159543,15.960924352297502,16.187802102106698,16.474269837858706,15.936494557783181, 15.780419457145868,15.283018703313582, 14.608379914730168],
+    "6-10": [15.632757662414805,15.627408549576069,15.948641884223639,15.912187993988933,14.935072408852303,14.944741674400241,14.79960386342691,14.631430892394002, 14.378277745163881,14.118271697056592],
+    "11-100": [15.446116676532212,15.0055028132117,14.799690714225637,14.822102470001267,14.832038213200583,14.568394784374943,14.435682562274105,14.276477514399625, 14.0611924390084,13.880215644749589],
+    "101-1000": [16.716943171826703,15.565430373421485,14.935989931859659,14.778913482337416,14.508674264491997,14.690158822673334,14.58651834886038,14.337146941773641,13.9886330091318,13.767106666731275 ],
+    "1001-30000000": [18.156821563322765,17.314701050452452,16.09585768919658,15.428145290012955, 14.800605964649103, 15.990949424635108,15.869365567783806,15.872474774329305, 15.769421394877273, 15.05749135510839],
 }
-
 # Create a line plot for each bucket
-fig = go.Figure()
+# Years
+years = ["2014", "2015", "2016", "2017", "2018", "2019", "2020", "2021", "2022", "2023"]
 
-for i, bucket in enumerate(buckets):
-    bucket_values = [data[year][i] for year in years]
-    fig.add_trace(go.Scatter(x=years, y=bucket_values, mode='lines+markers', name=bucket))
+# Create the figure
+fig22 = go.Figure()
+
+# Add lines for each bucket
+for bucket, perplexities in data.items():
+    fig22.add_trace(go.Scatter(x=years, y=perplexities, mode='lines+markers', name=bucket))
 
 # Update layout
-fig.update_layout(
-    title="Average Perplexity Over Years by Bucket",
+fig22.update_layout(
+    title="Perplexity Across Different Years (Global)",
     xaxis_title="Year",
     yaxis_title="Average Perplexity",
-    legend_title="Buckets",
-    hovermode="x unified"
+    legend_title="Bucket (duplicate count range)"
 )
 
-# Show plot
-graph2 = fig
+# Show the figure
+graph2222 = fig22
+
 
 #graph 3 tbd
 
@@ -277,8 +265,11 @@ data = {
         "y": [15.4293630385597, 14.827776633211421, 14.600432832118155, 14.43330043760322, 14.30586483681026, 14.203397641081045, 14.140361413924607, 14.101673126860582, 14.05840021694595, 14.033693337279875, 13.990003714671388, 14.001106927608756, 13.982387676044238, 13.960424890216352, 13.941694305252629, 13.928958405693843, 13.922871327026984, 13.90749356497257, 13.883187320364065, 13.870538613453949, 13.853682922141118, 13.839326154723096, 13.841949693311191, 13.851284862386178, 13.853606323578846, 13.851840301257587, 13.86166226046842, 13.872513892742713, 13.867295530090015, 13.870385212514083, 13.868578068850889, 13.848685425568009, 13.838736750620761, 13.825182992628129, 13.795340100698754, 13.809629502148145, 13.767911089744738, 13.75599639262174, 13.75925151191706, 13.751975433642748, 13.731931931502134, 13.714316407794309, 13.730848265421725, 13.681606796845, 13.670846152397202, 13.685040324991581, 13.687292733648798, 13.645627677729081, 13.635021438002346, 13.670103747374988, 13.62246956240464, 13.658574692934657, 13.65219324836813, 13.663836335892329, 13.666948307267594, 13.642643510414398, 13.672961601819406, 13.663525877548398, 13.667436573958156, 13.687522639036205, 13.647544546546508, 13.670704172336292, 13.643000490240736, 13.667107110344569, 13.660218070172265, 13.675550822990397, 13.59092599672469, 13.619850375757148, 13.651551988902462, 13.684288098100867, 13.599046453546292, 13.580905963181452, 13.620956017533885, 13.648408527460056, 13.60249514150298, 13.568078301644128, 13.6145797181229, 13.647719674647586, 13.531493703586886, 13.609260600121146, 13.687447710013837, 13.602215210547463, 13.563220813507392, 13.57176728376717, 13.664222431811334, 13.586259696651297, 13.571127927324502, 13.692682818016978, 13.599437831977406, 13.646024625160337, 13.645231022004468, 13.582760709524955, 13.597408008025921, 13.640791164279545, 13.58016693386471, 13.664139165839629, 13.553901320176095, 13.616471379535914, 13.574418885444663, 13.62049913139043, 13.595296717779055, 13.618520397945241, 13.619083853308746, 13.549404239296155, 13.515187660214737, 13.728226318739061, 13.618609127026419, 13.638021829042664, 13.684199453984574, 13.707676555543845, 13.541617144639595, 13.703834138276244, 13.519323579997998, 13.6031555746482, 13.56414632339735, 13.711610914557605, 13.529701508251849, 13.631734977501925, 13.657120297408555, 13.58489249685537, 13.64403630157245, 13.608235747585796, 13.512110498832085, 13.846959297419959, 13.488917791046676, 13.696834245583917, 13.61507766454769, 13.565377802829648, 13.790197786574588, 13.564346823139237, 13.5342157764538, 13.65148446769962, 13.570646849759997, 13.538878864721305, 13.608980817839505, 13.664477237928127, 13.532411719398455, 13.737437273256564, 13.519816331488395, 13.764368990604515, 13.653588472445573, 13.5302540795639, 13.57561188810073, 13.647608383912683, 13.728643890026184, 13.710097008284352, 13.75589344505474, 13.682502650080487, 13.684339202206958, 13.86749342599449, 13.645786331176726, 13.660817039716257, 13.638011389586667, 13.732387353692099, 13.687686391049773, 13.94728981311995, 13.614547412507926, 13.460552695752389, 13.857074456574574, 13.660326212827487, 13.718393121459215, 13.76973846040823, 13.778762864123724, 13.656959991670522, 14.214462072816556, 13.622739420457773, 13.645227120727897, 13.687497326712435, 13.649272554141808, 13.503550000252996, 14.052494437028535, 13.580311058963396, 14.14017314736784, 13.64182000911326, 13.886936632170794, 13.942674178813384, 13.801754161915504, 13.931144818346747, 13.584651322907064, 13.960391830857033, 14.178873989248789, 13.645923431288137, 13.97423140921055, 13.669569353257327, 13.915331700422486, 13.827145087821776, 13.736836827537942, 13.74766543270326, 13.770318536193258, 13.721033560006754, 13.710191655241298, 13.72744974500201, 13.845900599098753, 13.678446729208007, 13.902643304888189, 13.885717101405227, 13.800092557910519, 13.421930828385424, 14.22006344809192, 13.879986040380091, 14.134020605148754, 13.828712030750555, 13.713190050100788, 13.670401789346878, 13.948423234333882, 13.84481181587886, 13.818908981670926, 13.74582730003707, 13.94097096205736, 14.180988587482524, 13.792218775776533, 13.684855627228844, 13.881739174231223, 13.998746647008945, 14.576445279868192, 13.977974630190932, 13.537548783758675, 13.793279757360363, 13.712674433009878, 13.758696209000801, 13.912382937440464, 14.340144810498328, 14.03279240397904, 13.889996587251982, 13.862065986515601, 13.712302898403514, 13.797613183843772, 13.847295441275936, 14.422108283887903, 13.900169262027397, 14.247783256416513, 14.02056653491548, 13.99526374998638, 13.69358071050138, 13.828640744267325, 14.01321596237364, 13.824657271537166, 13.683602248925892, 13.933987455146445, 13.993921243916123, 13.720966315955001, 13.890770809341586, 13.426961283787957, 13.789886238259456, 13.904131224083592, 13.697354217113691, 13.710497864728065, 14.683006531539148, 14.315759264484964, 14.034699751802846, 13.890948305458895, 14.038907287423818, 13.298757873288494, 14.037824844116592, 14.162250808384407, 13.947058195824253, 13.806653590867487, 13.980386450979841, 13.667273172120268, 14.201737731565034, 14.447849496372461, 14.018697147254176, 14.002315381014313, 14.15231239942244, 13.73719042552595, 14.033594655653038, 14.03087108296539, 13.782578520604822, 13.660908086958573, 14.205910336717215, 13.930322566343703, 14.300591211697906, 13.808749910964323, 13.8076208806989, 14.169819824542373, 14.067537576689853, 14.306124453553517, 13.56827905088618, 13.592634469105525, 13.789396713937975, 13.853464456609156, 13.855768885291056, 13.710464058237172, 13.641314331273772, 14.295528532894258, 14.04977108393012, 14.153384726345907, 13.385524853965638, 13.484650040422745, 13.831061312642468, 14.15298734516191, 14.20289531849528, 14.116529526532855, 13.767652893233096, 13.715924776449405, 14.649027392365024, 13.603310028324861, 14.10228343978809, 13.704762155010586, 13.731715455443299], 
     },
 }
+
+fig33 = go.Figure()
+
 for year, year_data in data.items():
-    fig.add_trace(go.Scatter(
+    fig33.add_trace(go.Scatter(
         x=year_data['x'], 
         y=year_data['y'], 
         mode='lines+markers', 
@@ -287,7 +278,7 @@ for year, year_data in data.items():
     ))
 
 # Update layout
-fig.update_layout(
+fig33.update_layout(
     title="Perplexity vs. Number of Duplicate Documents Over Years",
     xaxis_title="Number of Duplicate Documents",
     yaxis_title="Average Perplexity",
@@ -295,7 +286,7 @@ fig.update_layout(
     hovermode="x unified"
 )
 
-graph3 = fig
+graph3 = fig33
 
 ##graph 4
 
@@ -839,6 +830,8 @@ intro_div = Div(
     Ul(
         Li("The Learning Curve of TxT360 with an Upsampling Recipe", style = "margin-bottom: 5px"),
         Li("Perplexity Analysis across time", style = "margin-bottom: 5px"),
+        Li("Topic Analysis on Data Cluster Groups", style = "margin-bottom: 5px"),
+        Li(B("Estimated Reading Time: 15 minutes"), style = "margin-bottom: 5px"),
     ),
 )
 
@@ -910,16 +903,15 @@ perp1_div = Div(
         Section(
             H3("Perplexity vs Years"),
             P("Taking the same data, we can convert it into a graph indicating the yearly trend. For most buckets, the average perplexity of dumps from more recent years seem to be lower than that of former years."),
-            Img(src="images/prep-across-diff-year-global-dup-buckets.png", height = "300", width = "600" ),
-            plotly2fasthtml(graph2),
-            P("NEED TO UPDATE - THIS GRAPH SHOULD MATCH THE IMAGE ABOVE AND YEAR SHOULD NOT BE a LINE OPTION"), 
+            #Img(src="images/prep-across-diff-year-global-dup-buckets.png", height = "300", width = "600" ),
+            plotly2fasthtml(graph2222),
+            
         ),
     Section(
             H3("Perplexity vs Document Duplication"),
             P("We can also break each bucket into distinct document counts. The graph becomes a bit noisy at the end because of insufficient samples with larger duplication counts."),
-            Img(src="images/prep-across-diff-docs-dup-count-global.png", height = "300", width = "600" ),
+            #Img(src="images/prep-across-diff-docs-dup-count-global.png", height = "300", width = "600" ),
             plotly2fasthtml(graph3),
-            P("NEED TO UPDATE - THIS GRAPH SHOULD MATCH THE IMAGE ABOVE AND BUCKET SHOULD NOT BE a LINE OPTION"), 
         ),
     Section(
             H3("Perplexity vs Dump Duplication"),
@@ -974,6 +966,51 @@ llama_div = Div(
         ),
 )
 
+with open(os.path.join(os.path.dirname(__file__), "data", "topic_charts.json"), 'r') as f:
+    topic_charts = json.load(f)
+topic_graphs = []
+
+for title, data in topic_charts:
+    if data["type"] == "barh":
+        topic_graphs.append(go.Figure(go.Bar(
+            x=data["kwargs"]["width"],
+            y=data["kwargs"]['y'],
+            orientation='h',
+            marker_color=[
+                "rgb(" + ", ".join(str(val * 255) for val in rgb) + ')'
+                for rgb in data["kwargs"]["color"]
+            ]
+        )))
+    elif data["type"] == "pie":
+        topic_graphs.append(go.Figure(go.Pie(
+            values=data["kwargs"]['x'],
+            labels=data["kwargs"]["labels"],
+            marker_colors=[
+                "rgb(" + ", ".join(str(val * 255) for val in rgb) + ')'
+                for rgb in data["kwargs"]["colors"]
+            ]
+        )))
+
+cluster_div = Div(
+    Section(
+        H2("Topic Analysis"),
+        P("We tried to classify data into topic groups and looked for correlations between topics and statistics of data. Data from different topic groups should manifest different characteristics of distribution, which can give us some insight into the composition of dataset."),
+        H3("Methodology"),
+        P("We took the ", A("common crawl", href="https://commoncrawl.org/"), " data and clustered them into 17 topic groups using ", A("BERTopic", href="https://maartengr.github.io/BERTopic/index.html"), ". We collected and aggregated a series of metrics which include quality signals and other useful metadata. For each topic group, we calculated average scores and generated the corresponding bar charts over different metrics for comparison and analysis."),
+        H3("Cluster Groups"),
+        P("We grouped data into the following 17 clusters"),
+        Ul(*(
+            Li(topic_name, style = "margin-bottom: 5px")
+            for topic_name in ("Arts", "Business & Economics & Finance", "Culture & Cultural geography", "Daily Life & Home & Lifestyle", "Education", "Entertainment & Travel & Hobby", "Environment", "Food & Drink & Cooking", "Health & Wellness & Medicine", "Law & Justice", "Natural Science & Formal Science & Technology", "Personal Development & Human Resources & Career", "Politics & Government", "Religion & Spirituality", "Shopping & Commodity", "Society & Social Issues & Human Rights", "Sports")
+        )),
+        H3("Results Analysis"),
+        *(
+            Section(H4(title), plotly2fasthtml(topic_graphs[i]), P(data.get("comment", '')))
+            for i, (title, data) in enumerate(topic_charts)
+        )
+    )
+)
+
 
 def results():
     return Div(
@@ -995,6 +1032,10 @@ def results():
                 ),
                 Section(
                     llama_div,
+                ),
+                Section(
+                    cluster_div,
+                    id="section55"
                 ),
                     id="inner-text"
                 )

web.py

@@ -376,6 +376,7 @@ def web_data():
     return Div(
         Section(
         Div(
+        H1("Web Data Processing"),
         H2("Common Crawl Snapshot Processing"),
         H3("What This Section Contains"),
         P("This section provides a complete discussion on the filtering applied to the 99 Common Crawl snapshots that comprise the web data section of TxT360. The section is split into the following topic areas: "),
@@ -387,6 +388,7 @@ def web_data():
             Li("Each section is complete with code and comparisons to Dolma,", D_cite(bibtex_key="soldaini2024dolma"),
                "DataTrove,", D_cite(bibtex_key="penedo2024datatrove"),
                "and/or RedPajama-V-2", D_cite(bibtex_key="redpajama-v2"), style = "margin-bottom: 5px"),
+            Li(B("Estimated Reading Time: 31 minutes"), style = "margin-bottom: 5px"),
         ),
         P("To generate a high-quality dataset from large-scale webpages, we have investigated the processing steps used by the community and made our choices based on careful manual inspection. Below is a comprehensive list of datasets we reviewed the comparison of filters we have applied."), 
         ),