Update arxiv examples

curated.py

@@ -24,7 +24,10 @@ overview = (
                 "Individual Filtering Discussion for Each Source",
                 style="margin-bottom: 5px",
             ),
-            Li(B("Estimated Reading Time: 25 minutes"),style="margin-bottom: 5px", ),
+            Li(
+                B("Estimated Reading Time: 25 minutes"),
+                style="margin-bottom: 5px",
+            ),
         ),
     ),
 )
@@ -34,9 +37,10 @@ curated_sources_intro = Div(
     P(
         "While massive amount of data can be crawled and obtained from the Internet, there are certain sources contain data in additional formats (e.g. PDF documents), or organized and published as official dumps (e.g. Wikipedia). We refer to these sources as curated sources. These dataset often comprises high-quality data that contain domain-specificity, such as academic publications or domain specific discussions. TxT360 was strongly influenced by The Pile",
         D_cite(bibtex_key="thepile"),
-        " regarding both inclusion of the dataset and filtering techniques.",        
+        " regarding both inclusion of the dataset and filtering techniques.",
     ),
-    P("These sources, such as Arxiv, Wikipedia, and Stack Exchange, provide high quality data. And as mentioned above, they are excluded from the web dataset via URL matching. Details about each of the sources are provided below. ",
+    P(
+        "These sources, such as Arxiv, Wikipedia, and Stack Exchange, provide high quality data. And as mentioned above, they are excluded from the web dataset via URL matching. Details about each of the sources are provided below. ",
     ),
     P(
         "TxT360 respects the copyright of the data sources and have not included the controversial data that was used in The Pile like YouTube and Opensubtitles, Reddit threads, and book3."
@@ -566,7 +570,7 @@ se_examples = DV2(
 )
 phil_examples = DV("data/curated_samples/philpapers_raw.json", 2, "PhilPapers")
 arx_examples = DV2(
-    "data/curated_samples/arxiv_raw.json", "data/curated_samples/arxiv_extract.json", 3
+    "data/curated_samples/arxiv_raw.json", "data/curated_samples/arxiv_markdown.json", 3
 )
 s2o_examples = DV("data/curated_samples/s2orc_raw.json", 0, "S2ORC")
 s2oa_examples = DV("data/curated_samples/s2orc_abstract_raw.json", 0, "S2ORC Abstract")
@@ -859,19 +863,19 @@ filtering_process = Div(
                 ),
             ),
             table_div_s2o,
-           # Details(
-           #     Summary("S2ORC Filtering Examples -- need to update"),
-           #     Div(
-           #         P("examples are missing"),
-           #         style="background-color: white; padding: 15px; margin-top: 10px; margin-bottom: 10px; border-radius: 8px; border: none; ",  # Styling for the DV2 part
-           #     ),
-           #     style="""
-           # background-color: #FFFAEA; /* Light yellow background */
-           # padding: 15px;
-           # border-radius: 12px;
-           # margin-bottom: 15px
-           # """,
-           # ),
+            # Details(
+            #     Summary("S2ORC Filtering Examples -- need to update"),
+            #     Div(
+            #         P("examples are missing"),
+            #         style="background-color: white; padding: 15px; margin-top: 10px; margin-bottom: 10px; border-radius: 8px; border: none; ",  # Styling for the DV2 part
+            #     ),
+            #     style="""
+            # background-color: #FFFAEA; /* Light yellow background */
+            # padding: 15px;
+            # border-radius: 12px;
+            # margin-bottom: 15px
+            # """,
+            # ),
         ),
     ),
     Section(
@@ -912,19 +916,19 @@ filtering_process = Div(
                 ),
             ),
             table_div_s2oa,
-            #Details(
+            # Details(
             #    Summary("S2ORC Abstract Filtering Examples "),
-           #     Div(
-           #         P("examples are missing"),
-           #         style="background-color: white; padding: 15px; margin-top: 10px; margin-bottom: 10px; border-radius: 8px; border: none; ",  # Styling for the DV2 part
-           #     ),
-           #     style="""
-           # background-color: #FFFAEA; /* Light yellow background */
-           # padding: 15px;
-           # border-radius: 12px;
-           # margin-bottom: 15px
-           # """,
-           # ),
+            #     Div(
+            #         P("examples are missing"),
+            #         style="background-color: white; padding: 15px; margin-top: 10px; margin-bottom: 10px; border-radius: 8px; border: none; ",  # Styling for the DV2 part
+            #     ),
+            #     style="""
+            # background-color: #FFFAEA; /* Light yellow background */
+            # padding: 15px;
+            # border-radius: 12px;
+            # margin-bottom: 15px
+            # """,
+            # ),
         )
     ),
     Section(
@@ -1201,9 +1205,9 @@ filtering_process = Div(
             P(B("Unique Data Preparation Challenges: ")),
             Ul(
                 Li(
-                    "The converesation and forum style structure can be a very helpful signal for language model training. During processing the dataset, we try to encode such structure but without introducing too much noise. We choose to use an", 
+                    "The converesation and forum style structure can be a very helpful signal for language model training. During processing the dataset, we try to encode such structure but without introducing too much noise. We choose to use an",
                     D_code("<AUTHOR>", language="html"),
-                    " tag to encode the main thread text by the original poster, and use a ", 
+                    " tag to encode the main thread text by the original poster, and use a ",
                     D_code("<COMMENT>", language="html"),
                     " tag to encode the replies. We initially choose ",
                     D_code("<P>", language="html"),
@@ -1289,7 +1293,9 @@ filtering_process = Div(
                 "All content was downloaded leading to high number of documents filtered during local deduplication. Following The Pile, priority was given to plain_text first, followed by the columns in the table in reverse order."
             ),
             P(B("Unique Data Preparation Challenges: ")),
-            P("The Freelaw text uses a lot of whitespaces and newlines to format the document visually. These lines are not necessary for language model learning and sometimes have confusing semantic meanings. We attempt to unify how whitespaces appear in this dataset with the following heuristics."),
+            P(
+                "The Freelaw text uses a lot of whitespaces and newlines to format the document visually. These lines are not necessary for language model learning and sometimes have confusing semantic meanings. We attempt to unify how whitespaces appear in this dataset with the following heuristics."
+            ),
             Ul(
                 Li(
                     "Consecutive whitespaces and tabs were found. Consecutive Whitespaces and tabes were reduce to one, single whitespace.",
@@ -1309,8 +1315,9 @@ filtering_process = Div(
                 ),
                 Li(
                     "All form feed (",
-                    D_code("\\f", language="bash"), 
-                    ")characters were removed.", style="margin-bottom: -3px"
+                    D_code("\\f", language="bash"),
+                    ")characters were removed.",
+                    style="margin-bottom: -3px",
                 ),
             ),
             P(B("Filters Applied: ")),