Update curated.py

curated.py

@@ -544,7 +544,7 @@ data_preprocessing_div = Div(
     P(
         "The ",
         B("Unigram Log Probability Filter"),
-        " calculates the log probability of each unigram to measure the significance of individual words. This step quantifies the importance of individual words but maay not capture the semantic meaning of words. To calculate the average log word probability, we use word frequencies extracted from the",
+        " calculates the log probability of each unigram to measure the significance of individual words. This step quantifies the importance of individual words but maay not capture the semantic meaning of words. To calculate the average log word probability, we use word frequencies extracted from the ",
         A("1T Web-gram corpus", href="https://catalog.ldc.upenn.edu/LDC2006T13"),
         ". Specifically, we use the list available created by ",
         A(
@@ -909,7 +909,7 @@ filtering_process = Div(
             ),
             P(
                 B("Download and Extraction: "),
-                "Original PDF files download from",
+                "Original PDF files download from ",
                 A(
                     "https://philarchive.org/oai.pl",
                     href="https://philarchive.org/oai.pl",
@@ -917,7 +917,7 @@ filtering_process = Div(
                 ". All available PDF's were downloaded. Each PDF was converted to text using java",
                 D_code(
                     "-jar ../philpapers_resources/src/pdfbox-app-2.0.21.jar ExtractText {f0} {FOUT.name}",
-                    language="java",
+                    language="python",
                 ),
                 ". After converting to text formatting, a language was detected and added using the langdetect (citation needed) library.",
             ),
@@ -1034,7 +1034,7 @@ filtering_process = Div(
             ),
             P(
                 B("Download and Extraction: "),
-                "Original dataset was downloaded from",
+                "Original dataset was downloaded from ",
                 A(
                     "http://www.statmt.org/europarl/v7/europarl.tgz",
                     href="http://www.statmt.org/europarl/v7/europarl.tgz",
@@ -1098,11 +1098,11 @@ filtering_process = Div(
         Div(
             H3("HackerNews"),
             P(
-                "High-quality dialog-based dataset where user comments on the links as the head post aggregated by Y Combinator."
+                "A dialog-based dataset where user comments on the links as the head post aggregated by Y Combinator."
             ),
             P(
                 B("Download and Extraction: "),
-                "The dataset was downloaded from the HackerNews repo here:",
+                "The dataset was downloaded from the HackerNews repo here: ",
                 A(
                     "https://hacker-news.firebaseio.com/v0/item/",
                     href="https://hacker-news.firebaseio.com/v0/item/",
@@ -1110,7 +1110,7 @@ filtering_process = Div(
                 ". The dataset was parsed using the Story ID. In this dataset each post is a story, and each reply is considered subsequent story. Story IDs were considered between ID 1 to 37500000.  The URL for all Story IDs was pinged. If that ID returned an error, the ID was removed. Each request was given a 2 second wait to account for network time.",
             ),
             P(
-                "The HackerNews dataset contains a vast amount of stories and is known for lively discussions. Due to the number of replies a story may contain, only longest comment thread for each story was sampled past level 3. All stories included the title (1st level) and all direct replies (2nd level). Replies to the replies (3rd level) are only included for X STORIES."
+                "The HackerNews dataset contains a vast amount of stories and is known for lively discussions. Due to the number of replies a story may contain, only longest comment thread for each story was sampled past level 3. All stories included the title (1st level) and all direct replies (2nd level)."
             ),
             P(B("Unique Data Preperation Challenges: ")),
             Ul(
@@ -1141,7 +1141,7 @@ filtering_process = Div(
             P("Patent documents from the United States Patent and Trademark Office."),
             P(
                 B("Download and Extraction: "),
-                "Data was downloaded and extracted using tags from",
+                "Data was downloaded and extracted using tags from ",
                 A(
                     "https://bulkdata.uspto.gov/data/patent/grant/redbook/fulltext/",
                     href="https://bulkdata.uspto.gov/data/patent/grant/redbook/fulltext/",
@@ -1171,7 +1171,7 @@ filtering_process = Div(
             ),
             P(
                 B("Download and Extraction"),
-                "The dataset was downloaded from:",
+                "The dataset was downloaded from: ",
                 A(
                     "https://storage.courtlistener.com/bulk-data/",
                     href="https://storage.courtlistener.com/bulk-data/",
@@ -1185,7 +1185,7 @@ filtering_process = Div(
         ("html_with_citations", html2text), ("xml_harvard", html2text), 
         plain_text
         """,
-                language="SQL",
+                language="python",
             ),
             P(
                 "All content was downloaded leading to high number of documents filtered during local deduplication. Following The Pile, priorty was given to plain_text first, followed by the columns in the table in reverse order."
@@ -1247,7 +1247,8 @@ filtering_process = Div(
                 A("math.stackexchange.com", href="math.stackexchange.com"),
                 ". Raw data was extracted an XML format and only two files Posts.xml and Comments.xml were considered. To match the StackExchange hierarchy, each file was parsed using post_id to connect questions to answers and then to comments. We will include the full list of sub URLs in when the code is released.",
             ),
-            P("""
+            D_code(
+                """ 
         1. Questions:
   2. Comment1:
   3. Comment2:
@@ -1256,8 +1257,10 @@ filtering_process = Div(
   6. Comment2:
   7. Answer2:
   8. Comment1:
-  9. Comment2:
-        """),
+  9. Comment2:""",
+                block="block",
+                language="python",
+            ),
             P(B("Unique Data Preperation Challenges: ")),
             Ul(
                 Li(
@@ -1301,7 +1304,7 @@ filtering_process = Div(
             ),
             P(
                 B("Download and Extraction: "),
-                "The dataset was downloaded from:",
+                "The dataset was downloaded from: ",
                 A(
                     "https://irclogs.ubuntu.com/{date.year}/{date.month:02d}/{date.day:02d}/",
                     href="https://irclogs.ubuntu.com/{date.year}/{date.month:02d}/{date.day:02d}/",
@@ -1349,7 +1352,7 @@ filtering_process = Div(
             ),
             P(
                 B("Download and Extraction: "),
-                "The dataset was downloaded rirectly downloaded from the Huggingface repo:",
+                "The dataset was downloaded rirectly downloaded from the Huggingface repo: ",
                 A(
                     "https://huggingface.co/datasets/deepmind/math_dataset",
                     href="https://huggingface.co/datasets/deepmind/math_dataset",
@@ -1359,7 +1362,7 @@ filtering_process = Div(
             D_code(
                 """ 
         Question: TEXT
-        Answer: TEXT""",
+    Answer: TEXT""",
                 block="block",
                 language="python",
             ),