Update curated.py

curated.py

@@ -680,17 +680,17 @@ filtering_process = Div(
         P(B("Download and Extraction: "), "Original PDF files download from", A("https://philarchive.org/oai.pl", href="https://philarchive.org/oai.pl"), ". All available PDF's were downloaded. Each PDF was converted to text using java", D_code("-jar ../philpapers_resources/src/pdfbox-app-2.0.21.jar ExtractText {f0} {FOUT.name}", language="java"), ". After converting to text formatting, a language was detected and added using the langdetect (citation needed) library."),
         H4("Filtering"), 
         Ul(
-            Li(P(B("Hyphenation Removal:"), D_code("end-of", language="python"), " becomes ", D_code("end of", language="python"))),
-            Li(P(B("Newline Filtering:"), D_code("This is/na sentence.", language="python"), " becomes ", D_code("This is a sentence.", language="python"))),
-            Li(P(B("Header/Footer Filtering:"), D_code("(c) 2023 Company Name.", language="python"), " is removed ",)),
-            Li(P(B("Double Whitespace Filtering:"), D_code("This  is  a  test.", language="python"), " becomes ", D_code("This is a test.", language="python"))),
-            Li(P(B("Mean Line Length Check: "), "removes paragraphs with an average line length of < 2.0")),
-            Li(P(B("CID Percentage Filter: "), "removes LaTex heavy paragraphs that contain over 10% “CID” font artifacts.")),
-            Li(P(B("Letterness Filter: "), "discards paragraphs with a low proportion of letters")),
-            Li(P(B("Removing Leading/Trailing Numbers: "), "removes numbers at the start or end of paragraphs. ", D_code("1 This is a sentence.", language="python"), " becomes ", D_code("This is a sentence.", language="python"))),
-            Li(P(B("Fixing Unicode Issues: "), "fixes Unicode issues.")),
-            Li(P(B("Combining Diacritics Correction: "), D_code("a'", language="python"), " becomes ", D_code("å", language="python"))),
-            Li(P(B("Unigram Log Probability: "), "the document must have higher than -20 average unigram log probability.")),
+            Li(P(B("Hyphenation Removal:"), D_code("end-of", language="python"), " becomes ", D_code("end of", language="python")), style = "margin-bottom: 2px"),
+            Li(P(B("Newline Filtering:"), D_code("This is/na sentence.", language="python"), " becomes ", D_code("This is a sentence.", language="python")), style = "margin-bottom: 2px"),
+            Li(P(B("Header/Footer Filtering:"), D_code("(c) 2023 Company Name.", language="python"), " is removed ",), style = "margin-bottom: 2px"),
+            Li(P(B("Double Whitespace Filtering:"), D_code("This  is  a  test.", language="python"), " becomes ", D_code("This is a test.", language="python")), style = "margin-bottom: 2px"),
+            Li(P(B("Mean Line Length Check: "), "removes paragraphs with an average line length of < 2.0"), style = "margin-bottom: 2px"),
+            Li(P(B("CID Percentage Filter: "), "removes LaTex heavy paragraphs that contain over 10% “CID” font artifacts."), style = "margin-bottom: 2px"),
+            Li(P(B("Letterness Filter: "), "discards paragraphs with a low proportion of letters"), style = "margin-bottom: 2px"),
+            Li(P(B("Removing Leading/Trailing Numbers: "), "removes numbers at the start or end of paragraphs. ", D_code("1 This is a sentence.", language="python"), " becomes ", D_code("This is a sentence.", language="python")), style = "margin-bottom: 2px"),
+            Li(P(B("Fixing Unicode Issues: "), "fixes Unicode issues."), style = "margin-bottom: 2px"),
+            Li(P(B("Combining Diacritics Correction: "), D_code("a'", language="python"), " becomes ", D_code("å", language="python")), style = "margin-bottom: 2px"),
+            Li(P(B("Unigram Log Probability: "), "the document must have higher than -20 average unigram log probability."), style = "margin-bottom: 2px"),
         ),
         table_div_phil,
         Details(
@@ -714,7 +714,39 @@ filtering_process = Div(
         P("A collection of multilingual parallel corpora of parliamentary debates from the European Parliament. This is a high-quality legacy dataset earlier used for translation tasks."),
         P(B("Download and Extraction: "), "Original dataset was downloaded from", A("http://www.statmt.org/europarl/v7/europarl.tgz", href="http://www.statmt.org/europarl/v7/europarl.tgz"),". The files were converted to jsonl lines for filtering."),
         H4("Filtering"),
-        P("EuroParl was initially filtered during the download process. Documents with fewer than 200 characters were removed. The documents also contained 'TAGS' which were removed."),
+        P("EuroParl was initially filtered during the download process. Documents with fewer than 200 characters were removed. The documents also contained HTML tags which were removed."),
+        D_code("""
+        Raw single line in data: <P> Hi I am speaker
+        After tag removal: P Hi I am speaker
+        We remove everything that starts with ["P", "BRK", "CHAPTER", "/P"]
+        and only keep tagnae == SPEAKER
+        because line starting with <SPEAKER> TEXT TEXT ....... has the relevant text
+        """, style="block", language = "python"),
+        D_code("""
+        def process_tag(original_tag):
+            tag = original_tag.strip(">").strip("<")
+        
+            # Skip empty tags
+            if not tag:
+                return None
+        
+            tagname = tag.split()[0]
+        
+            # Skip paragraph, break, and chapter tags
+            if tagname in ["P", "BRK", "CHAPTER", "/P"]:
+                return None
+        
+            # For speaker tags, return the name
+            if tagname == "SPEAKER":
+                soup = bs4.BeautifulSoup(original_tag, "html.parser")
+                name = soup.speaker["name"]
+                return name
+        
+            # Raise a error here if there is a tag we don't know
+                raise ValueError(f"Unknown tag {tag}")
+        
+        
+        """, style="block", language = "python"),
         table_div_up,
         Details(
             Summary("EuroParl Filtering Examples"),
@@ -736,12 +768,12 @@ filtering_process = Div(
         H3("HackerNews"),
         P("High-quality dialog-based dataset where user comments on the links as the head post aggregated by Y Combinator."),
         P(B("Download and Extraction: "), "The dataset was downloaded from the HackerNews repo here:", A("https://hacker-news.firebaseio.com/v0/item/", href="https://hacker-news.firebaseio.com/v0/item/"), ". The dataset was parsed using the Story ID. In this dataset each post is a story, and each reply is considered subsequent story. Story IDs were considered between ID 1 to 37500000.  The URL for all Story IDs was pinged. If that ID returned an error, the ID was removed. Each request was given a 2 second wait to account for network time."),
-        P("The HackerNews dataset contains a vast amount of stories and is known for lively discussions. Due to the number of replies a story may contain, only longest threads included stories from the 3rd level onwards. All stories included the title (1st level) and all direct replies (2nd level). Replies to the replies (3rd level) are only included for X STORIES."),
+        P("The HackerNews dataset contains a vast amount of stories and is known for lively discussions. Due to the number of replies a story may contain, only longest comment thread for each story was sampled past level 3. All stories included the title (1st level) and all direct replies (2nd level). Replies to the replies (3rd level) are only included for X STORIES."),
         H4("Filtering"),
-        Ol(
-            Li("Language Filter: English"),
-            Li("Minimum Word Count Filter: 10"),
-            Li("Unigram Log Probability"),
+        Ul(
+            Li("Language Filter: English", style = "margin-bottom: 2px"),
+            Li("Minimum Word Count Filter: 10", style = "margin-bottom: 2px"),
+            Li("Unigram Log Probability Threshold: -20", style = "margin-bottom: 2px"),
         ),
         table_div_hn,
         ),
@@ -750,12 +782,12 @@ filtering_process = Div(
         Div(
         H3("USPTO"),
         P("Patent documents from the United States Patent and Trademark Office."),
-        P(B("Download and Extraction: "), "Data was downloaded and extracted using tags from", A("https://bulkdata.uspto.gov/data/patent/grant/redbook/fulltext/", href="https://bulkdata.uspto.gov/data/patent/grant/redbook/fulltext/"),". There were three different formats that needed three different functions to download and extract the data based on year: I(Pre_2002), 2002_to_2004, and post_2004."),
+        P(B("Download and Extraction: "), "Data was downloaded and extracted using tags from", A("https://bulkdata.uspto.gov/data/patent/grant/redbook/fulltext/", href="https://bulkdata.uspto.gov/data/patent/grant/redbook/fulltext/"),". There were three different formats that needed three different functions to download and extract the data based on year:", I("Pre_2002"), ", ", I("2002_to_2004"), " and",  I("post_2004"),". We used the exact code used in The Pile (citation needed)."),
         H4("Filtering"),
         Ol(
-            Li("Language Filter: English"),
-            Li("Minimum Word Count Filter: 50"),
-            Li("Unigram Log Probability"),
+            Li("Language Filter: English", style = "margin-bottom: 2px"),
+            Li("Minimum Word Count Filter: 50", style = "margin-bottom: 2px"),
+            Li("Unigram Log Probability", style = "margin-bottom: 2px"),
         ),
         table_div_uspto,
         ),
@@ -778,9 +810,9 @@ filtering_process = Div(
         P("All content was downloaded leading to high number of documents filtered during local deduplication. Following The Pile, priorty was given to plain_text first, followed by the columns in the table in reverse order."),
         H4("Filtering"),
         Ol(
-            Li("Language Filter: English"),
-            Li("Minimum Word Count Filter: 50"),
-            Li("Unigram Log Probability"),
+            Li("Language Filter: English", style = "margin-bottom: 2px"),
+            Li("Minimum Word Count Filter: 50", style = "margin-bottom: 2px"),
+            Li("Unigram Log Probability", style = "margin-bottom: 2px"),
         ),
         H4("Local Deduplication Process"),
         Ol(
@@ -821,7 +853,7 @@ filtering_process = Div(
         """),
         H4("Filtering"),
         Ol(
-            Li("Minimum Word Count Filter: 10"),
+            Li("Minimum Word Count Filter: 10", style = "margin-bottom: 2px"),
         ),
         table_div_se,
         Details(
@@ -859,9 +891,9 @@ filtering_process = Div(
         """, block="block", language="python" ),
         H4("Filtering"),
         Ol(
-            Li("Language Filter: English"),
-            Li("Minimum Word Count Filter: 10"),
-            Li("Unigram Log Probability"),
+            Li("Language Filter: English", style = "margin-bottom: 2px"),
+            Li("Minimum Word Count Filter: 10", style = "margin-bottom: 2px"),
+            Li("Unigram Log Probability", style = "margin-bottom: 2px"),
         ),
         table_div_uirc,   
         ),
@@ -905,9 +937,9 @@ filtering_process = Div(
         P(B("Download and Extraction: "), "The dataset was downloaded directly from Huggingface:", A("https://huggingface.co/datasets/deepmind/pg19", href="https://huggingface.co/datasets/deepmind/pg19"), "."),
         H4("Filtering"),
         Ol(
-            Li("Language Filter: ???"),
-            Li("Minimum Word Count Filter: 20"),
-            Li("Unigram Log Probability"),
+            Li("Language Filter: ???", style = "margin-bottom: 2px"),
+            Li("Minimum Word Count Filter: 20", style = "margin-bottom: 2px"),
+            Li("Unigram Log Probability", style = "margin-bottom: 2px"),
         ),
         table_div_pg19,
         Details(