Update curated.py

curated.py

@@ -462,7 +462,7 @@ data_preprocessing_div = Div(
     P("The ", B("Unigram Log Probability Filter")," calculates the log probability of each unigram to measure the significance of individual words. This step quantifies the importance of individual words but maay not capture the semantic meaning of words."),
     H3("Data Processing for S2ORC"),
     P("The formating of the S2ORC dataset required special filters to be applied. These filters were not applied to the other data sources."),
-    P("The ", B("Title Abstract Filter")," extracts information from the title and abstract. This step provides additional information for analysis but may introduce bias in the analysis."),
+    P("The ", B("Title and Abstract Filter")," extracts information from the title and abstract. This step provides additional information for analysis but may introduce bias in the analysis."),
     P("The ", B("Majority Language Filter")," identifies the majority language in the dataset. This step displays the distribution of languages in the dataset to enable language-specific analysis and insights."),
     P("The ", B("Paragraph Count Filter")," counts the number of paragraphs in each document. This step helps to analyze the structure and length of documents which can be a useful hueristic for document complexity."),
     P("The ",B("Frequency Filter")," calculates the frequency of each word in the dataset. This step serves to identify important words and topics in the dataset but may be sensitive to noise and outliers."),
@@ -559,7 +559,7 @@ filtering_process = Div(
             Li("Language Filter: any language other than English are discarded"),
             Li("Minimum Word Count Filter: less than 500 words (not inclusive) are discarded"),
             Li("Unigram Log Probablity Filter: Documents were kept if they their average unigram log probability was higher than -20. To calculate the average log word probability, we use word frequencies extracted from the", A("1T Web-gram corpus", href= "https://catalog.ldc.upenn.edu/LDC2006T13"),". Specifically, we use the list available created by ", A("Rachel Tatman", href="https://www.kaggle.com/datasets/rtatman/english-word-frequency"),"."),
-            Li("Note: The Frequency Filter was calculated but not applied. The most frequent word in the paper consists of alpha characters only, and it appears in less than 7.5% of the document. Words are obtained by splitting the text on whitespace."),
+            Li("Note: the Frequency Filter was calculated but not applied. The most frequent word in the paper consists of alpha characters only, and it appears in less than 7.5% of the document. Words are obtained by splitting the text on whitespace."),
         ),
         table_div_arx,
         Details(
@@ -579,33 +579,24 @@ filtering_process = Div(
     ),
     Section(
         Div(
-        H3("S2ORC - NEED TO MAKE S2ORC ABSTRACT AND UPDATE THIS FILTERING SECTION"),
+        H3("S2ORC"),
         P("The Semantic Scholar Open Research Corpus (S2ORC) is a comprehensive dataset designed for natural language processing (NLP) and text-mining research over scientific papers. It includes rich metadata, and abstract and full-text content for millions of academic papers across various disciplines. This dataset is further divided into two components, S2ORC abstract and S2ORC full text."),
         H4("Download and Extraction"),
         Ol(
-            Li("This was downloaded directly in zip format using S2ORC api key and normal get request. code: response = urllib.request.urlopen(url)"),
-            Li("There were two kind of datasets that was downloaded S2ORC and S2ORC abstract"),
+            Li("This was downloaded directly in zip format using S2ORC api key and a get() request: ", D_code("response = urllib.request.urlopen(url)", language = "python")),
         ),
-        H4("Filtering - S2ORC"),
+        H4("Filtering"),
         P("1. Multiple filters are used here after manually verifying output of all the filters as suggested by peS2o dataset"),
         Ol(
-            Li("title_abstract: must have title and abstract"),
-            Li("The paper must be in English. To determine the language of each document, we use the pycld3 library. We run pycld3 on the first 2000 characters of each paragraph in the paper. The language of the paper is the most common language of the paragraphs."),
-            Li("word_count: less than 500 words (not inclusive) are discarded"),
-            Li("paragraph_count: The paper must have at least 5 paragraphs after removing paragraphs with less than -20 average log world probability"),
-            Li("frequency: The most frequent word in the paper consists of alpha characters only, and it appears in less than 7.5% of the document. Words are obtained by splitting the text on whitespace."),
-        ),
-        H4("Local Deduplication Process"),
-        Ol(
-            Li("Local dedup was done with all papers combined."),
-        ),
-        H4("Global Deduplication Process"),
-        Ol(
-            Li("This data was part of paper domain which are combined together and minhash was generated and deduped together with all the datasets after doing local dedup"),
+            Li("Title and Abstract Filter: must have title and abstract"),
+            Li("Language Filter: The paper must be in English. To determine the language of each document, we use the pycld3 library. We run pycld3 on the first 2000 characters of each paragraph in the paper. The language of the paper is the most common language of the paragraphs."),
+            Li("Word Count Filter: less than 500 words (not inclusive) are discarded"),
+            Li("Paragraph Count Filter: The paper must have at least 5 paragraphs after removing paragraphs with less than -20 average log world probability"),
+            Li("Frequency Filter: The most frequent word in the paper consists of alpha characters only, and it appears in less than 7.5% of the document. Words are obtained by splitting the text on whitespace."),
         ),
         table_div_s2o,
         Details(
-            Summary("FreeLaw Filtering Examples -- need to update"),
+            Summary("S2ORC Filtering Examples -- need to update"),
             Div(
                 P("examples are missing"),
                 style="background-color: white; padding: 15px; margin-top: 10px; margin-bottom: 10px; border-radius: 8px; border: none; "  # Styling for the DV2 part
@@ -621,21 +612,33 @@ filtering_process = Div(
     ),
     Section(
         Div(
-        H3("S2ORC ABSTRACT"),
+        H3("S2ORC Abstract"),
         P("The Semantic Scholar Open Research Corpus (S2ORC) is a comprehensive dataset designed for natural language processing (NLP) and text-mining research over scientific papers. It includes rich metadata, and abstract and full-text content for millions of academic papers across various disciplines. This dataset is further divided into two components, S2ORC abstract and S2ORC full text."),
         H4("Download and Extraction"),
         Ol(
-            Li("This was downloaded directly in zip format using S2ORC api key and normal get request. code: response = urllib.request.urlopen(url)"),
-            Li("There were two kind of datasets that was downloaded S2ORC and S2ORC abstract"),
+            Li("This was downloaded directly in zip format using S2ORC api key and a get() request: ", D_code("response = urllib.request.urlopen(url)", language = "python")),
         ),    
-        H4("Filtering - S2ORC Abstract"),
-        P("1. Multiple filters are used here after manually verifying output of all the filters as suggested by peS2o dataset. The frequency filter was not used as suggested by peS2o because it was removing good samples as inspected manually"),
+        H4("Filtering"),
+        P("Multiple filters are used here after manually verifying output of all the filters as suggested by peS2o dataset. The frequency filter was not used as suggested by peS2o because it was removing good samples as inspected manually"),
         Ol(
-            Li("title_abstract: must have title and abstract"),
-            Li("language: abstract must be in English"),
-            Li("word_count: less than 20 (not inclusive) are discarded"),
-            Li("Unigram log probablity"),
-            Li("frequency: The most frequent word in the paper consists of alpha characters only, and it appears in less than 7.5% of the document. Words are obtained by splitting the text on whitespace."),
+            Li("Title and Abstract Filter: must have title and abstract"),
+            Li("Majority Language Filter: abstract must be in English"),
+            Li("Minimum Word Count Filter: less than 20 (not inclusive) are discarded"),
+            Li("Unigram Log Probability Threshold: -20"),
+            Li("Note: Frequency Filter: The most frequent word in the paper consists of alpha characters only, and it appears in less than 7.5% of the document. Words are obtained by splitting the text on whitespace."),
+        ),
+        Details(
+            Summary("S2ORC Abstract Filtering Examples "),
+            Div(
+                P("examples are missing"),
+                style="background-color: white; padding: 15px; margin-top: 10px; margin-bottom: 10px; border-radius: 8px; border: none; "  # Styling for the DV2 part
+            ),
+            style="""
+            background-color: #FFFAEA; /* Light yellow background */
+            padding: 15px;
+            border-radius: 12px;
+            margin-bottom: 15px
+            """, 
         ),
         )
     ),
@@ -643,32 +646,15 @@ filtering_process = Div(
     
     Section(
         Div(
-        H3("PubMed - need to update with abstract vs central"),
-        P(""),
-        H4("Download and Extraction"),
-        Ol(
-            Li("First all the urls of PMC and PMA files are parsed and stored as text file from FTP server https://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/"),
-            Li("All the urls are downloaded and the downloaded data is in xml.tar format"),
-            Li("For pubmed central First tar files are opened using tarfile library and then converted to markdown format using pandoc: pandoc -f jats {nxml} -o {pmcid}.md --wrap=none"),
-            Li("All the markdown files are combined to create jsonl files. In jsonl files, 1 line correspond to 1 markdown file."),
-            Li("For pubmed abstract, the XML files are in very simple format and beautiful soup is directly used to extract the abstract, title and pmid and stored in jsonl format"),
-        ),
+        H3("PubMed Central and PubMed Abstract"),
+            P(B("Download and Extraction: "), "All files were downloaded from", A("ttps://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/",href="ttps://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/"),". PubMed Central (PMC) files are downloaded in an xml.tar format. The tar files are opened and converted to markdown format using pandoc", D_code("pandoc -f jats {nxml} -o {pmcid}.md", language="bash"),". The markdown files are combined to create jsonl files. PubMed Abstract (PMA) files  were downloaded in xml. The BeautifulSoup library was used to extract the abstract, title, and PMID. All files were stored in jsonl format.")
         H4("Filtering"),
         P("1. Multiple filters are used here after manually verifying output of all the filters as suggested by peS2o dataset."),
         Ol(
-            Li("min_word: less than 100 words (not inclusive) are discarded, less than 20 words for pubmed abstract"),
-            Li("Language: any language other than English are discarded"),
-            Li("Frequency: The most frequent word in the paper consists of alpha characters only, and it appears in less than 7.5% of the document. Words are obtained by splitting the text on whitespace. This filter is not used for pubmed abstract"),
-            Li("Unigram log probablity: Must have higher than -20 average unigram log probability. To calculate the average log word probability, we use word frequencies extracted from the 1T Web Ngram corpus; specifically, we use the list available created by Rachel Tatman. A copy is hosted here."),
-            Li("need to add the hyperlinks for the section above"),
-        ),
-        H4("Local Deduplication Process"),
-        Ol(
-            Li("Local dedup was done with all papers combined."),
-        ),
-        H4("Global Deduplication Process"),
-        Ol(
-            Li("This data was part of paper domain which are combined together and minhash was generated and deduped together with all the datasets after doing local dedup."),
+            Li("Minimum Word Count Filter: less than 100 words (not inclusive) are discarded, less than 20 words for pubmed abstract"),
+            Li("Language Filter: any language other than English are discarded"),
+            Li("Frequency Filter: The most frequent word in the paper consists of alpha characters only, and it appears in less than 7.5% of the document. Words are obtained by splitting the text on whitespace. This filter is not used for pubmed abstract"),
+            Li("Unigram Log Probability Threshold: -20"),
         ),
         table_div_med,
         Details(