Update curated.py

curated.py

@@ -484,6 +484,291 @@ freelaw_examples = Div(
     ),
 )
 
+
+def get_wiki_data(data_source: str = "Wikipedia", doc_id: int = 3, target: str = "foo"):
+    doc_id = max(0, min(int(doc_id), 9))
+
+    if data_source == "Wikipedia":
+        raw_sample_doc = extracted_sample_doc = json.load(
+            open("data/curated_samples/wiki.json")
+        )
+    else:
+        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
+
+    raw_json = raw_sample_doc[doc_id]
+    extracted_json = extracted_sample_doc[doc_id]
+    return view_data(
+        raw_json,
+        extracted_json,
+        doc_id=doc_id,
+        data_source="Wikipedia",
+        data_sources="Wikipedia",
+        target=target,
+    )
+
+wiki_examples = Div(
+    Div(
+        get_wiki_data(target=gen_random_id()),
+        style="border: 1px solid #ccc; padding: 20px;",
+    ),
+)
+
+def get_se_data(data_source: str = "StackExchange", doc_id: int = 3, target: str = "foo"):
+    doc_id = max(0, min(int(doc_id), 9))
+
+    if data_source == "StackExchange":
+        raw_sample_doc = json.load(open("data/curated_samples/stackexchange_raw.json"))
+        extracted_sample_doc = json.load(
+            open("data/curated_samples/stackexchange_extract.json")
+        )
+    else:
+        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
+
+    raw_json = raw_sample_doc[doc_id]
+    extracted_json = extracted_sample_doc[doc_id]
+    return view_data(
+        raw_json,
+        extracted_json,
+        doc_id=doc_id,
+        data_source="StackExchange",
+        data_sources="StackExchange",
+        target=target,
+    )
+
+se_examples = Div(
+    Div(
+        get_se_data(target=gen_random_id()),
+        style="border: 1px solid #ccc; padding: 20px;",
+    ),
+)
+
+def get_phil_data(data_source: str = "PhilPapers", doc_id: int = 3, target: str = "foo"):
+    doc_id = max(0, min(int(doc_id), 9))
+
+    if data_source == "PhilPapers":
+        raw_sample_doc = extracted_sample_doc = json.load(
+            open("data/curated_samples/philpapers_raw.json")
+        )
+    else:
+        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
+
+    raw_json = raw_sample_doc[doc_id]
+    extracted_json = extracted_sample_doc[doc_id]
+    return view_data(
+        raw_json,
+        extracted_json,
+        doc_id=doc_id,
+        data_source="PhilPapers",
+        data_sources="PhilPapers",
+        target=target,
+    )
+
+phil_examples = Div(
+    Div(
+        get_phil_data(target=gen_random_id()),
+        style="border: 1px solid #ccc; padding: 20px;",
+    ),
+)
+
+def get_arx_data(data_source: str = "Arxiv", doc_id: int = 3, target: str = "foo"):
+    doc_id = max(0, min(int(doc_id), 9))
+
+    if data_source == "Arxiv":
+        raw_sample_doc = json.load(open("data/curated_samples/arxiv_raw.json"))
+        extracted_sample_doc = json.load(
+            open("data/curated_samples/arxiv_extract.json")
+        )
+    else:
+        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
+
+    raw_json = raw_sample_doc[doc_id]
+    extracted_json = extracted_sample_doc[doc_id]
+    return view_data(
+        raw_json,
+        extracted_json,
+        doc_id=doc_id,
+        data_source="Arxiv",
+        data_sources="Arxiv",
+        target=target,
+    )
+
+arx_examples = Div(
+    Div(
+        get_arx_data(target=gen_random_id()),
+        style="border: 1px solid #ccc; padding: 20px;",
+    ),
+)
+
+def get_S2ORC_data(data_source: str = "S2ORC", doc_id: int = 3, target: str = "foo"):
+    doc_id = max(0, min(int(doc_id), 9))
+
+    if data_source == "S2ORC":
+        raw_sample_doc = extracted_sample_doc = json.load(
+            open("data/curated_samples/s2orc_raw.json")
+        )
+    else:
+        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
+
+    raw_json = raw_sample_doc[doc_id]
+    extracted_json = extracted_sample_doc[doc_id]
+    return view_data(
+        raw_json,
+        extracted_json,
+        doc_id=doc_id,
+        data_source="S2ORC",
+        data_sources="S2ORC",
+        target=target,
+    )
+
+s2o_examples = Div(
+    Div(
+        get_S2ORC_data(target=gen_random_id()),
+        style="border: 1px solid #ccc; padding: 20px;",
+    ),
+)
+
+def get_S2ORCA_data(data_source: str = "S2ORC Abstract", doc_id: int = 3, target: str = "foo"):
+    doc_id = max(0, min(int(doc_id), 9))
+
+    if data_source == "S2ORC":
+        raw_sample_doc = extracted_sample_doc = json.load(
+            open("data/curated_samples/s2orc_abstract_raw.json")
+        )
+    else:
+        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
+
+    raw_json = raw_sample_doc[doc_id]
+    extracted_json = extracted_sample_doc[doc_id]
+    return view_data(
+        raw_json,
+        extracted_json,
+        doc_id=doc_id,
+        data_source="S2ORC Abstract",
+        data_sources="S2ORC Abstract",
+        target=target,
+    )
+
+s2oa_examples = Div(
+    Div(
+        get_S2ORCA_data(target=gen_random_id()),
+        style="border: 1px solid #ccc; padding: 20px;",
+    ),
+)
+
+def get_pubmed_data(data_source: str = "Pubmed", doc_id: int = 3, target: str = "foo"):
+    doc_id = max(0, min(int(doc_id), 9))
+
+    if data_source == "Pubmed":
+        raw_sample_doc = json.load(open("data/curated_samples/pubmed_raw.json"))
+        extracted_sample_doc = json.load(
+            open("data/curated_samples/pubmed_extract.json")
+        )
+    else:
+        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
+
+    raw_json = raw_sample_doc[doc_id]
+    extracted_json = extracted_sample_doc[doc_id]
+    return view_data(
+        raw_json,
+        extracted_json,
+        doc_id=doc_id,
+        data_source="Pubmed",
+        data_sources="Pubmed",
+        target=target,
+    )
+
+pubmed_examples = Div(
+    Div(
+        get_pubmed_data(target=gen_random_id()),
+        style="border: 1px solid #ccc; padding: 20px;",
+    ),
+)
+
+def get_dmm_data(data_source: str = "DM Math", doc_id: int = 3, target: str = "foo"):
+    doc_id = max(0, min(int(doc_id), 9))
+
+    if data_source == "DM Math":
+        raw_sample_doc = json.load(open("data/curated_samples/dm_maths_raw.json"))
+        extracted_sample_doc = json.load(
+            open("data/curated_samples/dm_maths_extract.json")
+        )
+    else:
+        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
+
+    raw_json = raw_sample_doc[doc_id]
+    extracted_json = extracted_sample_doc[doc_id]
+    return view_data(
+        raw_json,
+        extracted_json,
+        doc_id=doc_id,
+        data_source="DM Math",
+        data_sources="DM Math",
+        target=target,
+    )
+
+dmm_examples = Div(
+    Div(
+        get_dmm_data(target=gen_random_id()),
+        style="border: 1px solid #ccc; padding: 20px;",
+    ),
+)
+
+def get_pg19_data(data_source: str = "PG19", doc_id: int = 3, target: str = "foo"):
+    doc_id = max(0, min(int(doc_id), 9))
+
+    if data_source == "PG19":
+        raw_sample_doc = extracted_sample_doc = json.load(
+            open("data/curated_samples/pg19_raw.json")
+        )
+    else:
+        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
+
+    raw_json = raw_sample_doc[doc_id]
+    extracted_json = extracted_sample_doc[doc_id]
+    return view_data(
+        raw_json,
+        extracted_json,
+        doc_id=doc_id,
+        data_source="PG19",
+        data_sources="PG19",
+        target=target,
+    )
+
+pg19_examples = Div(
+    Div(
+        get_pg19_data(target=gen_random_id()),
+        style="border: 1px solid #ccc; padding: 20px;",
+    ),
+)
+
+def get_eu_data(data_source: str = "Europarl", doc_id: int = 3, target: str = "foo"):
+    doc_id = max(0, min(int(doc_id), 9))
+
+    if data_source == "Europarl":
+        raw_sample_doc = extracted_sample_doc = json.load(
+            open("data/curated_samples/europarl_raw.json")
+        )
+    else:
+        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
+
+    raw_json = raw_sample_doc[doc_id]
+    extracted_json = extracted_sample_doc[doc_id]
+    return view_data(
+        raw_json,
+        extracted_json,
+        doc_id=doc_id,
+        data_source="Europarl",
+        data_sources="Europarl",
+        target=target,
+    )
+
+eu_examples = Div(
+    Div(
+        get_eu_data(target=gen_random_id()),
+        style="border: 1px solid #ccc; padding: 20px;",
+    ),
+)
+
 filtering_process = Div(
     Section(
           H3("This section contains the specific filtering steps taken for all 14 curated datasets."),
@@ -497,6 +782,10 @@ filtering_process = Div(
         H4("Filtering"),
         P("Manual inspection of the dataset demostrated high quality content. Only one filter was used to remove articles with few words. Based normal sentence constructs, the article was kept if it contained 10 or more words. Any article with fewer than 10 words was removed."),
         table_div_wikipedia,
+        Details(
+            Summary("Wikipedia Filtering Examples"),
+            wiki_examples_examples,
+        ),
         ), 
     ),
     Section(
@@ -514,6 +803,10 @@ filtering_process = Div(
             Li("Note: The Frequency Filter was calculated but not applied. The most frequent word in the paper consists of alpha characters only, and it appears in less than 7.5% of the document. Words are obtained by splitting the text on whitespace."),
         ),
         table_div_arx,
+        Details(
+            Summary("ArXiv Filtering Examples"),
+            arx_examples_examples,
+        ),
         ),
     ),
     Section(
@@ -552,6 +845,10 @@ filtering_process = Div(
             Li("This data was part of paper domain which are combined together and minhash was generated and deduped together with all the datasets after doing local dedup"),
         ),
         table_div_s2o,
+        Details(
+            Summary("FreeLaw Filtering Examples -- need to update"),
+            freelaw_examples,
+        ),
         ),
     ),
     Section(
@@ -584,6 +881,10 @@ filtering_process = Div(
             Li("This data was part of paper domain which are combined together and minhash was generated and deduped together with all the datasets after doing local dedup."),
         ),
         table_div_med,
+        Details(
+            Summary("PubMed Filtering Examples"),
+            pubmed_examples,
+        ),
         ),
     ),
     Section(
@@ -597,6 +898,10 @@ filtering_process = Div(
             Li("Many filters were used to clean the phil papers like double whitespaces, new lines etc. All filter details are here: https://github.com/thoppe/The-Pile-PhilPapers/blob/master/pdf_filter.py"),
         ),
         table_div_phil,
+        Details(
+            Summary("Phil Papers Filtering Examples"),
+            phil_examples,
+        ),
         ),
     ),
     Section(
@@ -608,6 +913,10 @@ filtering_process = Div(
         H4("Filtering"),
         P("EuroParl was initially filtered during the download process. Documents with fewer than 200 characters were removed. The documents also contained 'TAGS' which were removed."),
         table_div_up,
+        Details(
+            Summary("EuroParl Filtering Examples"),
+            eu_examples,
+        ),
     ),
     ),
     Section(
@@ -697,6 +1006,10 @@ filtering_process = Div(
             Li("Minimum Word Count Filter: 10"),
         ),
         table_div_se,
+        Details(
+            Summary("StackExchange Filtering Examples"),
+            se_examples,
+        ),
         ),
     ),
     Section(
@@ -724,7 +1037,7 @@ filtering_process = Div(
             Li("Minimum Word Count Filter: 10"),
             Li("Unigram Log Probability"),
         ),
-        table_div_uirc,
+        table_div_uirc,   
         ),
     ),
     Section(
@@ -745,6 +1058,10 @@ filtering_process = Div(
             Li("None"),
         ),
         table_div_dmm,
+        Details(
+            Summary("DM Math Filtering Examples"),
+            dmm_examples,
+        ),
        ),
     ),
     Section(
@@ -762,6 +1079,10 @@ filtering_process = Div(
             Li("Unigram Log Probability"),
         ),
         table_div_pg19,
+        Details(
+            Summary("PG-19 Filtering Examples"),
+            pg19_examples,
+        ),
         ),
     ),
 )
@@ -887,78 +1208,6 @@ table_div_data_pipe = Div(NotStr(table_html_data_pipe), style="margin: 40px;")
 
 
 
-
-def get_data(data_source: str = "Freelaw", doc_id: int = 3, target: str = "foo"):
-    doc_id = max(0, min(int(doc_id), 9))
-
-    if data_source == "Freelaw":
-        raw_sample_doc = json.load(open("data/curated_samples/freelaw_raw.json"))
-        extracted_sample_doc = json.load(
-            open("data/curated_samples/freelaw_extract.json")
-        )
-    elif data_source == "Wikipedia":
-        raw_sample_doc = extracted_sample_doc = json.load(
-            open("data/curated_samples/wiki.json")
-        )
-    elif data_source == "StackExchange":
-        raw_sample_doc = json.load(open("data/curated_samples/stackexchange_raw.json"))
-        extracted_sample_doc = json.load(
-            open("data/curated_samples/stackexchange_extract.json")
-        )
-    elif data_source == "PhilPapers":
-        raw_sample_doc = extracted_sample_doc = json.load(
-            open("data/curated_samples/philpapers_raw.json")
-        )
-    elif data_source == "Arxiv":
-        raw_sample_doc = json.load(open("data/curated_samples/arxiv_raw.json"))
-        extracted_sample_doc = json.load(
-            open("data/curated_samples/arxiv_extract.json")
-        )
-    elif data_source == "S2ORC":
-        raw_sample_doc = extracted_sample_doc = json.load(
-            open("data/curated_samples/s2orc_raw.json")
-        )
-    elif data_source == "S2ORC Abstract":
-        raw_sample_doc = extracted_sample_doc = json.load(
-            open("data/curated_samples/s2orc_abstract_raw.json")
-        )
-    elif data_source == "Pubmed":
-        raw_sample_doc = json.load(open("data/curated_samples/pubmed_raw.json"))
-        extracted_sample_doc = json.load(
-            open("data/curated_samples/pubmed_extract.json")
-        )
-    elif data_source == "DM Maths":
-        raw_sample_doc = json.load(open("data/curated_samples/dm_maths_raw.json"))
-        extracted_sample_doc = json.load(
-            open("data/curated_samples/dm_maths_extract.json")
-        )
-    elif data_source == "PG19":
-        raw_sample_doc = extracted_sample_doc = json.load(
-            open("data/curated_samples/pg19_raw.json")
-        )
-    elif data_source == "Europarl":
-        raw_sample_doc = extracted_sample_doc = json.load(
-            open("data/curated_samples/europarl_raw.json")
-        )
-    else:
-        raw_sample_doc = extracted_sample_doc = [{} for _ in range(10)]
-
-    raw_json = raw_sample_doc[doc_id]
-    extracted_json = extracted_sample_doc[doc_id]
-    return view_data(
-        raw_json,
-        extracted_json,
-        doc_id=doc_id,
-        data_source=data_source,
-        data_sources=data_sources,
-        target=target,
-    )
-
-
-
-
-
-
 def update(target: str, request):
     params = request.query_params
     if data_source := params.get(f"data_source_{target}"):
@@ -1082,58 +1331,6 @@ def curated(request):
     )
 
 
-    preprocessing_steps = pd.DataFrame(
-        {
-            "Step": [
-                "Language Filter",
-                "Min Word Count",
-                "Title Abstract",
-                "Majority Language",
-                "Paragraph Count",
-                "Frequency",
-                "Unigram Log Probability",
-            ],
-            "Description": [
-                "Filtering data based on language",
-                "Setting a minimum word count threshold",
-                "Extracting information from the title and abstract",
-                "Identifying the majority language in the dataset",
-                "Counting the number of paragraphs in each document",
-                "Calculating the frequency of each word in the dataset",
-                "Calculating the log probability of each unigram",
-            ],
-            "Need": [
-                "To remove documents in unwanted languages",
-                "To filter out documents with very few words",
-                "To extract relevant information for analysis",
-                "To understand the distribution of languages in the dataset",
-                "To analyze the structure and length of documents",
-                "To identify important words in the dataset",
-                "To measure the significance of individual words",
-            ],
-            "Pros": [
-                "Improves data quality by removing irrelevant documents",
-                "Filters out low-quality or incomplete documents",
-                "Provides additional information for analysis",
-                "Enables language-specific analysis and insights",
-                "Helps understand the complexity and content of documents",
-                "Identifies important terms and topics in the dataset",
-                "Quantifies the importance of individual words",
-            ],
-            "Cons": [
-                "May exclude documents in less common languages",
-                "May remove documents with valuable information",
-                "May introduce bias in the analysis",
-                "May not accurately represent the language distribution",
-                "May not capture the complexity of document structure",
-                "May be sensitive to noise and outliers",
-                "May not capture the semantic meaning of words",
-            ],
-        }
-    )
-
-    table_html = preprocessing_steps.to_html(index=False, border=0)
-    table_div = Div(NotStr(table_html), style="margin: 40px;")
     data_preprocessing_div = Div(
         H2("Data Preprocessing"),
         P("Data preprocessing is a crucial step in the data science pipeline. It involves cleaning and transforming raw data into a format that is suitable for analysis. This process includes handling missing values, normalizing data, encoding categorical variables, and more."),
@@ -1166,7 +1363,6 @@ def curated(request):
             plotly2fasthtml(diff2_stacked_bar),
             P("The figure above provides a global view of the document filtering results. ~8% of documents were removed during these three steps."),
             filtering_process,
-            freelaw_examples,
             data_preparation_div,
             #H2("Local Deduplication"), are these numbers even right?
             #local_dedup_text,