Update curated.py

curated.py

@@ -13,6 +13,47 @@ copyright_disclaimer = P("We respect the copyright of the data sources and have
 
 local_dedup_text = P("Each curated data source has been prepared using its specific rules and has been locally deduped using min-hash near deduplication. Details about the dataset are shown below in the table:")
 
+
+treemap_data = {
+  'Source': ['ArXiv', 'PubMed Central', 'PubMed Abstract', 'S2ORC Full Text', 'S2ORC Abstract', 'PhilPapers', 'Wikipedia', 'StackExchange', 'EuroParl', 'Ubuntu IRC', 'Freelaw', 'PG19', 'USPTO', 'HackerNews', 'DM Maths'],
+  'Category': ['Papers', 'Papers', 'Papers', 'Papers', 'Papers', 'Papers', 'Internet', 'Conversational', 'Legal/Formal', 'Conversational', 'Legal/Formal', 'Books', 'Legal/Formal', 'Conversational', 'Reasoning'],
+  'Count': [100, 200, 150, 120, 80, 90, 300, 250, 180, 150, 150, 250, 180, 120, 90],
+  'Details': [
+    'A repository of scientific papers in various disciplines, including computer science, physics, mathematics, and more.',
+    'A database of biomedical and life sciences research articles.',
+    'Abstracts of biomedical literature from various sources.',
+    'Full-text articles from the Semantic Scholar Open Research Corpus.',
+    'Abstracts of articles from the Semantic Scholar Open Research Corpus.',
+    'Papers from the PhilPapers database, a comprehensive index and bibliography of philosophy research.',
+    'A collaborative online encyclopedia that covers a wide range of topics.',
+    'A network of question-and-answer websites on various subjects, including programming, science, mathematics, and more.',
+    'A collection of multilingual parallel corpora of parliamentary debates from the European Parliament.',
+    'Chat logs from the Ubuntu Internet Relay Chat (IRC) channels.',
+    'Legal documents and court cases from various jurisdictions.',
+    'A collection of books from Project Gutenberg, a digital library of public domain works.',
+    'Patent documents from the United States Patent and Trademark Office.',
+    'User-generated news and discussion platform focused on technology and startups.',
+    'Deep Mind Maths dataset with generated questions.'
+  ]
+}
+# Calculate percentage for each data source
+total_count = sum(treemap_data['Count'])
+treemap_data['Percentage'] = [count / total_count * 100 for count in treemap_data['Count']]
+
+# Create treemap
+fig = px.treemap(treemap_data, path=['Category', 'Source'], values='Count', hover_data=['Details', 'Percentage'], hover_name='Source')
+
+# Set the size of the chart
+
+
+# Display treemap
+treemap_chart = fig.update_layout(width=800, height=600)
+
+
+
+
+
+
 data_pipeline_table = pd.DataFrame(
         {
             "Data Source": [
@@ -431,6 +472,7 @@ def curated(request):
             H2("Curated Sources: Overview"),
             overview_text,
             copyright_disclaimer,
+            treemap_chart,
             table_desc,
             H2("Curated Sources: Data Gathering and Filtering"),
             H3("Data Acquisition"),