topic-analysis (#1)

- Add topic analysis (d40cd38cb5959666f569699c1388f60dd0831bd5)
- Fix issues in topic analysis (cb27b88e7b216ab08d98928c4f7bc92313521ae9)
- Add comments to each topic graph (3dd0859f7cebc297557e5d327a5f6af82dea16f5)
- Merge branch 'main' of https://huggingface.co/spaces/LLM360/TxT360-New into pr/1 (9fc9d4a3ec7a1cc20176b2c99faaaa3330a40a60)
- Merge branch 'main' of https://huggingface.co/spaces/LLM360/TxT360-New into pr/1 (30fd7fc71eb78a6dec2c9d686f0db41c42e4941a)

main.py

@@ -352,6 +352,12 @@ def main():
                                     href="#section53",
                                 )
                             ),
+                            Li(
+                                A(
+                                    "Topic Analysis",
+                                    href="#section55",
+                                )
+                            )
                         ),
                     ),
                     role="navigation",

results.py

@@ -830,6 +830,7 @@ intro_div = Div(
     Ul(
         Li("The Learning Curve of TxT360 with an Upsampling Recipe", style = "margin-bottom: 5px"),
         Li("Perplexity Analysis across time", style = "margin-bottom: 5px"),
+        Li("Topic Analysis on Data Cluster Groups", style = "margin-bottom: 5px"),
         Li(B("Estimated Reading Time: 15 minutes"), style = "margin-bottom: 5px"),
     ),
 )
@@ -965,6 +966,51 @@ llama_div = Div(
         ),
 )
 
+with open(os.path.join(os.path.dirname(__file__), "data", "topic_charts.json"), 'r') as f:
+    topic_charts = json.load(f)
+topic_graphs = []
+
+for title, data in topic_charts:
+    if data["type"] == "barh":
+        topic_graphs.append(go.Figure(go.Bar(
+            x=data["kwargs"]["width"],
+            y=data["kwargs"]['y'],
+            orientation='h',
+            marker_color=[
+                "rgb(" + ", ".join(str(val * 255) for val in rgb) + ')'
+                for rgb in data["kwargs"]["color"]
+            ]
+        )))
+    elif data["type"] == "pie":
+        topic_graphs.append(go.Figure(go.Pie(
+            values=data["kwargs"]['x'],
+            labels=data["kwargs"]["labels"],
+            marker_colors=[
+                "rgb(" + ", ".join(str(val * 255) for val in rgb) + ')'
+                for rgb in data["kwargs"]["colors"]
+            ]
+        )))
+
+cluster_div = Div(
+    Section(
+        H2("Topic Analysis"),
+        P("We tried to classify data into topic groups and looked for correlations between topics and statistics of data. Data from different topic groups should manifest different characteristics of distribution, which can give us some insight into the composition of dataset."),
+        H3("Methodology"),
+        P("We took the ", A("common crawl", href="https://commoncrawl.org/"), " data and clustered them into 17 topic groups using ", A("BERTopic", href="https://maartengr.github.io/BERTopic/index.html"), ". We collected and aggregated a series of metrics which include quality signals and other useful metadata. For each topic group, we calculated average scores and generated the corresponding bar charts over different metrics for comparison and analysis."),
+        H3("Cluster Groups"),
+        P("We grouped data into the following 17 clusters"),
+        Ul(*(
+            Li(topic_name, style = "margin-bottom: 5px")
+            for topic_name in ("Arts", "Business & Economics & Finance", "Culture & Cultural geography", "Daily Life & Home & Lifestyle", "Education", "Entertainment & Travel & Hobby", "Environment", "Food & Drink & Cooking", "Health & Wellness & Medicine", "Law & Justice", "Natural Science & Formal Science & Technology", "Personal Development & Human Resources & Career", "Politics & Government", "Religion & Spirituality", "Shopping & Commodity", "Society & Social Issues & Human Rights", "Sports")
+        )),
+        H3("Results Analysis"),
+        *(
+            Section(H4(title), plotly2fasthtml(topic_graphs[i]), P(data.get("comment", '')))
+            for i, (title, data) in enumerate(topic_charts)
+        )
+    )
+)
+
 
 def results():
     return Div(
@@ -986,6 +1032,10 @@ def results():
                 ),
                 Section(
                     llama_div,
+                ),
+                Section(
+                    cluster_div,
+                    id="section55"
                 ),
                     id="inner-text"
                 )