Update app.py

app.py

@@ -206,6 +206,25 @@ def get_feature_from_index(subject, index):
     feature = next((f for f in subject_data[subject]['feature_analysis'] if f['index'] == index), None)
     return feature
 
+# def visualize_feature(subject, index):
+#     feature = next((f for f in subject_data[subject]['feature_analysis'] if f['index'] == index), None)
+#     if feature is None:
+#         return "Invalid feature index", None, None, None, None, None, None
+
+#     output = f"# {feature['label']}\n\n"
+#     output += f"* Pearson correlation: {feature['pearson_correlation']:.4f}\n\n"
+#     output += f"* Density: {feature['density']:.4f}\n\n"
+
+#     # Top m abstracts
+#     top_m_abstracts = get_feature_activations(subject, index)
+    
+#     # Create dataframe for top abstracts
+#     df_data = [
+#         {"Title": m[1].split('\n\n')[0], "Activation value": f"{m[2]:.4f}"}
+#         for m in top_m_abstracts
+#     ]
+#     df_top_abstracts = pd.DataFrame(df_data)
+
 def visualize_feature(subject, index):
     feature = next((f for f in subject_data[subject]['feature_analysis'] if f['index'] == index), None)
     if feature is None:
@@ -218,12 +237,30 @@ def visualize_feature(subject, index):
     # Top m abstracts
     top_m_abstracts = get_feature_activations(subject, index)
     
-    # Create dataframe for top abstracts
-    df_data = [
-        {"Title": m[1].split('\n\n')[0], "Activation value": f"{m[2]:.4f}"}
-        for m in top_m_abstracts
-    ]
+    # Create dataframe for top abstracts with clickable links
+    df_data = []
+    for doc_id, abstract, activation_value in top_m_abstracts:
+        title = abstract.split('\n\n')[0]
+        title = title.replace('[', '').replace(']', '')
+        title = title.replace("'", "")
+        title = title.replace('"', '')
+        url_id = doc_id.replace('_arXiv.txt', '')
+        if 'astro-ph' in url_id:
+            url_id = url_id.split('astro-ph')[1]
+            url = f"https://arxiv.org/abs/astro-ph/{url_id}"
+        else:
+            if '.' in doc_id:
+                url = f"https://arxiv.org/abs/{url_id}"
+            else:
+                url = f"https://arxiv.org/abs/hep-ph/{url_id}"
+        
+        linked_title = f"[{title}]({url})"
+        df_data.append({"Title": linked_title, "Activation value": activation_value})
+    
     df_top_abstracts = pd.DataFrame(df_data)
+    styled_top_abstracts = df_top_abstracts.style.format({
+        "Activation value": "{:.4f}"
+    })
 
     # Activation value distribution
     topk_indices = subject_data[subject]['topk_indices']
@@ -249,23 +286,62 @@ def visualize_feature(subject, index):
     topk_indices_cosine = np.argsort(-cosine_similarities)[:topk]
     topk_values_cosine = cosine_similarities[topk_indices_cosine]
 
-    # Create dataframe for top 5 correlated features
-    df_top_correlated = pd.DataFrame({
-        "Feature": [get_feature_from_index(subject, i)['label'] for i in topk_indices_cosine],
-        "Cosine similarity": [f"{v:.4f}" for v in topk_values_cosine]
-    })
-    df_top_correlated_styled = style_dataframe(df_top_correlated, is_top=True)
+    # # Create dataframe for top 5 correlated features
+    # df_top_correlated = pd.DataFrame({
+    #     "Feature": [get_feature_from_index(subject, i)['label'] for i in topk_indices_cosine],
+    #     "Cosine similarity": topk_values_cosine
+    # })
+    # df_top_correlated_styled = df_top_correlated.style.format({
+    #     "Cosine similarity": "{:.4f}"
+    # })
 
     bottomk = 5
     bottomk_indices_cosine = np.argsort(cosine_similarities)[:bottomk]
     bottomk_values_cosine = cosine_similarities[bottomk_indices_cosine]
+    
+    # # Create dataframe for bottom 5 correlated features
+    # df_bottom_correlated = pd.DataFrame({
+    #     "Feature": [get_feature_from_index(subject, i)['label'] for i in bottomk_indices_cosine],
+    #     "Cosine similarity": bottomk_values_cosine
+    # })
+    # df_bottom_correlated_styled = df_bottom_correlated.style.format({
+    #     "Cosine similarity": "{:.4f}"
+    # })
+
+    # # Co-occurrences
+    # co_occurrences = calculate_co_occurrences(subject, index)
+    # topk = 5
+    # topk_indices_co_occurrence = np.argsort(-co_occurrences)[:topk]
+    # topk_values_co_occurrence = co_occurrences[topk_indices_co_occurrence]
+
+    # # Create dataframe for top 5 co-occurring features
+    # df_co_occurrences = pd.DataFrame({
+    #     "Feature": [get_feature_from_index(subject, i)['label'] for i in topk_indices_co_occurrence],
+    #     "Co-occurrences": topk_values_co_occurrence
+    # })
+    # df_co_occurrences_styled = df_co_occurrences.style.format({
+    #     "Co-occurrences": "{:.4f}"
+    # })
+
+    # return output, styled_top_abstracts, df_top_correlated_styled, df_bottom_correlated_styled, df_co_occurrences_styled, fig2
+
+     # Create dataframe for top 5 correlated features
+    df_top_correlated = pd.DataFrame({
+        "Feature": [get_feature_from_index(subject, i)['label'] for i in topk_indices_cosine],
+        "Cosine similarity": topk_values_cosine
+    })
+    df_top_correlated_styled = df_top_correlated.style.format({
+        "Cosine similarity": "{:.4f}"
+    })
 
     # Create dataframe for bottom 5 correlated features
     df_bottom_correlated = pd.DataFrame({
         "Feature": [get_feature_from_index(subject, i)['label'] for i in bottomk_indices_cosine],
-        "Cosine similarity": [f"{v:.4f}" for v in bottomk_values_cosine]
+        "Cosine similarity": bottomk_values_cosine
+    })
+    df_bottom_correlated_styled = df_bottom_correlated.style.format({
+        "Cosine similarity": "{:.4f}"
     })
-    df_bottom_correlated_styled = style_dataframe(df_bottom_correlated, is_top=False)
 
     # Co-occurrences
     co_occurrences = calculate_co_occurrences(subject, index)
@@ -278,8 +354,102 @@ def visualize_feature(subject, index):
         "Feature": [get_feature_from_index(subject, i)['label'] for i in topk_indices_co_occurrence],
         "Co-occurrences": topk_values_co_occurrence
     })
+    df_co_occurrences_styled = df_co_occurrences.style.format({
+        "Co-occurrences": "{:.0f}"  # Keep as integer
+    })
 
-    return output, df_top_abstracts, df_top_correlated_styled, df_bottom_correlated_styled, df_co_occurrences, fig2
+    return output, styled_top_abstracts, df_top_correlated_styled, df_bottom_correlated_styled, df_co_occurrences_styled, fig2
+
+# def visualize_feature(subject, index):
+#     feature = next((f for f in subject_data[subject]['feature_analysis'] if f['index'] == index), None)
+#     if feature is None:
+#         return "Invalid feature index", None, None, None, None, None, None
+
+#     output = f"# {feature['label']}\n\n"
+#     output += f"* Pearson correlation: {feature['pearson_correlation']:.4f}\n\n"
+#     output += f"* Density: {feature['density']:.4f}\n\n"
+
+#     # Top m abstracts
+#     top_m_abstracts = get_feature_activations(subject, index)
+    
+#     # Create dataframe for top abstracts with clickable links
+#     df_data = []
+#     for doc_id, abstract, activation_value in top_m_abstracts:
+#         title = abstract.split('\n\n')[0]
+#         title = title.replace('[', '').replace(']', '')
+#         title = title.replace("'", "")
+#         title = title.replace('"', '')
+#         url_id = doc_id.replace('_arXiv.txt', '')
+#         if 'astro-ph' in url_id:
+#             url_id = url_id.split('astro-ph')[1]
+#             url = f"https://arxiv.org/abs/astro-ph/{url_id}"
+#         else:
+#             if '.' in doc_id:
+#                 url = f"https://arxiv.org/abs/{url_id}"
+#             else:
+#                 url = f"https://arxiv.org/abs/hep-ph/{url_id}"
+        
+#         linked_title = f"[{title}]({url})"
+#         df_data.append({"Title": linked_title, "Activation value": activation_value})
+    
+#     df_top_abstracts = pd.DataFrame(df_data)
+
+#     # Activation value distribution
+#     topk_indices = subject_data[subject]['topk_indices']
+#     topk_values = subject_data[subject]['topk_values']
+
+#     activation_values = np.where(topk_indices == index, topk_values, 0).max(axis=1)
+#     fig2 = px.histogram(x=activation_values, nbins=50)
+#     fig2.update_layout(
+#         #title=f'{feature["label"]}',
+#         xaxis_title='Activation value',
+#         yaxis_title=None,
+#         yaxis_type='log',
+#         height=220,
+#     )
+
+#     # Correlated features
+#     decoder = subject_data[subject]['decoder']
+#     feature_vector = decoder[:, index]
+#     decoder_without_feature = np.delete(decoder, index, axis=1)
+#     cosine_similarities = np.dot(feature_vector, decoder_without_feature) / (np.linalg.norm(decoder_without_feature, axis=0) * np.linalg.norm(feature_vector))
+
+#     topk = 5
+#     topk_indices_cosine = np.argsort(-cosine_similarities)[:topk]
+#     topk_values_cosine = cosine_similarities[topk_indices_cosine]
+
+#     # Create dataframe for top 5 correlated features
+#     df_top_correlated = pd.DataFrame({
+#         "Feature": [get_feature_from_index(subject, i)['label'] for i in topk_indices_cosine],
+#         "Cosine similarity": [f"{v:.4f}" for v in topk_values_cosine]
+#     })
+#     df_top_correlated_styled = style_dataframe(df_top_correlated, is_top=True)
+
+#     bottomk = 5
+#     bottomk_indices_cosine = np.argsort(cosine_similarities)[:bottomk]
+#     bottomk_values_cosine = cosine_similarities[bottomk_indices_cosine]
+
+#     # Create dataframe for bottom 5 correlated features
+#     df_bottom_correlated = pd.DataFrame({
+#         "Feature": [get_feature_from_index(subject, i)['label'] for i in bottomk_indices_cosine],
+#         "Cosine similarity": [f"{v:.4f}" for v in bottomk_values_cosine]
+#     })
+#     df_bottom_correlated_styled = style_dataframe(df_bottom_correlated, is_top=False)
+
+#     # Co-occurrences
+#     co_occurrences = calculate_co_occurrences(subject, index)
+#     topk = 5
+#     topk_indices_co_occurrence = np.argsort(-co_occurrences)[:topk]
+#     topk_values_co_occurrence = co_occurrences[topk_indices_co_occurrence]
+
+#     # Create dataframe for top 5 co-occurring features
+#     df_co_occurrences = pd.DataFrame({
+#         "Feature": [get_feature_from_index(subject, i)['label'] for i in topk_indices_co_occurrence],
+#         "Co-occurrences": topk_values_co_occurrence
+#     })
+
+#     #return output, df_top_abstracts, df_top_correlated_styled, df_bottom_correlated_styled, df_co_occurrences, fig2
+#     return output, df_top_abstracts, df_top_correlated_styled, df_bottom_correlated_styled, df_co_occurrences, fig2
 
 # Modify the main interface function
 def create_interface():
@@ -334,11 +504,16 @@ def create_interface():
                 manually_added_features_state = gr.State([])
 
                 def update_search_results(feature_values, feature_indices, manually_added_features, current_subject):
+                    # ae = subject_data[current_subject]['ae']
+                    # abstract_embeddings = subject_data[current_subject]['abstract_embeddings']
+                    # abstract_texts = subject_data[current_subject]['abstract_texts']
+                    # df_metadata = subject_data[current_subject]['df_metadata']
                     ae = subject_data[current_subject]['ae']
                     abstract_embeddings = subject_data[current_subject]['abstract_embeddings']
                     abstract_texts = subject_data[current_subject]['abstract_texts']
                     df_metadata = subject_data[current_subject]['df_metadata']
 
+
                     # Combine manually added features with query-generated features
                     all_indices = []
                     all_values = []
@@ -368,12 +543,40 @@ def create_interface():
                     doc_ids = abstract_texts['doc_ids']
                     topk_doc_ids = [doc_ids[i] for i in topk_indices_search]
 
+                    # # Prepare search results
+                    # search_results = []
+                    # for doc_id in topk_doc_ids:
+                    #     metadata = df_metadata[df_metadata['arxiv_id'] == doc_id].iloc[0]
+                    #     title = metadata['title'].replace('[', '').replace(']', '')
+                    #     # Remove single quotes from title
+                    #     title = title.replace("'", "")
+
+                    #     url_id = doc_id.replace('_arXiv.txt', '')
+                    #     if 'astro-ph' in url_id:
+                    #         url_id = url_id.split('astro-ph')[1]
+                    #         url = f"https://arxiv.org/abs/astro-ph/{url_id}"
+                    #     else:
+                    #         # Create the clickable link based on the doc_id
+                    #         if '.' in doc_id:
+                    #             url = f"https://arxiv.org/abs/{doc_id.replace('_arXiv.txt', '')}"
+                    #         else:
+                    #             url = f"https://arxiv.org/abs/hep-ph/{doc_id.replace('_arXiv.txt', '')}"
+                        
+                    #     linked_title = f"[{title}]({url})"
+                        
+                    #     search_results.append([
+                    #         linked_title,
+                    #         int(metadata['citation_count']),
+                    #         int(metadata['year'])
+                    #     ])
+
+                    # return search_results, all_values, all_indices
+
                     # Prepare search results
                     search_results = []
                     for doc_id in topk_doc_ids:
                         metadata = df_metadata[df_metadata['arxiv_id'] == doc_id].iloc[0]
                         title = metadata['title'].replace('[', '').replace(']', '')
-                        # Remove single quotes from title
                         title = title.replace("'", "")
 
                         url_id = doc_id.replace('_arXiv.txt', '')
@@ -381,7 +584,6 @@ def create_interface():
                             url_id = url_id.split('astro-ph')[1]
                             url = f"https://arxiv.org/abs/astro-ph/{url_id}"
                         else:
-                            # Create the clickable link based on the doc_id
                             if '.' in doc_id:
                                 url = f"https://arxiv.org/abs/{doc_id.replace('_arXiv.txt', '')}"
                             else:
@@ -395,7 +597,14 @@ def create_interface():
                             int(metadata['year'])
                         ])
 
-                    return search_results, all_values, all_indices
+                    # Convert search_results to a DataFrame and apply styling
+                    df_search_results = pd.DataFrame(search_results, columns=["Title", "Citation Count", "Year"])
+                    styled_search_results = df_search_results.style.format({
+                        "Citation Count": "{:.0f}",  # Keep as integer
+                        "Year": "{:.0f}"  # Keep as integer
+                    })
+
+                    return styled_search_results, all_values, all_indices
 
                 @gr.render(inputs=[input_text, search_results_state, feature_values_state, feature_indices_state, manually_added_features_state, subject])
                 def show_components(text, search_results, feature_values, feature_indices, manually_added_features, current_subject):
@@ -418,11 +627,6 @@ def create_interface():
 
                     with gr.Row():
                         with gr.Column(scale=2):
-                            # df = gr.Dataframe(
-                            #     headers=["Title", "Citation Count", "Year"],
-                            #     value=search_results,
-                            #     label="Top 10 Search Results"
-                            # )
                             df = gr.Dataframe(
                                 headers=["Title", "Citation Count", "Year"],
                                 value=search_results,
@@ -462,18 +666,7 @@ def create_interface():
                         with gr.Column(scale=1):
                             update_button = gr.Button("Update Results")
                             sliders = []
-                            # for i, (value, index) in enumerate(zip(feature_values, feature_indices)):
-                            #     feature = next((f for f in subject_data[current_subject]['feature_analysis'] if f['index'] == index), None)
-                            #     label = f"{feature['label']} ({index})" if feature else f"Feature {index}"
-                                
-                            #     # Add prefix and change color for manually added features
-                            #     if index in manually_added_features:
-                            #         label = f"[Custom] {label}"
-                            #         slider = gr.Slider(minimum=0, maximum=1, step=0.01, value=value, label=label, key=f"slider-{index}", elem_id=f"custom-slider-{index}")
-                            #     else:
-                            #         slider = gr.Slider(minimum=0, maximum=1, step=0.01, value=value, label=label, key=f"slider-{index}")
-                                
-                            #     sliders.append(slider)
+
                             for i, (value, index) in enumerate(zip(feature_values, feature_indices)):
                                 feature = next((f for f in subject_data[current_subject]['feature_analysis'] if f['index'] == index), None)
                                 label = f"{feature['label']} ({index})" if feature else f"Feature {index}"
@@ -491,16 +684,6 @@ def create_interface():
                                 
                                 sliders.append(slider)
 
-                    # def on_slider_change(*values):
-                    #     manually_added_features = values[-1]
-                    #     slider_values = list(values[:-1])
-                        
-                    #     # Reconstruct feature_indices based on the order of sliders
-                    #     reconstructed_indices = [int(slider.label.split('(')[-1].split(')')[0]) for slider in sliders]
-                        
-                    #     new_results, new_values, new_indices = update_search_results(slider_values, reconstructed_indices, manually_added_features, current_subject)
-                    #     return new_results, new_values, new_indices, manually_added_features
-
                     def on_slider_change(*values):
                         manually_added_features = values[-1]
                         slider_values = list(values[:-1])
@@ -530,10 +713,18 @@ def create_interface():
                     visualize_button = gr.Button("Visualize Feature")
                 
                 feature_info = gr.Markdown()
+                # abstracts_heading = gr.Markdown("## Top 5 Abstracts")
+                # top_abstracts = gr.Dataframe(
+                #     headers=["Title", "Activation value"],
+                #     interactive=False
+                # )
+
                 abstracts_heading = gr.Markdown("## Top 5 Abstracts")
                 top_abstracts = gr.Dataframe(
                     headers=["Title", "Activation value"],
-                    interactive=False
+                    datatype=["markdown", "number"],
+                    interactive=False,
+                    wrap=True
                 )
                 
                 gr.Markdown("## Correlated Features")