Upload 2 files

app.py

@@ -43,7 +43,7 @@ from openai import OpenAI
 # import anthropic
 import cohere
 import faiss
-
+import matplotlib.pyplot as plt
 import spacy
 from string import punctuation
 import pytextrank
@@ -282,8 +282,8 @@ class RetrievalSystem():
         indices = [i for i in top_results]
         df.insert(1,'ADS Link',links,True)
         df.insert(2,'Relevance',scores,True)
-        df.insert(3,'Indices',indices,True)
-        df = df[['ADS Link','Relevance','date','cites','title','authors','abstract','keywords','ads_id','Indices','embed']]
+        df.insert(3,'indices',indices,True)
+        df = df[['ADS Link','Relevance','date','cites','title','authors','abstract','keywords','ads_id','indices','embed']]
         df.index += 1
         return df
 
@@ -391,7 +391,7 @@ def Library(query):
     papers_df = run_query_ret(st.session_state.query)
     op_docs = ''
     for i in range(len(papers_df)):
-        op_docs = op_docs + 'Paper %.0f:' %(i+1) + papers_df['title'][i]  + '\n' + papers_df['abstract'][i] + '\n\n'
+        op_docs = op_docs + 'Paper %.0f:' %(i+1) + papers_df['title'][i+1]  + '\n' + papers_df['abstract'][i+1] + '\n\n'
 
     return op_docs
 
@@ -451,7 +451,7 @@ def run_rag_qa(query, papers_df):
             metadata = {"source": row['ads_id']}
             doc = Document(page_content=content, metadata=metadata)
             documents.append(doc)
-            my_bar.progress((i+1)/len(papers_df), text='adding documents to LLM context')
+            my_bar.progress((i)/len(papers_df), text='adding documents to LLM context')
 
         text_splitter = RecursiveCharacterTextSplitter(chunk_size=150, chunk_overlap=50, add_start_index=True)
 
@@ -562,7 +562,7 @@ def calc_outlier_flag(papers_df, top_k, cutoff_adjust = 0.1):
 
 def make_embedding_plot(papers_df, consensus_answer):
 
-    plt_indices = np.array(papers_df['Indices'].tolist())
+    plt_indices = np.array(papers_df['indices'].tolist())
 
     if 'arxiv_corpus' not in st.session_state:
         st.session_state.arxiv_corpus = load_arxiv_corpus()
@@ -574,10 +574,17 @@ def make_embedding_plot(papers_df, consensus_answer):
     alphas = np.ones((len(plt_indices),)) * 0.9
     alphas[outlier_flag] = 0.5
 
-    fig = plt.figure(figsize=(9,12))
+    fig = plt.figure(figsize=(9*2.,12*2.))
     plt.scatter(xax,yax, s=1, alpha=0.01, c='k')
-    plt.scatter(xax[plt_indices], yax[plt_indices], s=300*alphas**2, alpha=alphas, c='w')
-    plt.scatter(xax[plt_indices], yax[plt_indices], s=100*alphas**2, alpha=alphas, c='dodgerblue')
+
+    clkws = np.load('kw_tags.npz')
+    all_x, all_y, all_topics, repeat_flag = clkws['all_x'], clkws['all_y'], clkws['all_topics'], clkws['repeat_flag']
+    for i in range(len(all_topics)):
+        if repeat_flag[i] == False:
+            plt.text(all_x[i], all_y[i], all_topics[i],fontsize=9,ha="center", va="center",
+                         bbox=dict(facecolor='white', edgecolor='black', boxstyle='round,pad=0.3',alpha=0.81))
+    plt.scatter(xax[plt_indices], yax[plt_indices], s=300*alphas**2, alpha=alphas, c='w',zorder=1000)
+    plt.scatter(xax[plt_indices], yax[plt_indices], s=100*alphas**2, alpha=alphas, c='dodgerblue',zorder=1001)
     # plt.scatter(xax[plt_indices][outlier_flag], yax[plt_indices][outlier_flag], s=100, alpha=1., c='firebrick')
     plt.axis([0,20,-4.2,18])
     plt.axis('off')
@@ -589,7 +596,7 @@ def make_embedding_plot(papers_df, consensus_answer):
 
 if st.session_state.get('runpfdr'):
     with st.spinner(search_text_list[np.random.choice(len(search_text_list))]):
-        st.write('Settings: [Kw:',toggle_a, 'Time:',toggle_b, 'Cite:',toggle_c, '] top_k:',top_k, 'retrieval:',method)
+        st.write('Settings: [Kw:',toggle_a, 'Time:',toggle_b, 'Cite:',toggle_c, '] top_k:',top_k, 'retrieval: `',method+'`')
 
         papers_df = run_query_ret(st.session_state.query)
         st.header(st.session_state.query)
@@ -636,7 +643,7 @@ if st.session_state.get('runpfdr'):
 
         with st.spinner("Evaluating abstract consensus"):
             with st.expander("Abstract consensus", expanded=True):
-                consensus_answer = evaluate_overall_consensus(query, [papers_df['abstract'][i] for i in range(len(papers_df))])
+                consensus_answer = evaluate_overall_consensus(query, [papers_df['abstract'][i+1] for i in range(len(papers_df))])
                 st.subheader("Consensus: "+consensus_answer.consensus)
                 st.markdown(consensus_answer.explanation)
                 st.markdown('Relevance of retrieved papers to answer: %.1f' %consensus_answer.relevance_score)

kw_tags.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d7068524d3d3029b8d36f4dd2fdf20d5c8a12fc69d8d1a404921aa54a6b40a8
+size 17849