Update app.py

app.py

@@ -1,6 +1,5 @@
-# # !pip install langchain langchain-groq sentence-transformers langchainhub faiss-cpu gradio gradio_client yfinance duckduckgo-search
-# Import python libraries
-# !pip install  yfinance
+# !pip install langchain langchain-groq sentence-transformers langchainhub faiss-cpu gradio gradio_client yfinance duckduckgo-search
+
 import pandas as pd
 import io
 import requests
@@ -14,6 +13,8 @@ import requests
 import yfinance as yf
 import ast
 import re
+from datetime import datetime, timedelta
+import pytz   
 
 
 # import langchain libraries
@@ -150,7 +151,7 @@ data_old_2 = loader2.load()
 loader3 = CSVLoader("nse_ca.csv")
 data_ca = loader3.load()
 
-global vectorstore,vectorstore2,vectorstore3
+global vectorstore,vectorstore2,vectorstore3, colist, colist_tracked
 # Create vectorstores - I tried Chroma but FAISS turned out to be successful
 vectorstore = FAISS.from_documents(data_old, embedding_function)
 vectorstore2 = FAISS.from_documents(data_old_2, embedding_function)
@@ -184,7 +185,6 @@ co_list = co_list1 + co_list_tracked
 
 ####################################
 
-
 ##################################
 # Let us create some functions required
 ##################################
@@ -197,6 +197,8 @@ def give_announcement(llm,stock):
 
     else:
 
+
+      retriever1 = vectorstore.as_retriever()
       qa_chain = RetrievalQA.from_chain_type(llm,
                                         retriever=retriever1,
                                         return_source_documents=False)
@@ -217,7 +219,7 @@ def get_ca(llm,stock):
     # stock = resp1.content
 
 
-
+    retriever3 = vectorstore3.as_retriever()
     qa_chain2 = RetrievalQA.from_chain_type(llm,
                                        retriever=retriever3,
                                        return_source_documents=False)
@@ -250,12 +252,12 @@ def get_movements(llm,stock):
     return "This company has not made any announcements today or yesterday"
   else:
 
-    stock = stock[0] 
+    stock = stock[0]
 
     dfc = pd.read_csv('nse_data_old.csv')
 
-    stockdesc = dfc[dfc['COMPANY NAME'] == stock]['COMPANY NAME'].iloc[0] 
-    stock1 = dfc[dfc['COMPANY NAME'] == stock]['SYMBOL'].iloc[0] 
+    stockdesc = dfc[dfc['COMPANY NAME'] == stock]['COMPANY NAME'].iloc[0]
+    stock1 = dfc[dfc['COMPANY NAME'] == stock]['SYMBOL'].iloc[0]
     stock = get_ticker(stock1)
 
     print("stock is ",stock)
@@ -319,11 +321,13 @@ def get_sentiments(llm,stock):
   if not stock:
     return "This company has not made any announcements today or yesterday"
   else:
-    stock = stock[0] 
+    print("st1",stock)
+    stock = stock[0]
+    print("af ",stock)
     #####
     dfc = pd.read_csv('nse_data_old.csv')
 
-    stockdesc = dfc[dfc['COMPANY NAME'] == stock]['COMPANY NAME'].iloc[0] 
+    stockdesc = dfc[dfc['COMPANY NAME'] == stock]['COMPANY NAME'].iloc[0]
     stock1 = dfc[dfc['COMPANY NAME'] == stock]['SYMBOL'].iloc[0]
     stock = get_ticker(stock1)
     tools=[
@@ -356,12 +360,17 @@ def get_sentiments(llm,stock):
     )
 
     # Construct the Tools agent
-    agent = create_tool_calling_agent(llm, tools, prompt)
+    # agent = create_tool_calling_agent(llm, tools, prompt)
 
     # Create an agent executor by passing in the agent and tools
     agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True)
-    response = agent_executor.invoke({"input": f"Get broker sentiment for the stock {stock} and stock name {stockdesc}"})
-    return f"Broker sentiment analysis for {stock}. - {response['output']}"
+    try:
+      agent = create_tool_calling_agent(llm, tools, prompt)
+      agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True)
+      response = agent_executor.invoke({"input": f"Get broker sentiment for the stock {stock} and stock name {stockdesc}"})
+      return f"Broker sentiment analysis for {stock}. - {response['output']}"
+    except Exception as e:
+      return f"An error occurred: {str(e)}"
     #################
 
 # Fetch financial statements from Yahoo Finance
@@ -399,7 +408,9 @@ def get_ticker(company_name):
     ticker = yf.Ticker(com)
     return ticker.info['symbol']
 
-def get_financialratio(model, input):
+def get_financialratio(model, input,stock):
+
+    stock_name = get_companynames(stock)
 
     llm = get_model(model)
 
@@ -499,7 +510,7 @@ def get_financialratio(model, input):
               # Print only the results. Print the output in Json format.")
               return f"For the company: {stockname}, Here are the details: {response}"
             except Exception as e:
-              return f"An error occurred: {str(e)}" 
+              return f"An error occurred: {str(e)}"
 
 ##########################
 # Functions to plot a chart over ratios - this has scope for major enhancements!
@@ -529,10 +540,11 @@ def plot_chart(data):
 #     plt = plot_chart(response)
 #     return plt
 
-def get_chart(model,input):
+def get_chart(model,input,stock):
+    stock_name = get_companynames(stock)
     if stock_name:
 
-      response = get_financialratio(model,input)
+      response = get_financialratio(model,input,stock)
       # Extract the dictionary part using regex
       dict_match = re.search(r"\{.*\}", response)  # Search for content within curly braces
 
@@ -548,8 +560,8 @@ def get_chart(model,input):
     else: return None
 
 
-def combined_ratio(model, input):
-  return get_financialratio(model,input), get_chart(model, input)
+def combined_ratio(model, input,stock):
+  return get_financialratio(model,input,stock), get_chart(model, input,stock)
 
 ###############################
 
@@ -559,9 +571,9 @@ def combined_ratio(model, input):
 # Create the Gradio Blocks interface with a title and description
 
 ##################################
-
+global flag
 def incremental_process():
-    global vectorstore,vectorstore2,vectorstore3, dfco, co_list2, co_list3, retriever1, retriever2, retriever3, flag, timestamp
+    global vectorstore,vectorstore2,vectorstore3, flag
 
     try:
         df_new, _ = get_pd(1)
@@ -577,22 +589,20 @@ def incremental_process():
     #drop unnecessary common columns
     df_new.drop(['RECEIPT','DISSEMINATION','DIFFERENCE'],axis=1,inplace=True)
 
-    
-
-    # #find the difference and add incrementally for first store     
+    # #find the difference and add incrementally for first store
     df_merged = df_new.merge(df_old, how='left', indicator=True)
     # Filter rows that are unique to 'n' (i.e., where '_merge' is 'left_only')
     df_add1= df_merged[df_merged['_merge'] == 'left_only'].drop(columns=['_merge'])
 
     # Save it as a CSV file
     df_add1.to_csv("nse_data_add1.csv", index=False)
-    
+
     #drop unnecessary columns for second vector store
     df_new2 = df_new.drop(['ATTACHMENT'],axis=1)
 
     # add increment for second store
     df_merged = df_new2.merge(df_old2, how='left', indicator=True)
-    df_add2 = df_merged[df_merged['_merge'] == 'left_only'].drop(columns=['_merge']) 
+    df_add2 = df_merged[df_merged['_merge'] == 'left_only'].drop(columns=['_merge'])
     # Save it as a CSV file
     df_add2.to_csv("nse_data_add2.csv", index=False)
 
@@ -614,21 +624,7 @@ def incremental_process():
       # Append df2 at the end of df1
       dfco = pd.concat([dfold, dfadd], ignore_index=True)
 
-      # dfco = dfold.append(dfadd)
-
-    dfco1 = dfco[['COMPANY NAME']]
-    dfco2 = dfco1.drop_duplicates()
-
-    # Save the result to a new CSV file
-    dfco2.to_csv('companies.csv', index=False)
-
-    dfco3 = dfco2.head(10)
-
-    co_list3 = dfco3['COMPANY NAME'].unique().tolist()
-
-    filtered_df = dfco2[dfco2['COMPANY NAME'].isin(co_list)]
-
-    co_list2 = filtered_df['COMPANY NAME'].tolist()
+    dfco.to_csv("dfco.csv",index=False)
 
     # Here incremental RAG is achieved by adding additional data dynamically to vectorstore
     loader = CSVLoader("nse_data_add1.csv")
@@ -681,39 +677,78 @@ def incremental_process():
     vectorstore3 = FAISS.load_local("vectorstore3",embedding_function,allow_dangerous_deserialization=True)
     print("final size store 3",vectorstore3.index.ntotal)
 
-    retriever1 = vectorstore.as_retriever()
-    retriever2 = vectorstore2.as_retriever()
-    retriever3 = vectorstore3.as_retriever()
+    return flag
+
+
+def get_colist2():
+  dfco = pd.read_csv('dfco.csv')
+  dfco1 = dfco[['COMPANY NAME']]
+  dfco2 = dfco1.drop_duplicates()
+  # Save the result to a new CSV file
+  dfco2.to_csv('companies.csv', index=False)
+
+  dfco3 = dfco2.head(10)
+
+  co_list3 = dfco3['COMPANY NAME'].unique().tolist()
 
+  filtered_df = dfco2[dfco2['COMPANY NAME'].isin(co_list)]
+
+  co_list2 = filtered_df['COMPANY NAME'].tolist()
+  return co_list2, co_list3
+
+def get_timestampmessage(flag):
+    dfco = pd.read_csv('dfco.csv')
     timestamp = dfco[['BROADCAST DATE/TIME']].max().values.tolist()[0]
-    # return flag, co_list2, retriever1, retriever2, retriever3
     if flag == 1:
       message = f"There is NSE timeout error. The latest filing information is available upto {timestamp}"
     else: message = f"Lastest filing information is available upto {timestamp}"
     return message
 
-_ = incremental_process()
+
+def update():
+  global flag
+  flag = incremental_process()
+  message = get_timestampmessage(flag)
+  return message
 
 def give_time():
-  global dfco, timestamp
+  dfco = pd.read_csv("dfco.csv")
   timestamp = dfco[['BROADCAST DATE/TIME']].max().values.tolist()[0]
   return timestamp
 
 
+
+# Define the IST timezone
+ist_timezone = pytz.timezone("Asia/Kolkata")
+# Define UTC for server-side time
+utc_timezone = pytz.utc
+
 def refresh():
-  global timestamp
-  given_time = datetime.strptime(timestamp, "%d-%b-%Y %H:%M:%S")
-  current_time = datetime.now()
-  time_difference = current_time - given_time
-    
-  # Check if the difference is greater than one hour
-  if time_difference > timedelta(hours=1):
-    message = incremental_process()
-  else:
-    message = f"Refresh allowed only if data is stale for more than one hour. \n" \
-    f"present timestamp {timestamp}"
-  return message
-##########################################################################
+    # Get the client-side timestamp (assuming it is in IST)
+    timestamp_str = give_time()  # The format returned should match the expected format
+    given_time = datetime.strptime(timestamp_str, "%d-%b-%Y %H:%M:%S")
+    given_time_ist = ist_timezone.localize(given_time)  # Localize to IST
+
+    # Get the current server time in UTC
+    current_time_utc = datetime.now(tz=utc_timezone)
+
+    # Convert the client-side time to UTC for consistent comparison
+    given_time_utc = given_time_ist.astimezone(utc_timezone)
+
+    # Calculate the time difference
+    time_difference = current_time_utc - given_time_utc
+
+    print("the time diff is ", time_difference)
+
+    # Check if the time difference is greater than one hour
+    if time_difference > timedelta(hours=1):
+        message1 = update()
+        print("Incremental update run")
+    else:
+        message1 = f"Refresh allowed only if data is stale for more than one hour. Current client timestamp: {timestamp_str}"
+
+    return message1
+    ##########################################################################
 
 
 def plot1_top_20():
@@ -761,8 +796,9 @@ def plot1_top_20():
 
   ## Function to create company list specific chart
 def plot2_top_20():
+  co_list2,_ = get_colist2()
 
-  global co_list2
+  # global co_list2
   # Get the counts of each label
   df = pd.read_csv('nse_data_old.csv')
   subjects = ['Acquisition',
@@ -822,10 +858,11 @@ def get_companynames(stock):
       unique_companies = matched_rows['COMPANY NAME'].unique()
 
       return list(set(unique_companies))
-    else: return None 
+    else: return None
 
 # A combined function to be used in Gradio output box
 def print_model(llm):
+  co_list2,_ = get_colist2()
   if co_list2:
       return f"You are using {llm.model_name} model for this session. \n \n" \
       f"These are the companies you track: {co_list_tracked}. \n \n" \
@@ -841,11 +878,9 @@ def print_model1(llm):
 
 
 def combined_function1(model,stock):
-  global stock_name, flag, co_list2, retriever1, retriever2, retriever3
-  # flag, co_list2, retriever1, retriever2, retriever3 = incremental_process()
+  global flag
   llm = get_model(model)
   stock = get_companynames(stock)
-  stock_name = stock
   if flag == 0:
     return print_model(llm), give_announcement(llm,stock),get_ca(llm,stock),get_movements(llm,stock), get_sentiments(llm,stock)
   else:
@@ -865,6 +900,8 @@ def get_model(model_name):
 
 # This function is given here as company list is dynamic
 def give_names():
+  global co_list_tracked
+  co_list2, co_list3 = get_colist2()
   return f"Apart from NIFTY, these are the companies you track: \n \n" \
   f" {co_list_tracked}. \n \n" \
   f"These are the tracked companies that have made announcements: \n \n" \
@@ -874,9 +911,9 @@ def give_names():
 
 
 ##############################
-retrieval_qa_chat_prompt = hub.pull("langchain-ai/retrieval-qa-chat")  
+retrieval_qa_chat_prompt = hub.pull("langchain-ai/retrieval-qa-chat")
   ###############################
-  
+
 # This function is for chat queries. Given here due to retriever defined here
 def chat_chain(model,query):
   llm = get_model(model)
@@ -885,15 +922,19 @@ def chat_chain(model,query):
   else:
     combine_docs_chain = create_stuff_documents_chain(
     llm, retrieval_qa_chat_prompt)
+    retriever2 = vectorstore2.as_retriever()
     retrieval_chain = create_retrieval_chain(retriever2, combine_docs_chain)
     response = retrieval_chain.invoke({"input": query})
     return response['answer']
 
 
-# ################################
+#################################
+## Update the vectorstate with latest data
+flag = incremental_process()
+###########################################################################
 
-    ###########################################################################
 with gr.Blocks() as demo:
+
   # Add a Markdown block for the description
   gr.Markdown("""<h1 style='color: blue;'>Chat and Analyze with NSE Filings Information</h1>""")
   gr.Markdown("""Powered by Gradio, Groq, Llama3, FAISS, Langchain, YahooFinance""")
@@ -907,8 +948,8 @@ with gr.Blocks() as demo:
       """
   )
 
-  txt_output = gr.Text(give_time,label = "Opening Data - Timestamp of latest Filing")
-  txt_output = gr.Text(give_names,label = "Announcements for tracked companies")
+  txt_output = gr.Text(give_time(),label = "Opening Data - Timestamp of latest Filing")
+  txt_output = gr.Text(give_names(),label = "Announcements for tracked companies")
 
   # This is for defaulting charts when app is launched
   plot_output1 = gr.Plot(plot1_top_20(), label="Chart")  # Call the function to create the plot
@@ -916,11 +957,12 @@ with gr.Blocks() as demo:
   plot_output2 = gr.Plot(plot2_top_20(), label="Chart")  # Call the function to create the plot
   plt.close()
   gr.Markdown("""<h2 style='color: blue;'>Fetch Announcements/Corporate Actions/Price Movements/Broker Sentiments</h2>""")
-  # Use a Column to structure the inputs and outputs 
+  # Use a Column to structure the inputs and outputs
   with gr.Column():
         outputs5 = [gr.Textbox(label="Latest Filing Timestamp",placeholder="Refresh data if stale for more than an hour")]
         button5 = gr.Button("Refresh Data")
-        button5.click(refresh, inputs=None, outputs=outputs5)
+        # button5.click(lambda: refresh(dfco), inputs=None, outputs=outputs5)
+        button5.click(lambda: refresh(), inputs=None, outputs=outputs5)
 
         # Create a dropdown box for selecting the operation
         operation_dropdown = gr.Dropdown(
@@ -943,23 +985,21 @@ with gr.Blocks() as demo:
           gr.Textbox(label="Broker Sentiment", max_lines=100,show_copy_button=True),
       ]
 
-        button1.click(combined_function1, inputs=[operation_dropdown,text_input1], outputs=outputs1)
-        gr.Markdown("""<h1 style='color: green;'>Analyse Financial Statements for any Company</h1>""")
+        button1.click(lambda x,y: combined_function1(x,y), inputs=[operation_dropdown,text_input1], outputs=outputs1)
+        gr.Markdown("""<h1 style='color: green;'>Analyse the Financial Statements of the above Company</h1>""")
 
-      # third  button
         text_input3 = gr.Textbox(
           label="Enter Query",
           placeholder="Enter your query: e.g., What is the current ratio of the stock over three years?",
-          lines=2
-        
-      )
+          lines=1)
+
         button3 = gr.Button("Analyse")
         outputs3 = [
           gr.Textbox(label="Chat Response", max_lines=100,show_copy_button=True),
-          gr.Plot(label = "Chart")
-      ]
-      # button3.click(get_financialratio, inputs=[operation_dropdown,text_input3], outputs=outputs3)
-        button3.click(combined_ratio, inputs=[operation_dropdown,text_input3], outputs=outputs3)  
+          gr.Plot(label = "Chart")]
+
+
+        button3.click(combined_ratio, inputs=[operation_dropdown,text_input3,text_input1], outputs=outputs3)
 
         gr.Markdown("""<h1 style='color: orange;'>Chat With the NSE Filings Information</h1>""")
 
@@ -975,4 +1015,5 @@ with gr.Blocks() as demo:
         button2.click(chat_chain, inputs=[operation_dropdown,text_input2], outputs=outputs2)
 
 # Launch the Gradio app
-demo.launch()
+demo.launch()
+