Update controllers/fetch_microsoft_mails.py

controllers/fetch_microsoft_mails.py

@@ -62,40 +62,35 @@ def get_company_type(company_name:str)->str:
 
 async def get_messages(code: str,websocket:WebSocket,brand_name: Optional[str] = None):
     access_token = code
-    g_query = f'(subject:"your order" OR subject:receipts OR subject:receipt OR subject:aankoopbon OR subject:reçu OR subject:invoice OR subject:invoices OR category:purchases) has:attachment'
+    g_query = 'subject:"your order" OR subject:"receipts" OR subject:"receipt" OR subject:"aankoopbon" OR subject:"reçu" OR subject:"invoice" OR subject:"invoices" hasattachment:yes'
     if brand_name is not None:
-        g_query = f'(subject:"your order" OR subject:receipts OR subject:receipt OR  subject: aankoopbon  OR subject:reçu OR subject:invoice OR subject:invoices OR category:purchases OR from:{brand_name}) AND subject:{brand_name} has:attachment'    
-    page_token = None
+        g_query = f'(subject:"your order" OR subject:"receipts" OR subject:"receipt" OR  subject: "aankoopbon"  OR subject:"reçu" OR subject:"invoice" OR subject:"invoices" OR from:{brand_name}) AND subject:{brand_name} has:attachment'    
     messages = []
-    # max_results = 10
-    # gmail_url = f"https://www.googleapis.com/gmail/v1/users/me/messages?q={jobs_query}&maxResults={max_results}"
-    # gmail_response = requests.get(gmail_url, headers={"Authorization": f"Bearer {access_token}"})
-    # gmail_data = gmail_response.json()
-    # messages.append(gmail_data['messages'])
     def fetch_message_wrapper(message_data):
         message_id = message_data.get("id")
+        message_subject = message_data.get('subject')
+        message_body_preview = message_data.get('bodyPreview')
+        from_mail = message_data['from']['emailAddress']['address']
         if message_id:
-            return fetch_message_data(access_token, message_id)
+            return fetch_message_data(access_token,message_id,message_data,message_subject,message_body_preview,from_mail)
             
         return None
 
     while True:
-        gmail_url = f"https://www.googleapis.com/gmail/v1/users/me/messages?q={g_query}"
-        if page_token:
-            gmail_url += f"&pageToken={page_token}"
-
-        gmail_response = requests.get(gmail_url, headers={"Authorization": f"Bearer {access_token}"})
-        gmail_data = gmail_response.json()
-        print(len(gmail_data))
-        print(gmail_data)
+        outlook_url = f"https://graph.microsoft.com/v1.0/me/messages?q={g_query}"
+
+        outlook_response = requests.get(gmail_url, headers={"Authorization": f"Bearer {access_token}"})
+        outlook_data = gmail_response.json()
+        print(len(outlook_data))
+        print(outlook_data)
         
-        if "messages" in gmail_data:
+        if "value" in gmail_data:
             with ThreadPoolExecutor(max_workers=15) as executor:
                 
                            
                 
                 futures=[executor.submit(fetch_message_wrapper, message_data) for message_data in
-                           gmail_data["messages"]]
+                           outlook_data["value"]]
                 for future in futures:
                     message = future.result()
                     if message:
@@ -120,15 +115,15 @@ async def process_message(message:Message, websocket:WebSocket, chunk_size:int):
         await websocket.send_text("NEXT_MESSAGE")
 
 
-def fetch_message_data(access_token: str, message_id: str) -> Message:
-    message_url = f"https://www.googleapis.com/gmail/v1/users/me/messages/{message_id}"
-    message_response = requests.get(message_url, headers={"Authorization": f"Bearer {access_token}"})
-    message_data = message_response.json()
+def fetch_message_data(access_token: str,message_id:str, message_data,message_subject:str , message_body_preview,message_data,from_mail:str) -> Message:
+    # message_url = f"https://www.googleapis.com/gmail/v1/users/me/messages/{message_id}"
+    # message_response = requests.get(message_url, headers={"Authorization": f"Bearer {access_token}"})
+    # message_data = message_response.json()
     # print(message_data)
-    subject = extract_subject_from_mail(message_data)
-    company_from_mail = extract_domain_name(message_data['payload']['headers'], subject)
+    subject = message_subject
+    company_from_mail = from_mail
 
-    body = extract_body_from_mail(message_data)
+    body = message_body_preview
 
     attachments,structed_attachment_data = extract_attachments_from_mail(access_token, message_data)
     high_level_company_type = get_company_type(company_from_mail)
@@ -235,22 +230,44 @@ def fetch_attachment_data(access_token: str, message_id: str, attachment_id: str
 def extract_attachments_from_mail(access_token: str, message_data: dict) -> List[Attachment]:
     attachments = []
     structured_data = []
-    if "payload" in message_data and "parts" in message_data["payload"]:
-        for part in message_data["payload"]["parts"]:
-            if "body" in part and "attachmentId" in part["body"]:
-                attachment_id = part["body"]["attachmentId"]
-                attachment_data = fetch_attachment_data(access_token, message_data["id"], attachment_id)
-                filename = part.get("filename", "untitled.txt")
-                if filename.endswith(".zip") or filename.endswith(".txt") or filename.endswith(".png") or filename.endswith(".jpg") or filename.endswith(".jpeg") or filename.endswith(".gif"):
-                    continue
-                data = attachment_data.get("data", "")
-                raw_text=ut.extract_text_from_attachment(filename , data)
-                struct_data = ut.strcuture_document_data(raw_text)
-                if struct_data:
-                    structured_data.append(struct_data)
-                
-                attachments.append(Attachment(attachment_len = len(attachment_data.get("data", "")),filename=filename, data=attachment_data.get("data", "")))
+    message_id = message_data.get('id')
+    headers = {
+        'Authorization': f'Bearer {access_token}',
+        'Accept': 'application/json',
+    }
+    attachments_response = requests.get(f'https://graph.microsoft.com/v1.0/me/messages/{message_id}/attachments', headers=headers)
+    attachments = attachments_response.json().get('value', [])
+
+    for attachment in attachments:
+        attachment_id = attachment['id']
+        filename = attachment['name']
+        if filename.endswith(".zip") or filename.endswith(".txt") or filename.endswith(".png") or filename.endswith(".jpg") or filename.endswith(".jpeg") or filename.endswith(".gif"):
+            continue
+        attachment_response = requests.get(f'https://graph.microsoft.com/v1.0/me/messages/{message_id}/attachments/{attachment_id}/$value', headers=headers)
+        data = attachment_response.content
+        raw_text = ut.extract_text_from_attachment(filename , data)
+        struct_data = ut.strcuture_document_data(raw_text)
+        if struct_data:
+            structured_data.append(struct_data)
+        attachments.append(Attachment(attachment_len = len(data),filename=filename, data=data))
     return attachments,structured_data
+    #Gmail data fetching below
+    # if "payload" in message_data and "parts" in message_data["payload"]:
+    #     for part in message_data["payload"]["parts"]:
+    #         if "body" in part and "attachmentId" in part["body"]:
+    #             attachment_id = part["body"]["attachmentId"]
+    #             attachment_data = fetch_attachment_data(access_token, message_data["id"], attachment_id)
+    #             filename = part.get("filename", "untitled.txt")
+    #             if filename.endswith(".zip") or filename.endswith(".txt") or filename.endswith(".png") or filename.endswith(".jpg") or filename.endswith(".jpeg") or filename.endswith(".gif"):
+    #                 continue
+    #             data = attachment_data.get("data", "")
+    #             raw_text=ut.extract_text_from_attachment(filename , data)
+    #             struct_data = ut.strcuture_document_data(raw_text)
+    #             if struct_data:
+    #                 structured_data.append(struct_data)
+                
+    #             attachments.append(Attachment(attachment_len = len(attachment_data.get("data", "")),filename=filename, data=attachment_data.get("data", "")))
+    # return attachments,structured_data
 
 
 def extract_text(html_content: str) -> str: