Spaces:
Sleeping
Sleeping
File size: 10,820 Bytes
b7f28d9 8b7bbc9 05aa824 3360fd5 cb10ac5 c257b14 b202189 cb10ac5 8b7bbc9 7c15110 8b7bbc9 7c15110 05aa824 8b7bbc9 4d85d66 8b7bbc9 cae1b3e 732b374 8b7bbc9 e715b7e 8b7bbc9 aaebbaa a7c9f3e a4fff20 a07cfe0 347dbf6 68bbf2b a07cfe0 6470042 a4fff20 6470042 f562fef 90bfa42 f562fef 62bb4d3 f562fef 8b7bbc9 87cc66e 0f607c6 8b7bbc9 347dbf6 8b7bbc9 b427156 8b7bbc9 238f6ea 8b7bbc9 d580044 8b7bbc9 87cc66e 8b7bbc9 2e70b03 8b7bbc9 ffa92da 447ff33 edc454d ffa92da 447ff33 0b06f94 447ff33 52b5672 447ff33 52b5672 447ff33 52b5672 2e0507d b42bcc4 5c98dbc 05aa824 875d884 447ff33 0365945 4394a0e 447ff33 c64698c 2e0507d 62bb4d3 2e0507d 62bb4d3 2e0507d 62bb4d3 2e0507d 9105bbb 52be56a c64698c 2e0507d 4394a0e 2e0507d 62bb4d3 2e0507d 2f880cf 2e0507d 447ff33 09acd07 447ff33 ffa92da 2e0507d e715b7e 11792d4 4a1f890 fc80af7 3360fd5 fa3bee1 4a1f890 8b7bbc9 0eadc0e 7a6ea5f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 |
from fastapi import FastAPI, Depends , Request,WebSocket
from fastapi.security import OAuth2PasswordBearer
import requests
from jose import jwt
import webbrowser
import base64
import logging
import time
import asyncio
import PyPDF2
from docx import Document
from PIL import Image
import pytesseract
import io
import base64
app = FastAPI()
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
# Replace these with your own values from the Google Developer Console
# GOOGLE_CLIENT_ID = ""
# GOOGLE_CLIENT_SECRET = ""
# GOOGLE_REDIRECT_URI = ""
GOOGLE_CLIENT_ID = "485753721652-5uta3e18va2g6cnkldib2d68q39t4vod.apps.googleusercontent.com"
GOOGLE_CLIENT_SECRET = "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"
GOOGLE_REDIRECT_URI = "https://omkar008-receipt-radar-test.hf.space/auth/google"
# Configure the logger
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
@app.get("/")
async def login_google():
# oauth_url = f"https://accounts.google.com/o/oauth2/auth?response_type=code&client_id={GOOGLE_CLIENT_ID}&redirect_uri={GOOGLE_REDIRECT_URI}&scope=openid%20profile%20email&access_type=offline"
#Below is the URL to prompt the user to login to his specified gmail account and also give a readonly access to his gmail
oauth_url = f"https://accounts.google.com/o/oauth2/auth?response_type=code&client_id={GOOGLE_CLIENT_ID}&redirect_uri={GOOGLE_REDIRECT_URI}&scope=openid%20profile%20email%20https://www.googleapis.com/auth/gmail.readonly&access_type=offline"
oauth_url_hr = f"https://accounts.google.com/o/oauth2/auth?response_type=code&client_id={GOOGLE_CLIENT_ID}&redirect_uri={GOOGLE_REDIRECT_URI}&scope=openid%20profile%20email%20https://www.googleapis.com/auth/gmail.readonly&access_type=offline&state=receipts"
webbrowser.open(oauth_url)
return {
"url_hr": oauth_url_hr
}
@app.post("/auth/google")
async def auth_google(request: Request):
data = await request.json()
code = data.get("access_token")
print("Printing the access token")
print(code)
if not code:
raise HTTPException(status_code=400, detail="Authorization code not provided")
access_token_new = code
user_info = requests.get("https://www.googleapis.com/oauth2/v1/userinfo", headers={"Authorization": f"Bearer {access_token_new}"})
page_token = None
messages = []
jobs_query = "subject:receipt OR subject:receipts has:attachment "
while True:
# Construct Gmail API request with pageToken
gmail_url = f"https://www.googleapis.com/gmail/v1/users/me/messages?q={jobs_query}"
if page_token:
gmail_url += f"&pageToken={page_token}"
gmail_response = requests.get(gmail_url, headers={"Authorization": f"Bearer {access_token_new}"})
gmail_data = gmail_response.json()
# Check if there are messages in the response
if "messages" in gmail_data:
messages.extend(gmail_data["messages"])
# Check if there are more pages
if "nextPageToken" in gmail_data:
page_token = gmail_data["nextPageToken"]
else:
break # No more pages, exit the loop
attachments = []
attachment_no = 0
data_new = {}
for i,message in enumerate(messages) :
# print(i)
# print(message)
if message:
message_id = message.get("id")
print(message_id)
if message_id:
message_url = f"https://www.googleapis.com/gmail/v1/users/me/messages/{message_id}"
message_response = requests.get(message_url, headers={"Authorization": f"Bearer {access_token_new}"})
message_data = message_response.json()
# Check for parts in the message payload
if "payload" in message_data and "parts" in message_data["payload"]:
for part in message_data["payload"]["parts"]:
if "body" in part and "attachmentId" in part["body"]:
attachment_id = part["body"]["attachmentId"]
attachment_url = f"https://www.googleapis.com/gmail/v1/users/me/messages/{message_id}/attachments/{attachment_id}"
attachment_response = requests.get(attachment_url, headers={"Authorization": f"Bearer {access_token_new}"})
attachment_data = attachment_response.json()
data = attachment_data.get("data")
filename = part.get("filename", "untitled.txt")
if data:
data_new[filename]=data
attachment_content = base64.urlsafe_b64decode(data.encode("UTF-8"))
attachment_no+=1
return {"attachment_count":attachment_no,"attachment_content":data_new}
async def send_chunked_data(websocket: WebSocket, filename: str, data: str):
chunk_size = 1024 # Set an appropriate chunk size
for i in range(0, len(data), chunk_size):
await websocket.send_json({"filename": filename, "data_chunk": data[i:i + chunk_size]})
await asyncio.sleep(0.4)
await websocket.send_text("FinishedThisAttachment")
async def extract_text_from_pdf(pdf_data):
with io.BytesIO(base64.b64decode(pdf_data)) as pdf_file:
pdf_reader = PyPDF2.PdfReader(pdf_file)
text = ""
for page_num in range(pdf_reader.numPages):
page = pdf_reader.getPage(page_num)
text += page.extract_text()
return text
async def extract_text_from_docx(docx_data):
doc = Document(io.BytesIO(base64.b64decode(docx_data)))
text = ""
for para in doc.paragraphs:
text += para.text + "\n"
return text
async def extract_text_from_attachment(filename, data):
if filename.endswith('.pdf'):
return await extract_text_from_pdf(data)
elif filename.endswith('.docx'):
return await extract_text_from_docx(data)
else:
# Add handling for other document types if needed
return "Unsupported document type"
@app.websocket("/ws")
async def test_websocket(websocket: WebSocket):
await websocket.accept()
logger.info("Hi hi succefull in connecting !!")
data = await websocket.receive_text()
logger.info("Received JSON data: %s", data)
def get_messages(code:str):
logging.info("entered into the get_messages")
access_token = code
print("printing access_token")
print(access_token)
page_token = None
messages = []
jobs_query = "subject:receipt OR subject:receipts has:attachment"
while True:
# Construct Gmail API request with pageToken
print("into the gmail")
gmail_url = f"https://www.googleapis.com/gmail/v1/users/me/messages?q={jobs_query}"
if page_token:
gmail_url += f"&pageToken={page_token}"
gmail_response = requests.get(gmail_url, headers={"Authorization": f"Bearer {access_token}"})
logging.info(f"{gmail_response}")
print(gmail_response)
gmail_data = gmail_response.json()
# Check if there are messages in the response
if "messages" in gmail_data:
messages.extend(gmail_data["messages"])
# Check if there are more pages
if "nextPageToken" in gmail_data:
page_token = gmail_data["nextPageToken"]
else:
break # No more pages, exit the loop
print("returning the messages")
unique_thread_ids = set()
filtered_data_list = []
for entry in messages:
thread_id = entry['threadId']
if thread_id not in unique_thread_ids:
unique_thread_ids.add(thread_id)
filtered_data_list.append(entry)
return filtered_data_list
async def event_generator(code:str):
logging.info("entered into the event_generator")
access_token = code
messages=get_messages(access_token)
print(len(messages))
attachments = []
prev_data=""
data_new={}
attachment_no=0
batch_size = 5
prev_filename = None
for i,message in enumerate(messages) :
print(i)
logging.info(f"{i}")
logging.info(f"{message}")
print(message)
if message:
message_id = message.get("id")
thread_id = message.get("threadId")
print(message_id)
if message_id:
message_url = f"https://www.googleapis.com/gmail/v1/users/me/messages/{message_id}"
message_response = requests.get(message_url, headers={"Authorization": f"Bearer {access_token}"})
message_data = message_response.json()
# Check for parts in the message payload
if "payload" in message_data and "parts" in message_data["payload"]:
for part in message_data["payload"]["parts"]:
if "body" in part and "attachmentId" in part["body"]:
attachment_id = part["body"]["attachmentId"]
print(attachment_id)
attachment_url = f"https://www.googleapis.com/gmail/v1/users/me/messages/{message_id}/attachments/{attachment_id}"
attachment_response = requests.get(attachment_url, headers={"Authorization": f"Bearer {access_token}"})
attachment_data = attachment_response.json()
data = attachment_data.get("data",{})
filename = part.get("filename", "untitled.txt")
if data:
data_new[filename]=str(data[:10])
attachment_content = base64.urlsafe_b64decode(data.encode("UTF-8"))
logging.info(filename)
extracted_text = await extract_text_from_attachment(filename, data)
logging.info(extracted_text)
await send_chunked_data(websocket, filename, data)
attachment_no+=1
await websocket.send_text("CompletedFetchingMessages")
await event_generator(data)
logging.info("Closing connection")
await websocket.close()
@app.get("/token")
async def get_token(token: str = Depends(oauth2_scheme)):
return jwt.decode(token, GOOGLE_CLIENT_SECRET, algorithms=["HS256"])
|