File size: 14,043 Bytes
b7f28d9
8b7bbc9
 
 
 
 
05aa824
3360fd5
cb10ac5
c257b14
 
 
 
cb10ac5
8b7bbc9
 
 
 
 
 
 
 
7c15110
 
 
8b7bbc9
7c15110
 
 
b4bcc22
05aa824
 
 
 
8b7bbc9
4d85d66
8b7bbc9
 
 
 
 
388857b
732b374
8b7bbc9
e715b7e
8b7bbc9
 
aaebbaa
a7c9f3e
a4fff20
a07cfe0
 
347dbf6
68bbf2b
a07cfe0
 
 
6470042
 
 
 
 
 
 
 
 
 
 
 
a4fff20
6470042
f562fef
 
959407d
f562fef
 
 
 
 
 
 
 
 
62bb4d3
f562fef
 
 
 
 
 
 
 
 
 
8b7bbc9
87cc66e
0f607c6
8b7bbc9
347dbf6
 
8b7bbc9
 
 
 
 
 
b427156
8b7bbc9
 
 
 
 
 
 
 
238f6ea
8b7bbc9
 
 
 
 
d580044
8b7bbc9
87cc66e
8b7bbc9
87cc66e
 
 
8b7bbc9
87cc66e
 
 
 
 
 
8b7bbc9
2e70b03
8b7bbc9
ffa92da
 
 
 
 
edc454d
 
 
ffa92da
52b5672
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2e0507d
b42bcc4
e715b7e
 
2e0507d
 
 
 
5c98dbc
05aa824
 
 
f027aaa
05aa824
 
 
875d884
 
05aa824
b42bcc4
 
2e0507d
444c10d
 
 
0365945
4394a0e
2e0507d
 
c64698c
2e0507d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62bb4d3
2e0507d
 
62bb4d3
2e0507d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62bb4d3
2e0507d
62bb4d3
 
2e0507d
 
9105bbb
52be56a
c64698c
 
2e0507d
 
4394a0e
2e0507d
b6fdf84
 
 
 
2e0507d
 
 
 
 
 
 
 
 
 
62bb4d3
 
 
2e0507d
 
 
 
 
 
 
 
 
 
 
2f880cf
2e0507d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ffa92da
 
52b5672
ffa92da
 
2e0507d
ffa92da
0e83ece
2e0507d
 
 
 
e715b7e
 
2e0507d
e715b7e
 
 
11792d4
4a1f890
fc80af7
3360fd5
fa3bee1
4a1f890
8b7bbc9
 
 
 
0eadc0e
7a6ea5f
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
from fastapi import FastAPI, Depends , Request,WebSocket
from fastapi.security import OAuth2PasswordBearer
import requests
from jose import jwt
import webbrowser
import base64
import logging
import time
import asyncio
import PyPDF2
from docx import Document
from PIL import Image
import pytesseract

app = FastAPI()
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")





# Replace these with your own values from the Google Developer Console
# GOOGLE_CLIENT_ID = ""
# GOOGLE_CLIENT_SECRET = ""
# GOOGLE_REDIRECT_URI = ""

GOOGLE_CLIENT_ID = "485753721652-5uta3e18va2g6cnkldib2d68q39t4vod.apps.googleusercontent.com"
GOOGLE_CLIENT_SECRET = "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"
GOOGLE_REDIRECT_URI = "https://omkar008-receipt-radar-test.hf.space/auth/google"
GOOGLE_REDIRECT_URI_hr = "https://receiptradar-0bb387d81174.herokuapp.com/auth/google"
# Configure the logger
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)


@app.get("/")
async def login_google():
    # oauth_url = f"https://accounts.google.com/o/oauth2/auth?response_type=code&client_id={GOOGLE_CLIENT_ID}&redirect_uri={GOOGLE_REDIRECT_URI}&scope=openid%20profile%20email&access_type=offline"
    
    #Below is the URL to prompt the user to login to his specified gmail account and also give a readonly access to his gmail
    oauth_url = f"https://accounts.google.com/o/oauth2/auth?response_type=code&client_id={GOOGLE_CLIENT_ID}&redirect_uri={GOOGLE_REDIRECT_URI}&scope=openid%20profile%20email%20https://www.googleapis.com/auth/gmail.readonly&access_type=offline"
    oauth_url_hr = f"https://accounts.google.com/o/oauth2/auth?response_type=code&client_id={GOOGLE_CLIENT_ID}&redirect_uri={GOOGLE_REDIRECT_URI_hr}&scope=openid%20profile%20email%20https://www.googleapis.com/auth/gmail.readonly&access_type=offline&state=receipts"
    webbrowser.open(oauth_url)
    return {
        "url_hr": oauth_url_hr
    }

@app.post("/auth/google")
async def auth_google(request: Request):

    data = await request.json()
    code = data.get("access_token")
    print("Printing the access token")
    print(code)
    if not code:
        raise HTTPException(status_code=400, detail="Authorization code not provided")

    # token_url = "https://accounts.google.com/o/oauth2/token"
    # print(code)
    # data = {
    #     "code": code,
    #     "client_id": GOOGLE_CLIENT_ID,
    #     "client_secret": GOOGLE_CLIENT_SECRET,
    #     "redirect_uri": GOOGLE_REDIRECT_URI,
    #     "grant_type": "authorization_code",
    # }
    # response = requests.post(token_url, data=data)
    # access_token = response.json().get("access_token")
    access_token_new = code

    user_info = requests.get("https://www.googleapis.com/oauth2/v1/userinfo", headers={"Authorization": f"Bearer {access_token_new}"})
    page_token = None
    messages = []
    jobs_query = "subject:receipt OR subject:receipts OR subject:Invoice OR subject:invoice has:attachment "

    while True:
        # Construct Gmail API request with pageToken
        gmail_url = f"https://www.googleapis.com/gmail/v1/users/me/messages?q={jobs_query}"
        if page_token:
            gmail_url += f"&pageToken={page_token}"

        gmail_response = requests.get(gmail_url, headers={"Authorization": f"Bearer {access_token_new}"})
        gmail_data = gmail_response.json()
        
        # Check if there are messages in the response
        if "messages" in gmail_data:
            messages.extend(gmail_data["messages"])

        # Check if there are more pages
        if "nextPageToken" in gmail_data:
            page_token = gmail_data["nextPageToken"]
        else:
            break  # No more pages, exit the loop

    attachments = []
    attachment_no = 0
    data_new = {}
    for i,message in enumerate(messages) :
        # print(i)
        # print(message)

        if message:
            message_id = message.get("id")
            print(message_id)
            if message_id:
                message_url = f"https://www.googleapis.com/gmail/v1/users/me/messages/{message_id}"
                message_response = requests.get(message_url, headers={"Authorization": f"Bearer {access_token_new}"})
                message_data = message_response.json()

                # Check for parts in the message payload
                if "payload" in message_data and "parts" in message_data["payload"]:
                    for part in message_data["payload"]["parts"]:
                        if "body" in part and "attachmentId" in part["body"]:
                            attachment_id = part["body"]["attachmentId"]
                            attachment_url = f"https://www.googleapis.com/gmail/v1/users/me/messages/{message_id}/attachments/{attachment_id}"
                            attachment_response = requests.get(attachment_url, headers={"Authorization": f"Bearer {access_token_new}"})
                            attachment_data = attachment_response.json()
                            data = attachment_data.get("data")
                            filename = part.get("filename", "untitled.txt")

                            if data:
                                data_new[filename]=data
                                attachment_content = base64.urlsafe_b64decode(data.encode("UTF-8"))
                                attachment_no+=1

                            # if data:
                            #     # Decode base64-encoded attachment data
                            #     attachment_content = base64.urlsafe_b64decode(data.encode("UTF-8"))

                            #     # Save the attachment to a file
                            #     save_path = f"/Users/omkarmalpure/Documents/Gmail_API/attachments/{filename}"
                            #     with open(save_path, "wb") as file:
                            #         file.write(attachment_content)

                            #     attachments.append(save_path)

    return {"attachment_count":attachment_no,"attachment_content":data_new}


async def send_chunked_data(websocket: WebSocket, filename: str, data: str):
    chunk_size = 1024  # Set an appropriate chunk size
    for i in range(0, len(data), chunk_size):
        await websocket.send_json({"filename": filename, "data_chunk": data[i:i + chunk_size]})
        await asyncio.sleep(0.1)
    await websocket.send_text("FinishedThisAttachment")


# async def extract_text_from_pdf(pdf_data):
#     with io.BytesIO(base64.b64decode(pdf_data)) as pdf_file:
#         pdf_reader = PyPDF2.PdfFileReader(pdf_file)
#         text = ""
#         for page_num in range(pdf_reader.numPages):
#             page = pdf_reader.getPage(page_num)
#             text += page.extract_text()
#         return text

# async def extract_text_from_docx(docx_data):
#     doc = Document(io.BytesIO(base64.b64decode(docx_data)))
#     text = ""
#     for para in doc.paragraphs:
#         text += para.text + "\n"
#     return text

# async def extract_text_from_attachment(filename, data):
#     if filename.endswith('.pdf'):
#         return await extract_text_from_pdf(data)
#     elif filename.endswith('.docx'):
#         return await extract_text_from_docx(data)
#     else:
#         # Add handling for other document types if needed
#         return "Unsupported document type"



@app.websocket("/ws")
async def test_websocket(websocket: WebSocket):
    #This code is basically the authorization code and this authorization code helps us to get the access token with the required scopes that we have set .
    #We require the gmail.readonly scopes that requires verification of our application and all.
    # raw_body = await request.body()
    # return {"data":Yo Yo"}
    # data = await request.json()
    # code = data.get("access_token")
    await websocket.accept()
    # await print(sucess)
    # await print("Hi hi working")
    logger.info("Hi hi succefull in connecting !!")
    # await websocket.send_json({"message":"Yes Websockets successfull"})
    # await print("working after line 163")
    logger.info("Now receiving json!!")

    data = await websocket.receive_text()
    logger.info("Received JSON data: %s", data)
    # print(data)
    # code = data.get("message")
    # print(code)
    # code = raw_body.decode()
    # sent=event_generator(data)
    for i in range(1, 11):
        logging.info(f"printing value {i}")
    
    def get_messages(code:str):
        print()
        # print(code)
        logging.info("entered into the get_messages")
        access_token = code
        print("printing access_token")
        print(access_token)
        page_token = None
        messages = []
        jobs_query = "subject:receipt OR subject:receipts has:attachment"

        while True:
            # Construct Gmail API request with pageToken
            print("into the gmail")
            gmail_url = f"https://www.googleapis.com/gmail/v1/users/me/messages?q={jobs_query}"

            if page_token:
                gmail_url += f"&pageToken={page_token}"

            gmail_response = requests.get(gmail_url, headers={"Authorization": f"Bearer {access_token}"})
            logging.info(f"{gmail_response}")
            print(gmail_response)
            gmail_data = gmail_response.json()
            
            # Check if there are messages in the response
            if "messages" in gmail_data:
                messages.extend(gmail_data["messages"])

            # Check if there are more pages
            if "nextPageToken" in gmail_data:
                page_token = gmail_data["nextPageToken"]
            else:
                break  # No more pages, exit the loop
        print("returning the messages")

        unique_thread_ids = set()

        filtered_data_list = []

        for entry in messages:
            thread_id = entry['threadId']
            if thread_id not in unique_thread_ids:
                unique_thread_ids.add(thread_id)
                filtered_data_list.append(entry)


        print(messages)
        
        print(filtered_data_list)
        logging.info(f"{filtered_data_list}")

        return filtered_data_list
    
    async def event_generator(code:str):
        
        logging.info("entered into the event_generator")

        print(code)
        access_token = code
        messages=get_messages(access_token)
        print(len(messages))
        # await websocket.send_json({"message 1":0})
        # time.sleep(1)
        # await websocket.send_text("message 2")
        # time.sleep(1)

        attachments = []
        prev_data=""
        data_new={}
        attachment_no=0
        batch_size = 5
        prev_filename = None

        for i,message in enumerate(messages) :
            print(i)
            logging.info(f"{i}")
            logging.info(f"{message}")

            print(message)

            if message:
                message_id = message.get("id")
                thread_id = message.get("threadId")
                print(message_id)
                if message_id:
                    message_url = f"https://www.googleapis.com/gmail/v1/users/me/messages/{message_id}"
                    message_response = requests.get(message_url, headers={"Authorization": f"Bearer {access_token}"})
                    message_data = message_response.json()

                    
                    # Check for parts in the message payload
                    if "payload" in message_data and "parts" in message_data["payload"]:
                        for part in message_data["payload"]["parts"]:
                            if "body" in part and "attachmentId" in part["body"]:
                                attachment_id = part["body"]["attachmentId"]
                                print(attachment_id)
                                attachment_url = f"https://www.googleapis.com/gmail/v1/users/me/messages/{message_id}/attachments/{attachment_id}"
                                attachment_response = requests.get(attachment_url, headers={"Authorization": f"Bearer {access_token}"})
                                attachment_data = attachment_response.json()
                                data = attachment_data.get("data",{})
                                filename = part.get("filename", "untitled.txt")

                                # print("Print the data json response for that gmail")
                                print(filename)
                                
                                
                                # print(attachment_data)
                                # json_str = json.dumps(attachment_data, indent=2)

                                # with subprocess.Popen(["less"], stdin=subprocess.PIPE) as less_process:
                                #     less_process.communicate(input=json_str.encode("utf-8"))
                                if data:
                                    data_new[filename]=str(data[:10])
                                    attachment_content = base64.urlsafe_b64decode(data.encode("UTF-8"))
                                    # await websocket.send_json({"filename": filename})
                                    # await websocket.send_bytes(attachment_content)
                                    # extracted_text = await extract_text_from_attachment(filename, data)
                                    await send_chunked_data(websocket, filename, data)

                                    attachment_no+=1
                                    # time.sleep(2)
                                    # await websocket.send_json({filename:data})

                                    # yield f"data: {str(data_new)}\n\n"
                                    
                                    # data_new={}


                                



        await websocket.send_text("CompletedFetchingMessages")

    await event_generator(data)
    logging.info("Closing connection")
    await websocket.close()
    
@app.get("/token")
async def get_token(token: str = Depends(oauth2_scheme)):
    return jwt.decode(token, GOOGLE_CLIENT_SECRET, algorithms=["HS256"])