Synced repo using 'sync_with_huggingface' Github Action

main.py

@@ -1,20 +1,15 @@
-from pydantic import BaseModel
-from typing import List, Optional
-from fastapi.responses import JSONResponse
-from starlette.middleware import Middleware
+from fastapi import FastAPI
+from src.config.appConfig import ENV_VAR
 from starlette.middleware.gzip import GZipMiddleware
 from starlette.middleware.cors import CORSMiddleware
-from fastapi import FastAPI, File, UploadFile, Depends, Form, BackgroundTasks
 
-from OtherFun import * 
-from Middleware import Main
-from verifyToken import verify_token_and_role
+from src.main import main
+
 # import os
-origins = ["https://chatbotservernode.onrender.com","https://cbns.vercel.app", "https://hfhchatbot.vercel.app", "http://localhost:5000", "http://localhost:3000", "https://localhost:5000"]
+origins = ["http://localhost:3000"]
 
 # origins = os.getenv("ALLOWED_ORIGINS", "").split(",")
-
-app = FastAPI(debug=True)
+app = FastAPI(debug=ENV_VAR.DEBUG)
 
 app.add_middleware(GZipMiddleware)
 app.add_middleware(
@@ -22,65 +17,12 @@ app.add_middleware(
     allow_origins=origins,  # You can replace '*' with specific origins
     allow_credentials=True,
     allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],  # or specific methods
-    allow_headers=["Authorization", "Content-Type", "Accept"]  # or specific headers
+    allow_headers=["Authorization", "Content-Type", "Accept"],  # or specific headers
 )
 
-model = Main()
-
-class BodyModel(BaseModel):
-    query: str
-    chain_name: Optional[str] = None  # Made chain_name optional
-    
-@app.get("/")
-async def home():
-    return "chatbot api server is running..."
-
-
-@app.post("/ask")
-async def askQ(body: BodyModel, token: str = Depends(verify_token_and_role)):
-    try:
-        response = model.ask_question(body.query, token["username"] if body.chain_name is None else body.chain_name)
-        return JSONResponse(content={"success": True, "data": response})
-    except Exception as e:  # Catch specific exceptions
-        return JSONResponse(content={"success": False, "error": str(e)})
-
-
-@app.post("/create/embedding")
-async def createEmbedding(collection_name: str = Form(...), files: List[UploadFile] = File(None), token: str = Depends(verify_token_and_role)):
-    try:
-        if not files:
-            return JSONResponse(content={"success": False, "error":"No files provided"})
-        
-        responses = []
-        for file in files:
-            response = await process_file(model, collection_name, file)
-            responses.append(response)
-
-        return JSONResponse(content={"success": True,"responses": responses})
-    except Exception as e:
-        return JSONResponse(content={"success": False, "error": str(e)})
-
-
-@app.post("/create/tmp/chain")
-async def createTmpChain(background_tasks: BackgroundTasks, files: List[UploadFile] = File(...), token: str = Depends(verify_token_and_role)):
-    try:
-        if not files:
-            return JSONResponse(content={"success": False, "error":"No files provided"})
-                                
-        all_contents = b""
-        for file in files:
-            contents = await file.read()
-            all_contents += contents
+app.include_router(main)
 
-        file_extension = files[0].filename.split(".")[-1]
-        if file_extension == "pdf":
-            chain_name = token["username"]
-            model.generate_tmp_embedding_and_chain(all_contents, chain_name)
-            background_tasks.add_task(delete_chain_after_delay(model, chain_name))
-            return JSONResponse(content={"success": True, "message": "Chain created. Will be deleted after 2 hours."})
-        elif file_extension == "txt":
-            all_contents.decode("utf-8")
-            return JSONResponse(content={"success": False, "error": "Unsupported file format"})
-    except Exception as e:
-        return JSONResponse(content={"success": False, "error": str(e)})
+import uvicorn
 
+if __name__ == "__main__":
+    uvicorn.run("main:app", host="0.0.0.0", port=5000, reload=ENV_VAR.DEBUG)

requirements.txt

@@ -1,5 +1,7 @@
 langchain==0.1.8
 python-dotenv==1.0.0
+huggingface_hub
+typing
 
 # pdf
 PyPDF2==3.0.1
@@ -15,13 +17,16 @@ faiss-cpu
 # mongodb
 pymongo==4.6.1
 
+cloudinary
+
 # API-END point
 fastapi==0.109.2
 fastapi-cors
 uvicorn[standard]==0.17.*
 python-multipart==0.0.9
 PyJWT==2.8.0
+pydantic
 
 
-huggingface_hub
-typing
+
+# requests-toolbelt

src/bot/MongoChainGenerator.py

@@ -0,0 +1,80 @@
+from src.config.appConfig import ENV_VAR, CONST_VAR, LOG
+from langchain.chains import RetrievalQA
+from langchain.prompts import PromptTemplate
+from langchain.llms.huggingface_endpoint import HuggingFaceEndpoint
+from langchain.vectorstores.mongodb_atlas import MongoDBAtlasVectorSearch
+from langchain.vectorstores.faiss import FAISS
+from huggingface_hub import login
+
+login(
+    token=ENV_VAR.HUGGINGFACEHUB_API_TOKEN, 
+    write_permission=True,
+    add_to_git_credential=True,
+)
+
+
+class MongoChainGenerator:
+    LLM = None
+
+    def __init__(
+        self,
+        embedding_model,
+        template_context,
+        db_collection_name=None,
+        tmp_vector_embedding=None,
+    ):
+        if db_collection_name:
+            self._load_vectors(embedding_model, db_collection_name)
+        else:
+            self._create_tmp_retriever(tmp_vector_embedding)
+
+        self._initialize_prompt(template_context)
+
+        if MongoChainGenerator.LLM is None:
+            self._initialize_llm()
+
+    def _create_tmp_retriever(self, tmp_vector_embedding: FAISS):
+        self.qa_retriever = tmp_vector_embedding.as_retriever(
+            search_type="similarity", search_kwargs={"k": 7}
+        )
+        LOG.debug("Temporary retriever created")
+
+    def _load_vectors(self, embedding_model, db_collection_name):
+        self.qa_retriever = MongoDBAtlasVectorSearch.from_connection_string(
+            connection_string=ENV_VAR.MONGO_DB_URL,
+            namespace=ENV_VAR.MONGO_DB_NAME + "." + db_collection_name,
+            embedding=embedding_model,
+        ).as_retriever(search_type="similarity", search_kwargs={"k": 7})
+        LOG.debug("Retriever loaded from MongoDB Atlas")
+
+    def _initialize_prompt(self, template_context):
+        template = (
+            template_context
+            + """
+        {context}
+        
+        Question: {question} all related details.
+        Answer:"""
+        )
+        self.prompt = PromptTemplate(
+            template=template, input_variables=["context", "question"]
+        )
+        LOG.debug("Prompt template initialized")
+
+    def _initialize_llm(self):
+        MongoChainGenerator.LLM = HuggingFaceEndpoint(
+            repo_id=CONST_VAR.TEXT_GENERATOR_MODEL_REPO_ID,
+            temperature=0.8,
+            max_new_tokens=4096,
+        )
+        # MongoChainGenerator.LLM = HuggingFaceHub(repo_id=CONST_VAR.TEXT_GENERATOR_MODEL_REPO_ID, model_kwargs={"temperature": 0.85, "return_full_text": False, "max_length": 4096, "max_new_tokens": 4096})
+        LOG.info("LLM initialized")
+
+    def generate_retrieval_qa_chain(self):
+        chain = RetrievalQA.from_chain_type(
+            llm=MongoChainGenerator.LLM,
+            retriever=self.qa_retriever,
+            chain_type_kwargs={"prompt": self.prompt},
+        )
+        LOG.debug("Retrieval QA chain generated")
+        return chain

src/bot/MongoEmbeddingGenerator.py

@@ -0,0 +1,35 @@
+from io import BytesIO
+import PyPDF2
+from src.config.appConfig import *
+from src.config.databaseConfig import DATABASE
+
+from langchain.vectorstores.faiss import FAISS
+from langchain.vectorstores.mongodb_atlas import MongoDBAtlasVectorSearch
+from langchain.embeddings.huggingface_hub import HuggingFaceHubEmbeddings
+
+class MongoEmbeddingGenerator:
+
+    def __init__(self, repo_id):
+        self.embedding_model = HuggingFaceHubEmbeddings(repo_id=repo_id, huggingfacehub_api_token=ENV_VAR.HUGGINGFACEHUB_API_TOKEN)
+        LOG.info("Embedding model initialised")
+
+    def _extract_text_from_pdf(self, pdf_bytes):
+        pdf_file = BytesIO(pdf_bytes)
+        pdf_reader = PyPDF2.PdfReader(pdf_file)
+        return [pdf_reader.pages[page_num].extract_text() for page_num in range(len(pdf_reader.pages))]
+
+    def generate_tmp_embeddings(self, pdf_bytes):
+        texts = self._extract_text_from_pdf(pdf_bytes)
+        return FAISS.from_texts(texts=texts, embedding=self.embedding_model)
+
+    def generate_embeddings(self, pdf_bytes, file_name: str, collection_name: str):
+        client = DATABASE.client
+        if client[ENV_VAR.MONGO_DB_NAME_CACHE][collection_name].find_one({"src_file_name": file_name}):
+            LOG.debug(f"Vectors already exist in MongoDB for file {file_name}")
+            return f"Vectors already exist in MongoDB for file {file_name}"
+        else:
+            texts = self._extract_text_from_pdf(pdf_bytes)
+            client[ENV_VAR.MONGO_DB_NAME_CACHE][collection_name].insert_one({"src_file_name": file_name})
+            MongoDBAtlasVectorSearch.from_texts(texts=texts, embedding=self.embedding_model, collection=client[ENV_VAR.MONGO_DB_NAME][collection_name])
+            LOG.debug(f"Vectors stored in MongoDB for file {file_name}")
+            return f"Vectors stored in MongoDB for file {file_name}"

src/bot/OtherFun.py

@@ -0,0 +1,38 @@
+from fastapi import UploadFile
+import asyncio
+from src.bot.main import Main
+from src.config.appConfig import LOG
+
+
+def delete_chain_after_delay(model: Main, chain_name: str):
+    async def delete_chain():
+        try:
+            await asyncio.sleep(7200)  # Sleep for 2 hours
+            if chain_name in model.qa_chains:
+                del model.qa_chains[chain_name]
+                # Log deletion
+                LOG.info(f"Chain '{chain_name}' deleted after 2 hours")
+        except Exception as e:
+            LOG.error(f"An error occurred while deleting chain '{chain_name}': {e}")
+
+    return delete_chain
+
+
+async def process_file(model: Main, collection_name: str, file: UploadFile):
+    try:
+        contents = await file.read()
+
+        file_extension = file.filename.split(".")[-1]
+
+        if file_extension == "pdf":
+            response = model.generate_embedding(
+                contents, file.filename, collection_name)
+        elif file_extension == "txt":
+            response = contents.decode("utf-8")
+        else:
+            raise ValueError(f"Unsupported file format for {file.filename}")
+
+        return response
+    except Exception as e:
+        LOG.error(f"An error occurred while processing file '{file.filename}': {e}")
+        return f"Error processing file '{file.filename}': {e}"

src/bot/main.py

@@ -0,0 +1,66 @@
+from src.bot.MongoChainGenerator import *
+from src.bot.MongoEmbeddingGenerator import *
+
+from src.config.appConfig import LOG
+from src.config.databaseConfig import DATABASE
+
+
+class Main:
+    qa_chains = {}
+    embedding_generator = None
+
+    def __init__(self) -> None:
+        DATABASE()
+        self._initialize_embedding_generator()
+        self._load_existing_qa_chains()
+
+    def _initialize_embedding_generator(self):
+        if Main.embedding_generator is None:
+            Main.embedding_generator = MongoEmbeddingGenerator(repo_id=CONST_VAR.EMBEDDING_MODEL_REPO_ID)
+            LOG.debug("Embedding generator initialized")
+
+    def _load_existing_qa_chains(self):
+        chats = DATABASE.client["chatData"]["chats"].find()
+        for chat in chats:
+            if chat["collectionName"] not in Main.qa_chains:
+                self.create_exist_chains(chat)
+
+    def create_exist_chains(self, chat):
+        if chat["collectionName"] not in Main.qa_chains:
+            qa_generator = MongoChainGenerator(
+                embedding_model=Main.embedding_generator.embedding_model,
+                db_collection_name=chat["collectionName"],
+                template_context=chat["templateContext"]
+            )
+            Main.qa_chains[chat["collectionName"]] = qa_generator.generate_retrieval_qa_chain()
+            LOG.debug("Chain created for collection " + chat["collectionName"])
+        else:
+            LOG.debug("Chain already exists for collection " + chat["collectionName"])
+
+    def generate_embedding(self, content: str, file_name: str, collection_name: str):
+        return Main.embedding_generator.generate_embeddings(content, file_name, collection_name)
+
+    def generate_tmp_embedding_and_chain(self, contents: str, tmp_collection_name):
+        qa_generator = MongoChainGenerator(
+            embedding_model=Main.embedding_generator.embedding_model,
+            template_context=CONST_VAR.TEMPLATE_CONTEXT,
+            tmp_vector_embedding=Main.embedding_generator.generate_tmp_embeddings(pdf_bytes=contents)
+        )
+        Main.qa_chains[tmp_collection_name] = qa_generator.generate_retrieval_qa_chain()
+        LOG.debug(tmp_collection_name + ' chain created')
+
+    def ask_question(self, question: str, collection_name):
+        if collection_name in Main.qa_chains:
+            try:
+                LOG.debug(collection_name + " answering")
+                response = Main.qa_chains[collection_name]({"query": question, "early_stopping": True, "min_length": 2000, "max_tokens": 5000})
+                return response["result"]
+            except Exception as e:
+                LOG.error("An error occurred while answering question: {}".format(str(e)))
+                return "Retry to ask question! An error occurred: {}".format(str(e))
+        else:
+            LOG.warning("Chain for collection '{}' not found.".format(collection_name))
+            return "Chain for collection '{}' not found.".format(collection_name)
+
+    def check_collection_name(self, collection_name):
+        return collection_name in self.qa_chains

src/config/appConfig.py

@@ -0,0 +1,72 @@
+import os
+import logging
+from dotenv import load_dotenv
+
+load_dotenv()
+
+class CLOUDINARY():
+    CLOUDINARY_CLOUD_NAME = os.environ.get("CLOUDINARY_CLOUD_NAME")
+    CLOUDINARY_API_KEY = os.environ.get("CLOUDINARY_API_KEY")
+    CLOUDINARY_API_SECRET = os.environ.get("CLOUDINARY_API_SECRET")
+
+class ENV_VAR():
+    MONGO_DB_URL = os.environ.get("MONGO_DB_URL")
+    MONGO_DB_NAME = os.environ.get("MONGO_DB_NAME")
+    MONGO_DB_NAME_CHATS = os.environ.get("MONGO_DB_NAME_CHATS")
+    HUGGINGFACEHUB_API_TOKEN = os.environ.get("HUGGINGFACEHUB_API_TOKEN")
+    MONGO_DB_NAME_CACHE = os.environ.get("MONGO_DB_NAME_CACHE")
+    JWT_SECRET = os.environ.get("JWT_SECRET")
+    AUTH_API_END = os.environ.get("AUTH_API_END")
+    DEBUG = (
+        os.environ.get("DEBUG").lower() == "true" if os.environ.get("DEBUG") else False
+    )
+
+class CONST_VAR():
+    TEXT_GENERATOR_MODEL_REPO_ID = "mistralai/Mixtral-8x7B-Instruct-v0.1"
+    EMBEDDING_MODEL_REPO_ID = "sentence-transformers/all-MiniLM-L6-v2"
+    TEMPLATE_CONTEXT = """
+        Use the following pieces of context to answer the question at the end.
+        You should prefer information which are more related to asked question.
+        Make sure to rely on information from text only and not on questions to provide accurate responses.
+        When you find particular answer in given text, display its context useful, make sure to cite it in the your answer.
+        If you don't know the answer, just say that you don't know, don't try to make up an answer.
+        You can only use the given to you to answer the question.
+        Generate concise answers and relevant data related to the asked question.
+        You must represent the answer in proper format such as make points highlight some major information.
+        don't attach your created quetions. if you don't get answer from the given text just say i don't know and terminate answering.
+        if you get answer from the text than write all about the asked quetion and relevant data related to it.    
+        don't use your own knowledge just use the provided text to answer the question.  
+    """
+
+
+class LOG:
+    def __init__(self) -> None:
+        pass
+    
+    @staticmethod
+    def configure_logging(level=logging.INFO):
+        logging.basicConfig(level=level)  # Set the logging level
+        
+    @staticmethod
+    def debug(msg):
+        logging.debug(msg)
+        
+    @staticmethod
+    def info(msg):
+        logging.info(msg)
+        
+    @staticmethod
+    def warning(msg):
+        logging.warning(msg)
+        
+    @staticmethod
+    def error(msg):
+        logging.error(msg)
+        
+    @staticmethod
+    def critical(msg):
+        logging.critical(msg)
+
+
+if ENV_VAR.DEBUG:
+    LOG.configure_logging(logging.DEBUG)

src/config/cloudinaryConfig.py

@@ -0,0 +1,8 @@
+import cloudinary
+from src.config.appConfig import CLOUDINARY
+
+cloudinary.config(
+    cloud_name=CLOUDINARY.CLOUDINARY_CLOUD_NAME,
+    api_key=CLOUDINARY.CLOUDINARY_API_KEY,
+    api_secret=CLOUDINARY.CLOUDINARY_API_SECRET,
+)

src/config/databaseConfig.py

@@ -0,0 +1,14 @@
+from src.config.appConfig import *
+from pymongo import MongoClient
+
+
+class DATABASE:
+    client = None
+    collections=None
+    def __init__(self):
+        self._initialize_mongodb_client()
+
+    def _initialize_mongodb_client(self):
+        if DATABASE.client is None:
+            DATABASE.client = MongoClient(ENV_VAR.MONGO_DB_URL)
+            print("mongodb connected")

src/controllers/auth/auth_controller.py

@@ -0,0 +1,63 @@
+from fastapi import APIRouter, Response, Request, UploadFile
+from src.config.appConfig import ENV_VAR
+from src.helpers.response import ResponseHandler
+from src.helpers.send_request import sendRequest
+from typing import Optional
+
+router = APIRouter()
+AUTH_API_END = ENV_VAR.AUTH_API_END
+
+
+@router.post("/register/")
+async def register(req: Request, picPath: UploadFile = None):
+    try:
+        form_data = await req.form()
+
+        payload = {key: form_data[key] for key in form_data if key != "picPath"}
+
+        files = None
+
+        if picPath:
+            files = {"picPath": (picPath.filename, picPath.file, picPath.content_type)}
+
+        # else    f"{AUTH_API_END}/api/v1/auth/register/", json=payload}
+        response = sendRequest(
+            f"{AUTH_API_END}/api/v1/auth/register/", "post", payload, files
+        )
+
+        return ResponseHandler.success_mediator(response)
+    except Exception as e:
+        return ResponseHandler.error(9999, e)
+
+
+@router.get("/get/usernames")
+async def get_user_names():
+    try:
+        response = sendRequest(f"{AUTH_API_END}/api/v1/auth/get/usernames")
+        return ResponseHandler.success_mediator(response)
+    except Exception as e:
+        return ResponseHandler.error(9999, e)
+
+
+@router.post("/login/")
+async def login_control(req: dict):
+    try:
+        response = sendRequest(f"{AUTH_API_END}/api/v1/auth/login/", "post", req)
+        return ResponseHandler.success_mediator(response)
+    except Exception as e:
+        return ResponseHandler.error(9999, e)
+
+
+@router.post("/change/password/")
+async def change_pass_control(req: dict, authorization: str = None):
+    try:
+        headers = {"Content-Type": "application/json"}
+        if authorization:
+            headers["Authorization"] = authorization
+
+        response = sendRequest(
+            f"{AUTH_API_END}/api/v1/auth/change/password", "post", req, None, headers
+        )
+        return ResponseHandler.success_mediator(response)
+    except Exception as e:
+        return ResponseHandler.error(9999, e)

src/controllers/auth/otp_controller.py

@@ -0,0 +1,26 @@
+from fastapi import APIRouter, Request
+from src.config.appConfig import ENV_VAR
+from src.helpers.response import ResponseHandler
+from src.helpers.send_request import sendRequest
+
+router = APIRouter()
+AUTH_API_END = ENV_VAR.AUTH_API_END
+
+
+@router.post("/send-otp")
+async def send_otp_controller(req: Request):
+    try:
+        # Make the request to the authentication API endpoint
+        response = sendRequest(
+            f"{AUTH_API_END}/api/v1/mail/send-otp",
+            "post",
+            req,
+            None,
+            {"Content-Type": "application/json"},
+        )
+        # Handle the response using the ResponseHandler
+        return ResponseHandler.success_mediator(response)
+    except Exception as e:
+        # Log and handle any exceptions
+        print(e)
+        return ResponseHandler.error(9999, e)

src/controllers/auth/user_controller.py

@@ -0,0 +1,49 @@
+from fastapi import APIRouter, Request, UploadFile, Header
+from src.config.appConfig import ENV_VAR
+from src.helpers.response import ResponseHandler
+from src.helpers.send_request import sendRequest
+from typing import Optional
+
+router = APIRouter()
+AUTH_API_END = ENV_VAR.AUTH_API_END
+
+
+@router.get("/userid/{uid}")
+async def get_users(uid: str):
+    try:
+        response = get_user_date(uid)
+        return ResponseHandler.success_mediator(response)
+    except Exception as error:
+        return ResponseHandler.error(error)
+
+def get_user_date(uid):
+    return sendRequest(
+        url=f"{AUTH_API_END}/api/v1/user/get/userid/{uid}",
+        headers={"Content-Type": "application/json"},
+    )
+
+@router.put("/update/")
+async def update_user_data(
+    req: Request, picPath: UploadFile = None, authorization: str = Header(None)
+):
+    try:
+        form_data = await req.form()
+
+        payload = {key: form_data[key] for key in form_data if key != "picPath"}
+
+        files = None
+
+        if picPath:
+            files = {"picPath": (picPath.filename, picPath.file, picPath.content_type)}
+
+        headers = {"Authorization": authorization}
+        response = sendRequest(
+            f"{AUTH_API_END}/api/v1/user/update/",
+            "put",
+            payload,
+            files,
+            headers,
+        )
+        return ResponseHandler.success_mediator(response)
+    except Exception as e:
+        return ResponseHandler.error(9999, e)

src/controllers/chat/bot_controller.py

@@ -0,0 +1,92 @@
+from fastapi import BackgroundTasks, APIRouter, Request, UploadFile, Depends, File
+from typing import List
+
+from src.helpers.response import ResponseHandler
+from src.helpers.json_convertor import convert_to_json
+
+from src.middleware.verifyToken import verify_token
+from src.services.message_services import save_question_and_answer_to_chat_history
+
+from src.bot.main import Main
+from src.bot.OtherFun import delete_chain_after_delay, process_file
+
+router = APIRouter()
+
+model = Main()
+
+
+@router.post("/ask-question")
+async def askQ(req: Request, token: str = Depends(verify_token)):
+    try:
+        form_data = await req.json()
+        answer = model.ask_question(
+            form_data["query"],
+            (
+                token["username"]
+                if form_data["collectionName"] is None
+                else form_data["collectionName"]
+            ),
+        )
+        await save_question_and_answer_to_chat_history(
+            token["username"],
+            {
+                "question": form_data["query"],
+                "answer": answer,
+                "collectionName": (
+                    form_data["query"] if form_data["query"] else "CHAT WITH YOUR PDF"
+                ),
+            },
+        )
+        return ResponseHandler.success(2001, answer)
+    except Exception as error:
+        print(error)
+        return ResponseHandler.error(9999, error, 500)
+
+
+@router.post("/create/embedding/{collection_name}")
+async def createEmbedding(
+    collection_name: str,
+    files: List[UploadFile] = File(None),
+    tokenData: str = Depends(verify_token),
+):
+    try:
+        if not files:
+            return ResponseHandler.error(2003)
+
+        responses = []
+        for file in files:
+            response = process_file(model, collection_name, file)
+            responses.append(response)
+
+        return ResponseHandler.success(2002, response)
+    except Exception as error:
+        return ResponseHandler.error(9999, error, 500)
+
+
+@router.post("/create/tmp/chain")
+async def createTmpChain(
+    background_tasks: BackgroundTasks,
+    files: List[UploadFile] = File(...),
+    tokenData: str = Depends(verify_token),
+):
+    try:
+        if not files:
+            return ResponseHandler.error(2003, error, 500)
+
+        all_contents = b""
+        for file in files:
+            contents = await file.read()
+            all_contents += contents
+
+        file_extension = files[0].filename.split(".")[-1]
+        if file_extension == "pdf":
+            chain_name = tokenData["username"]
+            model.generate_tmp_embedding_and_chain(all_contents, chain_name)
+            background_tasks.add_task(delete_chain_after_delay(model, chain_name))
+            return ResponseHandler.success(2001)
+        elif file_extension == "txt":
+            all_contents.decode("utf-8")
+            return ResponseHandler.error(2004, error, 500)
+
+    except Exception as error:
+        return ResponseHandler.error(9999, error, 500)

src/controllers/chat/chats_controller.py

@@ -0,0 +1,191 @@
+from fastapi import APIRouter, Depends, Request, UploadFile
+
+from src.config.databaseConfig import DATABASE
+from src.config.appConfig import ENV_VAR
+
+from src.helpers.pagination import *
+from src.helpers.response import ResponseHandler
+from src.helpers.upload_file_cloudinary import upload_file
+
+from src.middleware.verifyToken import verify_token
+
+router = APIRouter()
+
+# Your route definitions
+
+
+from typing import Dict, List
+
+
+def convert_form_data_to_dict(form_data) -> Dict[str, List[str]]:
+    data_dict = {}
+
+    for key, value in form_data.items():
+        if isinstance(value, str):
+            if key in data_dict:
+                if isinstance(data_dict[key], list):
+                    data_dict[key].append(value)
+                else:
+                    data_dict[key] = [data_dict[key], value]
+            else:
+                data_dict[key] = value
+        elif isinstance(value, UploadFile):
+            # Handle UploadFile separately if needed
+            pass
+        else:
+            # Handle other types of values if needed
+            pass
+
+    # Convert keys with multiple values to lists
+    for key in data_dict:
+        if isinstance(data_dict[key], list) and len(data_dict[key]) > 1:
+            continue  # Skip if it's already a list with multiple values
+        elif key in form_data.getlist():
+            data_dict[key] = form_data.getlist(key)
+
+    return data_dict
+
+
+@router.post("/create")
+async def create_chat(
+    req: Request, icon: UploadFile = None, tokenData=Depends(verify_token)
+):
+    try:
+        username = tokenData["username"]
+        form_data = await req.form()
+        # payload = {key: form_data[key] for key in form_data if key != "picPath"}
+        # dt = {}
+        # print(convert_form_data_to_dict(form_data))
+
+        if icon is None:
+            return ResponseHandler.error(3007, None, 400)
+
+        file_data = await upload_file(
+            icon, form_data["collectionName"] + "_icon", "ChatIcons/"
+        )
+
+        icon_public_id = file_data["public_id"]
+        # print(icon_public_id)
+        # print(form_data["title"])
+        # print(form_data["templateContext"])
+        # print(form_data["collectionName"])
+        # print(form_data["sampleQuetions"])
+        # print(form_data)
+        # print(ENV_VAR.MONGO_DB_NAME_CHATS)
+
+        chat = DATABASE.client[ENV_VAR.MONGO_DB_NAME_CHATS]["chats"].insert_one(
+            {
+                "username": username,
+                "title": form_data["title"],
+                "templateContext": form_data["templateContext"],
+                "collectionName": form_data["collectionName"],
+                "sampleQuetions": (
+                    (form_data["sampleQuetions"])
+                    if form_data["sampleQuetions"] is not None
+                    else []
+                ),
+                "buttonIcon": icon_public_id,
+            }
+        )
+
+        return ResponseHandler.success(3000, chat.acknowledged)
+    except Exception as error:
+        print(error)
+        return ResponseHandler.error(9999, error, 500)
+
+
+@router.get("/get")
+async def get_paginated_chats(req: Request):
+    try:
+        page = int(req.query_params.get("page", 1))
+        limit = int(req.query_params.get("limit", 10))
+
+        total_count = DATABASE.client[ENV_VAR.MONGO_DB_NAME_CHATS][
+            "chats"
+        ].estimated_document_count()
+
+        chats = (
+            DATABASE.client[ENV_VAR.MONGO_DB_NAME_CHATS]["chats"]
+            .find()
+            .skip((page - 1) * limit)
+            .limit(limit)
+        )
+
+        paginated_response = get_paginated_response(
+            list(chats), page, limit, total_count
+        )
+        # print(paginated_response)
+        return ResponseHandler.success(3001, paginated_response)
+    except Exception as error:
+        return ResponseHandler.error(9000, 500, error)
+
+
+@router.get("/get/{collection_name}")
+async def get_chat_by_collection_name(req: Request, collection_name: str):
+    try:
+        chat = DATABASE.client[ENV_VAR.MONGO_DB_NAME_CHATS]["chats"].find_one(
+            {"collectionName": collection_name}
+        )
+        if not chat:
+            return ResponseHandler.error(3003, 404)
+        return ResponseHandler.success(3002, chat)
+    except Exception as error:
+        return ResponseHandler.error(9000, 500, error)
+
+
+@router.put("/edit/{collection_name}")
+async def update_chat(
+    req: Request,
+    collection_name: str,
+    icon: UploadFile = None,
+    tokenData=Depends(verify_token),
+):
+    try:
+        username = tokenData["username"]
+        form_data = await req.form()
+        chat = DATABASE.client[ENV_VAR.MONGO_DB_NAME_CHATS]["chats"].find_one(
+            {"collectionName": collection_name, "username": username}
+        )
+
+        if not chat:
+            return ResponseHandler.error(3003, 404)
+
+        icon_public_id = None
+        if icon is not None:
+            file_data = await upload_file(
+                icon, form_data["collectionName"] + "_icon", "ChatIcons/"
+            )
+            icon_public_id = file_data["public_id"]
+
+        updated_chat = DATABASE.client[ENV_VAR.MONGO_DB_NAME_CHATS][
+            "chats"
+        ].find_one_and_update(
+            {"collectionName": collection_name},
+            {
+                "$set": {
+                    "title": form_data["title"],
+                    "templateContext": form_data["templateContext"],
+                    "buttonIcon": icon_public_id or chat.get("buttonIcon"),
+                }
+            },
+        )
+
+        if not updated_chat:
+            return ResponseHandler.error(3003, 404)
+        return ResponseHandler.success(3004)
+    except Exception as error:
+        return ResponseHandler.error(9000, 500, error)
+
+
+@router.delete("/delete/{collection_name}")
+async def delete_chat(collection_name: str, tokenData=Depends(verify_token)):
+    try:
+        username = tokenData["username"]
+        deleted_chat = DATABASE.client[ENV_VAR.MONGO_DB_NAME_CHATS][
+            "chats"
+        ].find_one_and_delete({"collectionName": collection_name, "username": username})
+        if not deleted_chat:
+            return ResponseHandler.error(3003, 404)
+        return ResponseHandler.success(3005)
+    except Exception as error:
+        return ResponseHandler.error(9000, 500, error)

src/controllers/chat/history_controller.py

@@ -0,0 +1,85 @@
+from typing import Optional
+from fastapi import APIRouter, Depends
+from bson import ObjectId
+
+from src.config.appConfig import ENV_VAR
+from src.config.databaseConfig import DATABASE
+
+from src.helpers.response import ResponseHandler
+from src.helpers.pagination import get_paginated_response
+
+from src.middleware.verifyToken import verify_token
+
+router = APIRouter()
+
+
+@router.delete("/delete")
+async def delete_chat_history(tokenData: str = Depends(verify_token)):
+    try:
+        username = tokenData["username"]
+        deleted_chat_history = DATABASE.client[ENV_VAR.MONGO_DB_NAME_CHATS][
+            "chathistories"
+        ].delete_one({"username": username})
+
+        if deleted_chat_history.deleted_count == 0:
+            return ResponseHandler.error(4002, 404)
+        else:
+            return ResponseHandler.success(4004)
+    except Exception as error:
+        print("Error deleting chat history:", error)
+        return ResponseHandler.error(9000, 500, error)
+
+
+@router.delete("/delete/question/{questionId}")
+async def delete_question_from_history(
+    questionId: str, tokenData: str = Depends(verify_token)
+):
+    try:
+        username = tokenData["username"]
+        update_query = {
+            "$pull": {"history": {"_id": ObjectId(questionId)}},
+            "$inc": {"historyCount": -1},
+        }
+
+        result = DATABASE.client[ENV_VAR.MONGO_DB_NAME_CHATS][
+            "chathistories"
+        ].update_one(
+            {"username": username, "history._id": ObjectId(questionId)}, update_query
+        )
+        print(result)
+        if result.matched_count != 0:
+            return ResponseHandler.success(4005)
+        else:
+            return ResponseHandler.error(4006, 404)
+    except Exception as error:
+        print("error:", error)
+        return ResponseHandler.error(9000, 500, error)
+
+
+@router.get("/get")
+async def get_chat_history_by_user_id(
+    page: Optional[int] = 1,
+    limit: Optional[int] = 10,
+    tokenData: str = Depends(verify_token),
+):
+    try:
+        username = tokenData["username"]
+
+        start_index = (page - 1) * limit
+
+        chat_history = DATABASE.client[ENV_VAR.MONGO_DB_NAME_CHATS][
+            "chathistories"
+        ].find_one(
+            {"username": username}, {"history": {"$slice": [start_index, limit]}}
+        )
+
+        if chat_history:
+            paginated_response = get_paginated_response(
+                chat_history["history"], page, limit, chat_history["historyCount"]
+            )
+            return ResponseHandler.success(4001, paginated_response)
+        else:
+            return ResponseHandler.error(4002, 404)
+    except Exception as error:
+        print("Error getting chat history:", error)
+        return ResponseHandler.error(9000)

src/helpers/json_convertor.py

@@ -0,0 +1,15 @@
+import json
+from bson import ObjectId
+from datetime import datetime
+
+
+def custom_serializer(obj):
+    if isinstance(obj, ObjectId):
+        return str(obj)
+    if isinstance(obj, datetime):
+        return obj.isoformat()  # Convert datetime to ISO 8601 format
+    raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
+
+def convert_to_json(data):
+    return json.dumps(data, default=custom_serializer, indent=4)
+

src/helpers/pagination.py

@@ -0,0 +1,19 @@
+import math
+
+
+def get_paginated_response(data, current_page, limit, total):
+    total_pages = math.ceil(total / limit)
+    current_page = int(current_page)
+    previous_page = current_page - 1 if current_page > 1 else None
+    next_page = current_page + 1 if current_page < total_pages else None
+
+    return {
+        "page_data": data,
+        "page_information": {
+            "total_data": total,
+            "last_page": total_pages,
+            "current_page": current_page,
+            "previous_page": previous_page,
+            "next_page": next_page,
+        },
+    }

src/helpers/response.py

@@ -0,0 +1,27 @@
+from fastapi import Response
+from src.langs.en.messages import get_message
+from src.helpers.json_convertor import convert_to_json
+
+class ResponseHandler:
+    @staticmethod
+    def success(message_code=None, data=None, status_code=200):
+        response = {
+            "success": True,
+            "message": get_message(message_code),
+            "data": data
+        }
+        return Response(content=convert_to_json(response), status_code=status_code, media_type="application/json")
+
+    @staticmethod
+    def error(message_code=9999, error=None, status_code=422, data=None):
+        response = {
+            "success": False,
+            "message": get_message(message_code),
+            "error": str(error) if error else None,
+            "data": data
+        }
+        return Response(content=convert_to_json(response), status_code=(500 if message_code == 9999 else status_code), media_type="application/json")
+
+    @staticmethod
+    def success_mediator(response):
+        return Response(content=(response.content), status_code=response.status_code, media_type="application/json")

src/helpers/send_request.py

@@ -0,0 +1,12 @@
+from requests import request
+
+
+def sendRequest(url, method="get", data=None, files=None, headers=None):
+    try:
+        if files:
+            return request(method, url, data=data, files=files, headers=headers)
+        return request(method, url, json=data, headers=headers)
+    except Exception as e:
+        # Handle exceptions and return None
+        print(f"Error occurred during request: {e}")
+        return None

src/helpers/upload_file_cloudinary.py

@@ -0,0 +1,22 @@
+import cloudinary
+from cloudinary.uploader import upload
+import io
+from fastapi import UploadFile
+
+
+# Define the upload_file function as an async function
+async def upload_file(file: UploadFile, new_img_file_name: str, dir_address: str):
+    try:
+
+        # Upload the file to Cloudinary
+        result = upload(
+            file=file.file,
+            resource_type="auto",
+            public_id=new_img_file_name,
+            folder=dir_address,
+        )
+
+        return result
+    except Exception as error:
+        print(error)
+        raise error

src/langs/en/messages.py

@@ -0,0 +1,39 @@
+MESSAGES = {
+    # Bot messages
+    2000: "Answered successfully.",
+    2001: "Temporary chain created successfully.Will be deleted after 2 hours.",
+    2002: "Embedding added successfully.",
+    2003: "No files provided",
+    2004: "Unsupported file format",
+    # Chat messages
+    3000: "Chat created successfully.",
+    3001: "Chats fetched successfully.",
+    3002: "Chat fetched successfully.",
+    3003: "Chat not found.",
+    3004: "Chat updated successfully.",
+    3005: "Chat deleted successfully.",
+    3006: "Chat with this collection name already exists.",
+    3007: "Icon file must be uploaded.",
+    # Chat history messages
+    4000: "Chat history created successfully.",
+    4001: "Chat history retrieved successfully.",
+    4002: "Chat history not found.",
+    4003: "Chat history updated successfully.",
+    4004: "Chat history deleted successfully.",
+    4005: "Question deleted successfully.",
+    4006: "Question not found.",
+    # Authorization messages
+    5001: "Unauthorized - Admin access required.",
+    5002: "Access denied - Unauthorized.",
+    5003: "Your session expired! Please log in again.",
+    # General messages
+    9000: "Chatbot API server is running...",
+    9999: "Internal Server Error.",
+}
+
+
+def get_message(message_code):
+    if isinstance(message_code, int) and message_code in MESSAGES:
+        return MESSAGES[message_code]
+
+    return message_code

src/main.py

@@ -0,0 +1,19 @@
+from fastapi import APIRouter
+from src.routes.main import main_router
+import src.config.cloudinaryConfig
+from src.config.databaseConfig import DATABASE
+
+origins = ["http://localhost:3000"]
+
+main = APIRouter()
+
+DATABASE()
+
+main.include_router(main_router, prefix="/api", tags=["main"])
+
+from src.helpers.response import ResponseHandler
+
+
+@main.get("/")
+async def home():
+    return ResponseHandler.success(message_code=9000)

src/middleware/verifyToken.py

@@ -0,0 +1,22 @@
+from fastapi import HTTPException, Header, status
+from src.config.appConfig import ENV_VAR, LOG
+import jwt
+
+async def verify_token(authorization: str = Header(None)):
+    try:
+        if not authorization or not authorization.startswith("Bearer "):
+            raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Token not provided or invalid")
+        
+        token = authorization.split("Bearer ")[1]
+
+        try:
+            verified = jwt.decode(token, ENV_VAR.JWT_SECRET, algorithms=["HS256"])
+            LOG.debug("Token verified successfully")
+        except jwt.ExpiredSignatureError:
+            LOG.debug("Token expired")
+            raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Token expired")
+          
+        return verified
+    except Exception as e:
+        LOG.error(f"An error occurred: {e}")
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e))

src/routes/auth_routes.py

@@ -0,0 +1,10 @@
+from fastapi import APIRouter
+from src.controllers.auth.auth_controller import router as auth_routes
+from src.controllers.auth.user_controller import router as user_routes
+from src.controllers.auth.otp_controller import router as otp_routes
+
+main_auth_router = APIRouter()
+
+main_auth_router.include_router(auth_routes, prefix="/auth", tags=["Authentication"])
+main_auth_router.include_router(user_routes, prefix="/user", tags=["Authentication"])
+main_auth_router.include_router(otp_routes, prefix="/mail", tags=["Authentication"])

src/routes/chat_routes.py

@@ -0,0 +1,10 @@
+from fastapi import APIRouter
+from src.controllers.chat.chats_controller import router as chat_routes
+from src.controllers.chat.bot_controller import router as bot_routes
+from src.controllers.chat.history_controller import router as history_routes
+
+main_chat_router = APIRouter()
+
+main_chat_router.include_router(chat_routes, prefix="", tags=["chatbot"])
+main_chat_router.include_router(bot_routes, prefix="/bot", tags=["chatbot"])
+main_chat_router.include_router(history_routes, prefix="/history", tags=["chatbot"])

src/routes/main.py

@@ -0,0 +1,8 @@
+from fastapi import APIRouter
+from src.routes.auth_routes import main_auth_router
+from src.routes.chat_routes import main_chat_router
+
+main_router = APIRouter()
+
+main_router.include_router(main_auth_router)
+main_router.include_router(main_chat_router, prefix="/chat")

src/services/message_services.py

@@ -0,0 +1,30 @@
+from src.config.databaseConfig import DATABASE
+from src.config.appConfig import ENV_VAR
+from bson import ObjectId
+
+
+async def save_question_and_answer_to_chat_history(username, history_obj):
+    try:
+        existing_chat_history = DATABASE.client[ENV_VAR.MONGO_DB_NAME_CHATS][
+            "chathistories"
+        ].find_one({"username": username})
+        history_obj["_id"] = ObjectId()
+        if not existing_chat_history:
+            new_chat_history = {
+                "username": username,
+                "history": [history_obj],
+                "historyCount": 1,
+            }
+            DATABASE.client[ENV_VAR.MONGO_DB_NAME_CHATS]["chathistories"].insert_one(
+                new_chat_history
+            )
+        else:
+            DATABASE.client[ENV_VAR.MONGO_DB_NAME_CHATS]["chathistories"].update_one(
+                {"username": username},
+                {
+                    "$push": {"history": {"$each": [history_obj], "$position": 0}},
+                    "$inc": {"historyCount": 1},
+                },
+            )
+    except Exception as error:
+        raise ValueError(f"Error saving question and answer to chat history: {error}")