Add all vector similarity feature

Procfile.yaml

@@ -1 +1 @@
-web: gunicorn -w 1 -k uvicorn.workers.UvicornWorker main.py:app --bind 0.0.0.0:8000 & streamlit run app.py --server.port 7860
+web: gunicorn -w 1 -k uvicorn.workers.UvicornWorker main:app --bind 0.0.0.0:8000 & streamlit run app.py --server.port 7860

main.py

@@ -8,14 +8,14 @@ import datetime
 from models.vectorizer import Vectorizer
 from models.prompt_search_engine import PromptSearchEngine
 from models.data_reader import load_prompts_from_jsonl
-from models.Query import Query, Query_Multiple, SearchResponse, SimilarPrompt
+from models.Query import Query, Query_Multiple, SearchResponse, SimilarPrompt, PromptVector, VectorResponse
 from decouple import config
 from fastapi import FastAPI, HTTPException, Depends, Body
 from sentence_transformers import SentenceTransformer
 
 
 
-prompt_path = r"C:\Users\jov2bg\Desktop\PromptSearch\models\prompts_data.jsonl"
+prompt_path = r"C:\Users\jov2bg\Desktop\PromptSearch\search_engine\models\prompts_data.jsonl"
 
 
 app = FastAPI(title="Search Prompt Engine", description="API for prompt search", version="1.0")
@@ -46,16 +46,17 @@ async def search_prompts(query: Query, k: int = 3):
 @app.post("/all_vectors_similarities/")
 async def all_vectors(query: Query):
 
-    all_similarities = search_engine.cosine_similarity(query.prompt, search_engine.index)
+    query_embedding = search_engine.model.encode([query.prompt])  # Encode the prompt to a vector
+    all_similarities = search_engine.cosine_similarity(query_embedding, search_engine.index)
     response = [
-        SimilarPrompt(prompt=prompt, distance=float(distance)) 
-        for prompt, distance in all_similarities.items()
+        PromptVector(vector=index, distance=float(distance)) 
+        for index, distance in enumerate(all_similarities)
     ]
-    return SearchResponse(results=response)
+    return VectorResponse(results=response)
 
 if __name__ == "__main__":
     # Server Config
-    Search_SERVER_HOST_IP = socket.gethostbyname(socket.gethostname())
-    # Search_SERVER_HOST_IP = socket.gethostbyname("localhost") # for local deployment
-    Search_SERVER_PORT = int(8084)
-    uvicorn.run(app, host=Search_SERVER_HOST_IP, port=Search_SERVER_PORT)
+    # Search_SERVER_HOST_IP = socket.gethostbyname(socket.gethostname())
+    SERVER_HOST_IP = socket.gethostbyname("localhost") # for local deployment
+    SERVER_PORT = int(8084)
+    uvicorn.run(app, host=SERVER_HOST_IP, port=SERVER_PORT)

models/Query.py

@@ -17,4 +17,11 @@ class SimilarPrompt(BaseModel):
     distance: float  
 
 class SearchResponse(BaseModel):
-    results: List[SimilarPrompt]
+    results: List[SimilarPrompt]
+
+class PromptVector(BaseModel):
+    vector: int
+    distance: float
+
+class VectorResponse(BaseModel):
+    results: List[PromptVector] 

models/data_reader.py

@@ -41,7 +41,7 @@ def load_prompts_from_jsonl(file_path):
 
 
 if __name__ == "__main__":
-    jsonl_file_path = r"C:\Users\jov2bg\Desktop\PromptSearch\models\prompts_data.jsonl"
+    jsonl_file_path = r"C:\Users\jov2bg\Desktop\PromptSearch\search_engine\models\prompts_data.jsonl"
     num_shards = 1
     dataset = download_data(num_shards, base_url)
     extract_prompts(dataset, jsonl_file_path)

models/prompt_search_engine.py

@@ -32,17 +32,27 @@ class PromptSearchEngine:
         return similar_prompts, distances[0]  # Return both the similar prompts and their distances
 
 
-    def cosine_similarity(query_vector: np.ndarray, corpus_vectors: np.ndarray) -> np.ndarray:
+    def cosine_similarity(self, query_vector, index):
         """Compute the cosine similarity between a query vector and a set of corpus vectors.
-        Args: query_vector: The query vector to compare against the corpus vectors. corpus_vectors: The set of corpus vectors to compare against the query vector. 
-        Returns: The cosine similarity between the query vector and the corpus vectors.
-        """
-        similarities = {}
-        for index, vector in enumerate(corpus_vectors):
-            if np.linalg.norm(vector) == 0:
-                raise ValueError("One of the corpus vectors has zero norm.")
-            cos_similarity = np.dot(vector, query_vector) / (np.linalg.norm(vector) * np.linalg.norm(query_vector))
-            similarities[index] = cos_similarity
-        return similarities
+            Args: query_vector: The query vector to compare against the corpus vectors. corpus_vectors: The set of corpus vectors to compare against the query vector. 
+            Returns: The cosine similarity between the query vector and the corpus vectors.
+            """
+     
+        query_vector = np.array(query_vector).astype('float32')
+        query_norm = query_vector / np.linalg.norm(query_vector)
+
+        # Get all vectors from FAISS
+        index_vectors = index.reconstruct_n(0, index.ntotal)  # Reconstruct all vectors in the index
+
+       
+        index_norms = np.linalg.norm(index_vectors, axis=1, keepdims=True)
+        normalized_index_vectors = index_vectors / index_norms
+
+       
+        cosine_similarities = np.dot(normalized_index_vectors, query_norm.T)
+
+        return cosine_similarities
+
+