99 additions of files in the repo, 99 additions of files...

.gitignore

@@ -63,3 +63,7 @@ dist/
 # Ignore Hugging Face Spaces outputs
 output/
 runs/
+
+# OTHER ELEMENTS TO IGNORE
+pdfs/
+persona.json

FAISS-index.py

@@ -0,0 +1,27 @@
+from datasets import Dataset, load_from_disk
+import faiss
+import numpy as np
+from transformers import RagRetriever, RagTokenizer, RagSequenceForGeneration
+
+# Example: Create a dataset
+data = {"text": ["This is a sample text.", "Another sample text."]}
+dataset = Dataset.from_dict(data)
+
+# Save the dataset to disk
+dataset_path = "path/to/your/dataset"
+dataset.save_to_disk(dataset_path)
+
+# Create FAISS index
+passages = dataset["text"]
+tokenizer = RagTokenizer.from_pretrained("facebook/rag-sequence-nq")
+model = RagSequenceForGeneration.from_pretrained("facebook/rag-sequence-nq")
+passage_embeddings = model.get_encoder()(
+    tokenizer(passages, return_tensors="pt", padding=True, truncation=True)
+).last_hidden_state.mean(dim=1).detach().numpy()
+
+index = faiss.IndexFlatL2(passage_embeddings.shape[1])
+index.add(passage_embeddings)
+
+# Save the index to disk
+index_path = "path/to/your/index"
+faiss.write_index(index, index_path)

ILYA/Lib/site-packages/_distutils_hack/__init__.py

@@ -3,16 +3,9 @@ import sys
 import os
 
 
-is_pypy = '__pypy__' in sys.builtin_module_names
-
-
 def warn_distutils_present():
     if 'distutils' not in sys.modules:
         return
-    if is_pypy and sys.version_info < (3, 7):
-        # PyPy for 3.6 unconditionally imports distutils, so bypass the warning
-        # https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250
-        return
     import warnings
 
     warnings.warn(
@@ -90,7 +83,7 @@ class DistutilsMetaFinder:
         # optimization: only consider top level modules and those
         # found in the CPython test suite.
         if path is not None and not fullname.startswith('test.'):
-            return
+            return None
 
         method_name = 'spec_for_{fullname}'.format(**locals())
         method = getattr(self, method_name, lambda: None)
@@ -98,7 +91,7 @@ class DistutilsMetaFinder:
 
     def spec_for_distutils(self):
         if self.is_cpython():
-            return
+            return None
 
         import importlib
         import importlib.abc
@@ -115,7 +108,7 @@ class DistutilsMetaFinder:
             #   setuptools from the path but only after the hook
             #   has been loaded. Ref #2980.
             # In either case, fall back to stdlib behavior.
-            return
+            return None
 
         class DistutilsLoader(importlib.abc.Loader):
             def create_module(self, spec):
@@ -142,7 +135,7 @@ class DistutilsMetaFinder:
         Ensure stdlib distutils when running under pip.
         See pypa/pip#8761 for rationale.
         """
-        if self.pip_imported_during_build():
+        if sys.version_info >= (3, 12) or self.pip_imported_during_build():
             return
         clear_distutils()
         self.spec_for_distutils = lambda: None
@@ -208,15 +201,20 @@ class shim:
         insert_shim()
 
     def __exit__(self, exc, value, tb):
-        remove_shim()
+        _remove_shim()
 
 
 def insert_shim():
     sys.meta_path.insert(0, DISTUTILS_FINDER)
 
 
-def remove_shim():
+def _remove_shim():
     try:
         sys.meta_path.remove(DISTUTILS_FINDER)
     except ValueError:
         pass
+
+
+if sys.version_info < (3, 12):
+    # DistutilsMetaFinder can only be disabled in Python < 3.12 (PEP 632)
+    remove_shim = _remove_shim

ILYA/Lib/site-packages/pip-24.0.dist-info/RECORD

@@ -1,6 +1,6 @@
-../../Scripts/pip.exe,sha256=k35SUQimLVeTsIlYlMm6g1Xzaf6Lmk60-OnSuQKVbfQ,108478
-../../Scripts/pip3.11.exe,sha256=k35SUQimLVeTsIlYlMm6g1Xzaf6Lmk60-OnSuQKVbfQ,108478
-../../Scripts/pip3.exe,sha256=k35SUQimLVeTsIlYlMm6g1Xzaf6Lmk60-OnSuQKVbfQ,108478
+../../Scripts/pip.exe,sha256=hFNddXUbkFihSvcCqAYN--Wk2Cwe9FggCeLF2RC_5yc,108478
+../../Scripts/pip3.11.exe,sha256=hFNddXUbkFihSvcCqAYN--Wk2Cwe9FggCeLF2RC_5yc,108478
+../../Scripts/pip3.exe,sha256=hFNddXUbkFihSvcCqAYN--Wk2Cwe9FggCeLF2RC_5yc,108478
 pip-24.0.dist-info/AUTHORS.txt,sha256=SwXm4nkwRkmtnO1ZY-dLy7EPeoQNXMNLby5CN3GlNhY,10388
 pip-24.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
 pip-24.0.dist-info/LICENSE.txt,sha256=Y0MApmnUmurmWxLGxIySTFGkzfPR_whtw0VtyLyqIQQ,1093