Enhanced code to use .vec or .bin files instead of json

Incorporated english word embeddings (with corresponding examples)
Starting code enhancement with pythons' typing

.gitattributes

@@ -33,4 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 data/semi_embedding_v6.zip filter=lfs diff=lfs merge=lfs -text
 data/half_embedding_v6.zip filter=lfs diff=lfs merge=lfs -text
-data/wiki-news-300d-1M.vec filter=lfs diff=lfs merge=lfs -text
+data/wiki-news-300d-1M.vec filter=lfs diff=lfs merge=lfs -text
+data/GoogleNews-vectors-negative300-SLIM.bin filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1 +1,3 @@
 __pycache__/
+bias_tool_logs/
+*.env

app.py

@@ -13,11 +13,16 @@ from interfaces.interface_BiasWordExplorer import interface as biasWordExplorer_
 # --- Tool config ---
 AVAILABLE_LOGS      = True                          # [True     | False]
 LANGUAGE            = "english"                     # [spanish  | english]
-EMBEDDING_SUBSET    = "fasttext"                    # [fasttext | mini]
+EMBEDDINGS_PATH     = "data/GoogleNews-vectors-negative300-SLIM.bin"
+MAX_NEIGHBORS       = 20
 
 # --- Init classes ---
 embedding = Embedding(
-    subset_name=EMBEDDING_SUBSET
+    path=EMBEDDINGS_PATH,
+    binary=EMBEDDINGS_PATH.endswith('.bin'),
+    limit=100_000,
+    randomizedPCA=False,
+    max_neighbors=20
 )
 labels = pd.read_json(f"language/{LANGUAGE}.json")["app"]
 
@@ -30,6 +35,7 @@ INTERFACE_LIST = [
     wordExplorer_interface(
         embedding=embedding,
         available_logs=AVAILABLE_LOGS,
+        max_neighbors=MAX_NEIGHBORS,
         lang=LANGUAGE),
 ]
 

data/.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+data_loader.py

data/{fasttext_embedding_v6.zip → GoogleNews-vectors-negative300-SLIM.bin}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c35f3dda1d216d9baed3fc77f3b6bb51130f07faf0ee418029344635a0b732b7
-size 165727812
+oid sha256:046e0921bcb665f50d646b0963fcef8c5abb5f830d0daba8f686e1dffd6ad832
+size 362017275

data/data_loader.py

@@ -13,16 +13,12 @@ def load_embeddings(path, binary = False, randomPCA = False, limit = None):
     else:
         pca = PCA(n_components=2)
     
+    print("--------> PATH:", path)
     model = KeyedVectors.load_word2vec_format(path, binary=binary, limit=limit)
 
     # Cased Vocab
-    cased_words = model.vocab.keys()
-
-    #Normalized vectors
-    model.init_sims(replace=True)
-    cased_emb = [model[word] for word in cased_words]
-
-    # PCA reduction
+    cased_words = model.index_to_key
+    cased_emb = model.get_normed_vectors()
     cased_pca = pca.fit_transform(cased_emb)
 
     df_cased = pd.DataFrame(
@@ -36,6 +32,6 @@ def load_embeddings(path, binary = False, randomPCA = False, limit = None):
 
     df_cased['word'] = df_cased.word.apply(lambda w: w.lower())
     df_uncased = df_cased.drop_duplicates(subset='word')
-    df_uncased.to_json(path[:-3] + 'json')
+    return df_uncased
 
-load_embeddings('./wiki-news-300d-1M.vec', limit=10000)
+#load_embeddings('data/fasttext-sbwc.100k.vec', limit=1000)

data/mini_embedding_v6.zip

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6fa1594f66f29388719f9125eebdd529054f31bc9564e609d5162ba328a054be
-size 94479

data/wiki-news-300d-1M.vec

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dd4d0ea4f00dbd94ea4948957506f5c6601dd06c54150f898ce1acc15621284b
-size 2259088777

examples/examples.py

@@ -1,69 +1,15 @@
-example_fem = {
-    "mujer": "la mente de una mujer que durante los últimos",
-    "chica": "enamorado de la misma chica desde la infancia mary",
-    "ella": "ella llego a la final",
-            "madre": "su padre y su madre margarita de parma",
-            "hija": "hija de inmigrantes españoles en",
-            "femenino": "campeonato mundial de voleibol femenino fue la duodécima edición",
-}
-example_joven = {
-    "joven": "",
-    "inmaduro": "",
-    "niño": "",
-    "crio": ""
-}
-example_viejo = {
-    "viejo": "",
-    "maduro": "",
-    "anciano": "",
-    "adulto": ""
-}
+fem_words = 'woman,girl,her,mother,daughter,feminine'
+masc_words = 'man,boy,him,father,son,masculine'
+young_words = 'young,immature,kid,child'
+old_words = 'old,mature,elderly,adult'
+diagnose_words = 'education,cook,wash,sports,clothes,fight,nurse,win,leader,saw,nurse,surgeon,king,queen'
 
+positive_money_words = 'save,economize,administer,manage,business,benefits'
+negative_money_words = 'waste,squander'
+diagnose_money = 'german,australian,argentinian,millionaire,rich,poor'
 
-example_masc = {
-    "hombre": "deseo innato que todo hombre tiene de comunicar su",
-    "chico": "fue un chico interesado en artes",
-    "el": "el parque nacional liwonde",
-    "padre": "la muerte de su padre en 1832 se formó",
-    "hijo": "le dice a su hijo aún no nacido como",
-            "masculino": "el mito es esencialmente masculino y entre las causas",
-}
-
-example_diagnose = {
-    "ario": "establecer que el pueblo ario vivió en inmemoriales tiempos",
-    "educación": "sentido de vida religión educación y cultura para cada mujer",
-    "pagado": "un rescate muy grande pagado por sus seguidores a",
-    "cocinar": "empezó a cocinar una sopa usando",
-    "lavar": "era directamente usado para lavar ropa por eso la",
-    "deporte": "se convirtió en el deporte más popular del país",
-    "ropa": "usan el kimono una ropa tradicional japonesa",
-    "pelea": "mal por la violenta pelea entre ambos hermanos",
-    "enfermero": "en enfermería el diagnóstico enfermero o diagnóstico de enfermería es",
-    "ganar": "una necesidad un modo de ganar",
-    "líder": "del estado en manos del líder opositor henrique capriles para el",
-    "coser": "realizar tareas domésticas básicas como coser y poner la mesa",
-    "cuidar": "de la fpf encargada de cuidar los intereses de los clubes",
-    "cirujano": "afrancesado ocupando el puesto de cirujano militar en el ejército josefino",
-    "rey": "la princesa jeongsung esposa del rey danjong que ascendió al trono",
-    "reina": "año ganó el título de reina de la bahía en el"
-}
-
-
-fem_words = ','.join([word for word, context in example_fem.items()])
-fem_contexts = ','.join([context for word, context in example_fem.items()])
-masc_words = ','.join([word for word, context in example_masc.items()])
-masc_contexts = ','.join([context for word, context in example_masc.items()])
-young_words = ','.join([word for word, context in example_joven.items()])
-old_words = ','.join([word for word, context in example_viejo.items()])
-diagnose_words = ','.join([word for word, context in example_diagnose.items()])
-diagnose_contexts = ','.join([context for word, context in example_diagnose.items()])
-
-positive_money_words = 'ahorrar,economizar,administrar,manejar,negocio,beneficios'
-negative_money_words = 'malgastar,derrochar'
-diagnose_money = 'alemán,australiano,argentino,millonario,rico,pobre'
-
-lazy_words = 'vago, perezoso, gandul'
-active_words = 'trabajar, esfuerzo, trabajador'
+lazy_words = 'lazy, sluggish, slacker'
+active_words = 'active, effort , worker'
 
 examples1_explorar_sesgo_en_palabras = [
     [fem_words, masc_words, diagnose_words],
@@ -81,42 +27,4 @@ examples2_explorar_sesgo_en_palabras = [
 examples_explorar_relaciones_entre_palabras = [
     [diagnose_words, fem_words, masc_words, young_words, old_words],
     [diagnose_money, lazy_words, active_words, positive_money_words, negative_money_words],
-]
-
-
-# Examples Datos
-examples_datos = ["ronronear", "ella", "el", "ellos"]
-
-
-# Examples Sesgos en frases
-examples_sesgos_frases = [
-    ["los * manejan bien la plata", "argentinos,pobres,millonarios,australianos"],
-    ["los cordobeses bailan *", ""],
-    ["en la década de los ochenta, el * fue lo más escuchada en Argentina", "rock,tango,jazz,blues"],
-    ["",""]
-]
-
-# Examples Crows_pairs
-examples_crows_pairs = [
-    [   "Los hombres 'musculosos' tienen más suerte con las mujeres",
-        "Los hombres 'gordos' tienen más suerte con las mujeres",
-        "Los hombres 'delgados' tienen más suerte con las mujeres", 
-        "Los hombres 'bajos' tienen más suerte con las mujeres",
-        "Los hombres 'rubios' tienen más suerte con las mujeres",
-        "Los hombres 'altos' tienen más suerte con las mujeres"],
-    [   "No cualquier 'mujer' con hijos logra ser 'una buena' profesional",
-        "No cualquier 'hombre' con hijos logra ser 'un buen' profesional",
-        "","","",""],
-    [   "Todos sabemos que los 'mapuches' son alcohólicos.", 
-        "Todos sabemos que los 'descendientes de europeos' son alcohólicos.",
-        "","","",""],
-    [   "Los 'orientales' no pueden manejar porque tiene los ojos achinados.",
-        "Los 'occidentales' no pueden manejar porque tiene los ojos achinados.",
-        "","","",""],
-    [   "A 'las viejas' les gusta contar chusmerio",
-        "A 'los viejos' les gusta contar chusmerio",
-        "","","",""],
-    [   "Mi hermana fue al shopping para Navidad a comprar regalos a última hora y se perdió ya que es una 'morocha' tonta.",
-        "Mi hermana fue al shopping para Navidad a comprar regalos a última hora y se perdió ya que es una 'rubia' tonta.",
-        "","","",""]
 ]

interfaces/.gitignore

@@ -0,0 +1 @@
+__pycache__/

interfaces/interface_WordExplorer.py

@@ -9,7 +9,13 @@ from examples.examples import examples_explorar_relaciones_entre_palabras
 
 plt.rcParams.update({'font.size': 14})
 
-def interface(embedding, available_logs, lang="spanish"):
+def interface(
+    embedding, 
+    available_logs: bool, 
+    max_neighbors: int, # Updated
+    lang: str="spanish",
+) -> gr.Blocks:
+
     # --- Init logs ---
     log_callback = HuggingFaceDatasetSaver(
         available_logs=available_logs
@@ -53,10 +59,10 @@ def interface(embedding, available_logs, lang="spanish"):
                 with gr.Row():
                     with gr.Row():
                         gr.Markdown(labels["plotNeighbours"]["title"])
-                        n_neighbors = gr.Slider(minimum=0,maximum=100,step=1,label=labels["plotNeighbours"]["quantity"])
+                        n_neighbors = gr.Slider(minimum=0,maximum=max_neighbors,step=1,label=labels["plotNeighbours"]["quantity"])
                     with gr.Row():
                         alpha = gr.Slider(minimum=0.1,maximum=0.9, value=0.3, step=0.1,label=labels["options"]["transparency"])
-                        fontsize=gr.Number(value=18, label=labels["options"]["font-size"])
+                        fontsize=gr.Number(value=25, label=labels["options"]["font-size"])
                     with gr.Row():
                         btn_plot = gr.Button(labels["plot_button"])
                 with gr.Row(): 

modules/.gitignore

@@ -0,0 +1 @@
+__pycache__/

modules/model_embbeding.py

@@ -1,57 +1,126 @@
+from modules.module_ann import Ann
+from memory_profiler import profile
+from sklearn.neighbors import NearestNeighbors
+from sklearn.decomposition import PCA
+from gensim.models import KeyedVectors
+from typing import List
+import os
 import operator
-import numpy as np
 import pandas as pd
+
+import numpy as np
 from numpy import dot
 from gensim import matutils
-from modules.module_ann import Ann
-from memory_profiler import profile
-from sklearn.neighbors import NearestNeighbors
 
 
 class Embedding:
     @profile
-    def __init__(self, subset_name):
-        # Dataset info
-        self.ds_subset = subset_name
-        self.ds_path = f"data/{subset_name}_embedding_v6.zip"
+    def __init__(self, 
+        path: str, 
+        binary: bool, 
+        limit: int=None, 
+        randomizedPCA: bool=False,
+        max_neighbors: int=20
+    ) -> None:
+
+        # Embedding vars
+        self.path = path
+        self.limit = limit
+        self.randomizedPCA = randomizedPCA
+        self.binary = binary
+        self.max_neighbors = max_neighbors
         
-        # Pandas dataset
+        # Full embedding dataset 
         self.ds = None
 
-        # All Words embedding List[List[float]]
-        self.embedding = None
-
-        # Estimate AproximateNearestNeighbors
-        self.ann = None
+        # Estimate NearestNeighbors
+        self.ann = None     # Aproximate with Annoy method
+        self.neigh = None   # Exact with Sklearn method
 
         # Load embedding and pca dataset
         self.__load()
 
-    def __contains__(self, word):
-        return word in self.ds['word'].to_list()
-
-    def __load(self):
-        print(f"Preparing {self.ds_subset} embedding...")
+    def __load(
+        self, 
+    ) -> None:
 
-        # --- Download dataset ---
-        self.ds = pd.read_json(self.ds_path)
+        print(f"Preparing {os.path.basename(self.path)} embeddings...")
 
-        # --- Get embedding from string
-        self.embedding = self.ds['embedding'].to_list()
+        # --- Prepare dataset ---
+        self.ds = self.__preparate(
+            self.path, self.binary, self.limit, self.randomizedPCA
+        )
 
-        # --- Get forest tree to estimate Nearest Neighbors ---
+        # --- Estimate Nearest Neighbors
+        # Method A: Througth annoy using forest tree
         self.ann = Ann(
             words=self.ds['word'], 
             vectors=self.ds['embedding'], 
             coord=self.ds['pca']
         )
-        self.ann.init(n_trees=20, metric='dot', n_jobs=-1)
+        self.ann.init(
+            n_trees=20, metric='dot', n_jobs=-1
+        )
+
+        # Method B: Througth Sklearn method
+        self.neigh = NearestNeighbors(
+            n_neighbors=self.max_neighbors
+        )
+        self.neigh.fit(
+            X=self.ds['embedding'].to_list()
+        )
+    
+    def __preparate(
+        self, 
+        path: str, 
+        binary: bool, 
+        limit: int,
+        randomizedPCA: bool
+    ) -> pd.DataFrame:
+
+        if randomizedPCA:
+            pca = PCA(
+                n_components=2, 
+                copy=False, 
+                whiten=False, 
+                svd_solver='randomized', 
+                iterated_power='auto'
+            )
+
+        else:
+            pca = PCA(
+                n_components=2
+            )
+        
+        model = KeyedVectors.load_word2vec_format(
+            fname=path, 
+            binary=binary, 
+            limit=limit
+        )
+
+        # Cased Vocab
+        cased_words = model.index_to_key
+        cased_emb = model.get_normed_vectors()
+        cased_pca = pca.fit_transform(cased_emb)
+
+        df_cased = pd.DataFrame(
+            zip(
+                cased_words,
+                cased_emb,
+                cased_pca
+            ),
+            columns=['word', 'embedding', 'pca']
+        )
 
-        # --- Fit Sklearn NN method ---
-        self.neigh = NearestNeighbors(n_neighbors=20)
-        self.neigh.fit(self.embedding)
+        df_cased['word'] = df_cased.word.apply(lambda w: w.lower())
+        df_uncased = df_cased.drop_duplicates(subset='word')
+        return df_uncased
 
-    def __getValue(self, word, feature):
+    def __getValue(
+        self, 
+        word: str, 
+        feature: str
+    ):
         word_id, value = None, None
 
         if word in self:
@@ -62,30 +131,57 @@ class Embedding:
 
         return value
     
-    def getEmbedding(self, word):
+    def getEmbedding(
+        self, 
+        word: str
+    ):
+
         return self.__getValue(word, 'embedding')
 
-    def getPCA(self, word):
+    def getPCA(
+        self, 
+        word: str
+    ):
+
         return self.__getValue(word, 'pca')
     
-    def cosineSimilarities(self, vector_1, vectors_all):
-        norm = np.linalg.norm(vector_1)
-        all_norms = np.linalg.norm(vectors_all, axis=1)
-        dot_products = dot(vectors_all, vector_1)
-        similarities = dot_products / (norm * all_norms)
-        return similarities 
-    
-    def getNearestNeighbors(self, word, n_neighbors=10, nn_method='sklearn'):
+    def getNearestNeighbors(
+        self, 
+        word: str, 
+        n_neighbors: int=10, 
+        nn_method: str='sklearn'
+    ) -> List[str]:
+
+        assert(n_neighbors <= self.max_neighbors), f"Error: The value of the parameter 'n_neighbors:{n_neighbors}' must less than or equal to {self.max_neighbors}!."
+
         if nn_method == 'ann':
             words = self.ann.get(word, n_neighbors)
+            
         elif nn_method == 'sklearn':
-            word_emb = self.getEmbedding(word)
-            neighbors = self.neigh.kneighbors([word_emb], n_neighbors)[1][0]
-            words = operator.itemgetter(*neighbors)(self.ds['word'])
+            word_emb = self.getEmbedding(word).reshape(1,-1)
+            _, nn_ids = self.neigh.kneighbors(word_emb, n_neighbors+1)
+            #words = operator.itemgetter(*nn_ids[0])(self.ds['word'].to_list())
+            words = [self.ds['word'].to_list()[idx] for idx in nn_ids[0]][1:]
         else: 
             words = []
         return words
 
+    def __contains__(
+        self, 
+        word: str
+    ) -> bool:
+
+        return word in self.ds['word'].to_list()
+
+    # ToDo: Revisar estos dos métodos usados en la pestaña sesgoEnPalabras
+    # ya que ahora los embedding vienen normalizados
+    def cosineSimilarities(self, vector_1, vectors_all):
+        norm = np.linalg.norm(vector_1)
+        all_norms = np.linalg.norm(vectors_all, axis=1)
+        dot_products = dot(vectors_all, vector_1)
+        similarities = dot_products / (norm * all_norms)
+        return similarities 
+
     def getCosineSimilarities(self, w1, w2):
         return dot(
             matutils.unitvec(self.getEmbedding(w1)), 

modules/module_WordExplorer.py

@@ -142,8 +142,8 @@ class WordExplorer:
                 processed_word_list.append(WordToPlot(word, color_dict[color], color, 1))
 
                 if n_neighbors > 0:
-                    neighbors = self.get_neighbors(word, 
-                                                   n_neighbors=n_neighbors+1,
+                    neighbors = self.get_neighbors(word,
+                                                   n_neighbors=n_neighbors,
                                                    nn_method=kwargs.get('nn_method', 'sklearn')
                                                    )
                     for n in neighbors: