<jupyter_start><jupyter_text>Importing Libraries<jupyter_code>from collections import defaultdict
from pathlib import Path
import numpy as np
import pandas as pd
from collections import namedtuple
from tabulate import tabulate

import torch
import os

from adversarial_debiasing import AdversarialDebiasing
from load_data import load_data, transform_data, Datapoint

from load_vectors import load_pretrained_vectors, load_vectors
import config
import utility_functions
import qualitative_evaluation

import gensim
import gzip
import pickle<jupyter_output><empty_output><jupyter_text>Autoreloading changes made in other python scripts<jupyter_code>%load_ext autoreload
%autoreload 2<jupyter_output><empty_output><jupyter_text>Loading the desired word vectors<jupyter_code># For Wikipedia2Vec - pass "Wikipedia2Vec"
# For Glove - pass "Glove"
# For GoogleNews (Word2Vec) - pass "GoogleNews"

word_vectors = load_pretrained_vectors("Glove")<jupyter_output>Loading from saved file.<jupyter_text>Loading the Google Analogies Training Dataset<jupyter_code>analogy_dataset = load_data()
print("Total Number of Analogies : {}".format(len(analogy_dataset)))
print("The first 5 analogies:")
analogy_dataset[0:6]<jupyter_output>Total Number of Analogies : 19544
The first 5 analogies:<jupyter_text>Transforming the above dataset to include the respective word embeddings<jupyter_code>transformed_analogy_dataset, gender_subspace = transform_data(word_vectors, analogy_dataset, use_boluk = False)
print("Transformed datapoint for the first analogy")
transformed_analogy_dataset[0]<jupyter_output>Transformed datapoint for the first analogy<jupyter_text>Obtaining the dimensionality of the word embeddings<jupyter_code>word_embedding_dim = transformed_analogy_dataset[0].gt_embedding.shape[0]
print("Dimensions of the word embedding : {}".format(word_embedding_dim))<jupyter_output>Dimensions of the word embedding : 300<jupyter_text>Test the dimensions of the transformed analogy dataset components<jupyter_code># Testing the transformed analogy dataset
assert transformed_analogy_dataset[0].analogy_embeddings.shape[0] == word_embedding_dim * 3
assert transformed_analogy_dataset[0].gt_embedding.shape[0] == word_embedding_dim
assert transformed_analogy_dataset[0].protected.shape[0] == 1

print("Dimensions of the network input : {}".format(transformed_analogy_dataset[0].analogy_embeddings.shape))
print("Dimensions of the ground-truth embedding : {}".format(transformed_analogy_dataset[0].gt_embedding.shape))
print("Dimensions of the ground-truth protected variable : {}".format(transformed_analogy_dataset[0].protected.shape))<jupyter_output>Dimensions of the network input : (900,)
Dimensions of the ground-truth embedding : (300,)
Dimensions of the ground-truth protected variable : (1,)<jupyter_text>Flag to indicate whether you want to use a pre-trained model or you want to train a model from scratch<jupyter_code>use_pretrained = True<jupyter_output><empty_output><jupyter_text>In case you want to use a pre-trained model, then specify the type of word embeddings upon which the model was trained<jupyter_code># "GNews" for Google News (Word2Vec)
# "WikipediaVec" for Wikipedia2Vec
# "Glove" for Glove vectors

word_embedding_type = "Glove"<jupyter_output><empty_output><jupyter_text>In case of using a pre-trained model<jupyter_code>if use_pretrained:
    
    # Obtaining the saved parameters dictionary
    pretrained_parameters = utility_functions.obtain_trained_parameters('models')
    
    # Obtaining the best weights for the non-debiased model
    non_debiased_W1 = pretrained_parameters['non_debiased'][word_embedding_type]["W1"]
    
    # Creating an instance of the non-debiased model
    non_debiased_model = AdversarialDebiasing(word_embedding_dim = word_embedding_dim, debias = False)
    non_debiased_model.W1 = non_debiased_W1
    
    # Obtaining the best weights for the debiased model
    debiased_W1 = pretrained_parameters['debiased'][word_embedding_type]["W1"]
    
    # Creating an instance of the debiased model
    debiased_model = AdversarialDebiasing(word_embedding_dim = word_embedding_dim, debias = True)
    debiased_model.W1 = debiased_W1<jupyter_output><empty_output><jupyter_text>In case of using a model trained from scratch<jupyter_code>if not use_pretrained:
    
    # Best parameters from grid search
    best_adversary_loss_weight = 1.0
    best_learning_rate = 2 ** -6
    
    # Creating an instance of the non-debiased model
    non_debiased_model = AdversarialDebiasing(word_embedding_dim = word_embedding_dim, num_epochs = 500, debias = False, \
                                             gender_subspace = gender_subspace, batch_size = 256, \
                                              adversary_loss_weight = best_adversary_loss_weight, \
                                             classifier_learning_rate = best_learning_rate, \
                                             adversary_learning_rate = best_learning_rate)
    
    # Fitting the non-debiased model to the training dataset
    print("****************** Training the non-debiased model ********************")
    non_debiased_model.fit(dataset = transformed_analogy_dataset)
    
    # Creating an instance of the debiased model
    debiased_model = AdversarialDebiasing(word_embedding_dim = word_embedding_dim, num_epochs = 500, debias = True, \
                                             gender_subspace = gender_subspace, batch_size = 256, \
                                              adversary_loss_weight = best_adversary_loss_weight, \
                                             classifier_learning_rate = best_learning_rate, \
                                             adversary_learning_rate = best_learning_rate)
    
    Fitting the non-debiased model to the training dataset
    print("****************** Training the debiased model ********************")
    debiased_model.fit(dataset = transformed_analogy_dataset)
# Handling memory issues
word_vectors.init_sims(replace = True)<jupyter_output><empty_output><jupyter_text>Qualitative Evaluation (on some evaluation analogies)<jupyter_code># Get sexism traps as word embeddings and words
datapoints, test_analogies = qualitative_evaluation.get_datapoints(word_vectors)

# Predictions of the non debiased model
non_debiased_predictions = qualitative_evaluation.get_non_debiased_predictions(datapoints, word_embedding_dim)
non_debiased_most_similar_list = utility_functions.obtain_most_similar(non_debiased_predictions, word_vectors)

# Predictions of the debiased model
debiased_predictions = debiased_model.predict(datapoints)
debiased_most_similar_list = utility_functions.obtain_most_similar(debiased_predictions, word_vectors)

# Print similarity results for both models
qualitative_evaluation.print_combined_table(non_debiased_most_similar_list, debiased_most_similar_list, test_analogies)<jupyter_output>he : doctor :: she : 
+-------------+--------------+---------------+--------------+
| Biased      |       Biased | Debiased      |     Debiased |
| Neighbour   |   Similarity | Neighbour     |   Similarity |
|-------------+--------------+---------------+--------------|
| nurse       |        0.653 | mordrid       |        0.544 |
| doctors     |        0.652 | gynecologist  |        0.485 |
| physician   |        0.627 | midwife       |        0.48  |
| pregnant    |        0.618 | kerish        |        0.467 |
| pregnancy   |        0.589 | obstetrician  |        0.464 |
| she         |        0.584 | nurse         |        0.462 |
| midwife     |        0.568 | pediatrician  |        0.454 |
| her         |        0.566 | naturopath    |        0.451 |
| patient     |        0.565 | dermatologist |        0.45  |
+-------------+--------------+---------------+--------------+
he : director :: she : 
+-------------+--------------+--------------------+--------------+
| Biased      |    [...]
{'fraction_non_alphanumeric': 0.10749968382445935, 'fraction_numerical': 0.016820538763121284, 'mean_word_length': 3.485536018150879, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '468944', 'license': 'MIT', 'n_tokens_mistral': 2417, 'n_tokens_neox': 2230, 'n_words': 532, 'path': 'demo_adverserial_debiasing.ipynb', 'repo_name': 'victoriafoing/FACT-AI', 'size': '7907'}