<jupyter_start><jupyter_text>After learning the embeddings, obtain a _common_ vocabulary. This vocabulary should:- be common to both the early documents as well as the late documents.- should not contain any names The input embeddings are ```model.wv.vectors``` and the outout vectors are ```model.trainables.syn1neg```.<jupyter_code>import os
from random import shuffle
import gensim
import sys
if "../" not in sys.path: sys.path.append ("../")

from modules.propernames import classifier
import numpy as np
from sklearn.metrics import accuracy_score, roc_auc_score
from collections import Counter
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

torch.manual_seed (1)
MODELS_DIR = "/hg191/corpora/legaldata/models"
STATS_DIR = "/hg191/corpora/legaldata/data/stats"
EARLY_FILE = os.path.join (MODELS_DIR, "sgns.500K.early.100.model")
LATER_FILE = os.path.join (MODELS_DIR, "sgns.500K.later.100.model")
early_model = gensim.models.Word2Vec.load(EARLY_FILE)
later_model = gensim.models.Word2Vec.load(LATER_FILE)<jupyter_output><empty_output><jupyter_text>**Common vocabulary**<jupyter_code>early_vocab = {key for key in early_model.wv.vocab.keys()}
later_vocab = {key for key in later_model.wv.vocab.keys()}
common_vocab = set.intersection (early_vocab, later_vocab)<jupyter_output><empty_output><jupyter_text>**Name removal**<jupyter_code>def readExamples (filename):
    with open (filename) as fin:
        sample = [line.strip() for line in fin]
    positives = [candidate.strip("+") for candidate in sample if candidate.startswith ("+")]
    negatives = [candidate for candidate in sample if not candidate.startswith ("+")]
    
    return positives, negatives

positives, negatives = readExamples ("../data/names/annotated.txt")
sample = positives + negatives
shuffle (sample)
# All the constants!
embeds_dim = 300
hidden_dim = 300
label_size = 2
labels_map = {"Name":1, "NonName":0}
iLabels_map = {1:"Name", 0:"NonName"}
nEpochs = 10
train_size, dev_size = 1000, 200
data = [(candidate, iLabels_map[int (candidate in positives)]) 
        for candidate in sample 
        if candidate in early_vocab and candidate in later_vocab]
def use_embeddings (instance, emds):
    """Takes an instance and coverts into a vector"""
    vec = torch.Tensor (emds.wv[instance])
    return vec.view(1,-1)

def make_target (label, mapping):
    """Takes a label and coverts it into a categorical variable"""
    return torch.LongTensor([mapping[label]])
train_data, dev_data, test_data = data[:train_size], data[train_size:train_size+dev_size], data[train_size+dev_size:]
print (len (train_data), len (dev_data), len (test_data))<jupyter_output>1000 200 381<jupyter_text>Name removal using word embeddings approach!<jupyter_code>def train (instances, emds, input_dim, hidden_dim, output_dim, iters, verbose=False):
    global labels_map
    model = classifier.TwoLayerNet(input_dim=input_dim, hidden_dim=hidden_dim, output_dim=output_dim)
    loss_function = nn.NLLLoss()
    optimizer = optim.SGD(model.parameters(), lr=0.1)
    
    for epoch in range (iters):
        training_loss = 0
        for instance, label in instances:
            # Step 1. Torch always accumulates the gradient by 
            # default, so make it zero if we want to backprop 
            # for every instance.
            model.zero_grad()
            
            # Step2. Create the input and output
            src = use_embeddings(instance, emds)
            tgt = make_target (label, labels_map)
            
            # Step 3. Run the forward pass
            log_probs = model.forward (src, F.relu)
            
            # Step 4. Calculate the loss; run the backward pass
            # and make the gradient update.
            loss = loss_function(log_probs, tgt)
            loss.backward ()
            optimizer.step ()
        
            training_loss += loss.item()
            
        if verbose:
            print ("Loss after epoch {0}:{1}".format (epoch+1, training_loss/len(instances)))
    return model

def evaluate (model, instances, verbose=False):
    global iLabels_map
    predictions = list ()
    labels = list ()
    with torch.no_grad():
        for instance, label in instances:
            src = use_embeddings(instance, emds)
            log_probs = model.forward (src, F.relu)
            #prediction = F.softmax (log_probs, dim=1)
            prediction = torch.argmax (log_probs, dim=1)[0].item()
            predictions.append (iLabels_map[prediction])
            labels.append (label)

    acc = accuracy_score (labels, predictions)
    auc = roc_auc_score (labels, predictions)
    if verbose:
        print ("Accuracy after epoch {0}:{1}".format (epoch+1, acc))
        print ("AUC after epoch {0}:{1}".format (epoch+1, auc))
            
    return labels, predictions

def train_and_evaluate (train_instances, test_instances, emds, input_dim, hidden_dim, output_dim, iters, verbose=False):
    global labels_map
    global iLabels_map
    model = classifier.TwoLayerNet(input_dim=input_dim, hidden_dim=hidden_dim, output_dim=output_dim)
    loss_function = nn.NLLLoss()
    optimizer = optim.SGD(model.parameters(), lr=0.1)
    
    for epoch in range (iters):
        training_loss = 0
        for instance, label in train_instances:
            # Step 1. Torch always accumulates the gradient by 
            # default, so make it zero if we want to backprop 
            # for every instance.
            model.zero_grad()
            
            # Step2. Create the input and output
            src = use_embeddings(instance, emds)
            tgt = make_target (label, labels_map)
            
            # Step 3. Run the forward pass
            log_probs = model.forward (src, F.relu)
            
            # Step 4. Calculate the loss; run the backward pass
            # and make the gradient update.
            loss = loss_function(log_probs, tgt)
            loss.backward ()
            optimizer.step ()
        
            training_loss += loss.item()
            
        if verbose:
            print ("Loss after epoch {0}:{1}".format (epoch+1, training_loss/len(train_instances)))
            
        test_predictions = list ()
        test_labels = list ()
        with torch.no_grad():
            for instance, label in test_instances:
                src = use_embeddings(instance, emds)
                log_probs = model.forward (src, F.relu)
                #prediction = F.softmax (log_probs, dim=1)
                prediction = torch.argmax (log_probs, dim=1)[0].item()
                test_predictions.append (prediction)
                test_labels.append (labels_map[label])
        acc = accuracy_score (test_labels, test_predictions)
        auc = roc_auc_score (test_labels, test_predictions)
        if verbose:
            print ("Accuracy after epoch {0}:{1}".format (epoch+1, acc))
            print ("AUC after epoch {0}:{1}".format (epoch+1, auc))
            
    return model, test_labels, test_predictions
twoLayerNet = classifier.TwoLayerNet(300, 300, 2)
m1, _, _ = train_and_evaluate (train_data+dev_data, test_data, early_model, embeds_dim, hidden_dim, label_size, nEpochs, verbose=True)
m2, _, _ = train_and_evaluate (train_data+dev_data, test_data, later_model, embeds_dim, hidden_dim, label_size, nEpochs, verbose=True)
# Baseline (most frequent class)
print ("Baseline")
baseline_predictions = [0 for instance, label in test_data]
baseline_truth = [labels_map[label] for instance, label in test_data]
print (accuracy_score (baseline_truth, baseline_predictions))
print (roc_auc_score (baseline_truth, baseline_predictions))
def apply2vocab (model, emds):
    global iLabels_map
    predictions = dict()
    with torch.no_grad():
        for instance in emds.wv.vocab:
            src = use_embeddings(instance, emds)
            log_probs = model.forward (src, F.relu)
            prediction = torch.argmax (log_probs, dim=1)[0].item()
            predictions[instance] = iLabels_map[prediction]
            
    return predictions
V1 = apply2vocab (m1, early_model)
V2 = apply2vocab (m2, later_model)
neural_candidates = set (list (V1.keys())) & set (list (V2.keys()))
neural_names = {candidate for candidate in neural_candidates if V1[candidate] == "Name" or V2[candidate] == "Name"}

print (len(V1))
print (len(V2))
print (len (neural_names))<jupyter_output>159639
247005
46512<jupyter_text>Name removal using part of speech tagging heuristic!<jupyter_code>thresh = 0.5
with open (os.path.join (STATS_DIR, "ops.500K.early.nounprob")) as fin:
    P1 = {line.strip().split("\t")[0] for line in fin if float(line.strip().split("\t")[1]) >= thresh}
with open (os.path.join (STATS_DIR, "ops.500K.later.nounprob")) as fin:
    P2 = {line.strip().split("\t")[0] for line in fin if float(line.strip().split("\t")[1]) >= thresh}
pos_names = P1 | P2
print (len (pos_names))
with open (os.path.join(STATS_DIR, "names.neural"), "w") as fout:
    for name in neural_names:
        fout.write ("{0}\n".format (name))

with open (os.path.join (STATS_DIR, "names.tagging"), "w") as fout:
    for name in pos_names:
        fout.write ("{0}\n".format (name))<jupyter_output><empty_output><jupyter_text>Finally, eliminate all words in the vocabulary that are either identified as names by the embedding approach or by part of speech tagging.<jupyter_code>len({v for v in common_vocab if v not in pos_names and v not in neural_names})<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10806247361756015, 'fraction_numerical': 0.016040523427606587, 'mean_word_length': 3.165714285714286, 'pattern_counts': {'":': 2, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '435105', 'license': 'MIT', 'n_tokens_mistral': 3120, 'n_tokens_neox': 2975, 'n_words': 822, 'path': 'notebooks/CL.vocab-filtering.ipynb', 'repo_name': 'sandeepsoni/semantic-progressiveness', 'size': '9476'}