<jupyter_start><jupyter_text><jupyter_code># Import Dependencies
from gensim.downloader import load
from gensim.models import word2vec
from gensim.similarities import SoftCosineSimilarity
import pandas as pd
import csv
import random # XD RaWR L0L im so R4nDOM
# Grab dataset csv
!wget https://raw.githubusercontent.com/m-triassi/ai-projects-472/main/Project%203/synonyms.csv?token=ACIA3M7EM7SLE5PLCHH3CRTBWZZ2S -O synonyms.csv
headers = ["question","answer","A","B","C","D"]
synonyms = pd.read_csv("/content/synonyms.csv", header=None, names=headers)[1:]
# Define similarity run method
# accepts model to be used
def similarity_all(model: word2vec) -> list:
  details = []
  ## iterates over synonyms
  for s in synonyms.iterrows():
    ## Catch if the word is not present, and perfrom a guess.
    guesses = {}
    label = ""
    guess = ""
    if not s[1]["question"] in model.vocab:
      guess = s[1][random.choice(["A", "B", "C", "D"])]
      label = "guess"
  
    for q in ["A", "B", "C", "D"]:
      try:
        guesses[q] = (model.similarity(s[1]["question"], s[1][q]))
      except KeyError:
        continue

    if guesses:
      # sort the guesses by their value
      max_key = max(guesses, key=guesses.get)
      guess = s[1][max_key]
    elif (label is not "guess"):
      # none of the guess-words were in the vocab, just guess something..
      guess = model.most_similar(positive=s[1]["question"], topn=1)[0]
      label = "guess"
    
    if label is not "guess" and guess == s[1]["answer"]:
      label = "correct"
    elif label is not "guess":
      label = "wrong"

    # build the details array
    details.append([s[1]["question"], s[1]["answer"], guess, label])
  
  return details

# Build analysis for a particular run function
## could just read the details csv and generate a new line in analysis.csv
def analyze_details(model: word2vec, details: pd.DataFrame, model_name):
  analysis = []
  correct_count = 0
  guess_count = 0
  for d in details.iterrows():
    correct_count += d[1]['label'] == "correct"
    guess_count += d[1]['label'] == "guess"

  V = 80 - guess_count
  analysis_header = ['model_name', 'vocab_size', 'C', 'V', 'accuracy']
  analysis_data = [model_name, len(model.vocab), correct_count, V, correct_count/V]
  pd.DataFrame([analysis_data], columns=analysis_header).to_csv('analysis.csv', mode="a", header=False)
def generate_reports(model: word2vec, label):
  details = similarity_all(model)
  detail_headers = ["question-word", "answer-word", "guess-word", "label"]
  detail_data = pd.DataFrame(details, columns=detail_headers)

  detail_data.to_csv(f"{label}-details.csv")
  analyze_details(model, detail_data, label)<jupyter_output><empty_output><jupyter_text>Task 1Load the Word2Vec google news pretrained model, evaluate performance using gold standard test.<jupyter_code># Load google-news-300 pre-trained model
model = load('word2vec-google-news-300')
# Test Word2Vec model, save details, generate analysis
generate_reports(model, "word2vec-google-news-300")<jupyter_output><empty_output><jupyter_text>Task 2Repeat analysis for 4 more models. 2 with same embedding but different corpus, and 2 with differnt embedding but same corpus.Leverage functions written for task 1.Models: - glove-twitter-50 vs glove-wiki-gigaword-50- glove-wiki-gigaword-100 vs glove-wiki-gigaword-300<jupyter_code>model = load('glove-twitter-50')
# Test Word2Vec model, save details, generate analysis
generate_reports(model, "glove-twitter-50")
model = load('glove-wiki-gigaword-50')
# Test Word2Vec model, save details, generate analysis
generate_reports(model, "glove-wiki-gigaword-50")
model = load('glove-wiki-gigaword-300')
# Test Word2Vec model, save details, generate analysis
generate_reports(model, "glove-wiki-gigaword-300")
model = load("glove-wiki-gigaword-100")
# Test Word2Vec model, save details, generate analysis
generate_reports(model, "glove-wiki-gigaword-100")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13225234355206486, 'fraction_numerical': 0.022548771218647075, 'mean_word_length': 4.680575539568346, 'pattern_counts': {'":': 1, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '589462', 'license': 'MIT', 'n_tokens_mistral': 1393, 'n_tokens_neox': 1326, 'n_words': 345, 'path': 'Project 3/Project3.ipynb', 'repo_name': 'm-triassi/ai-projects-472', 'size': '3947'}