<jupyter_start><jupyter_text>NLP Commonsense - Assignment 1This file contains the code to generate the evaluations presented and explained in the report.<jupyter_code>%load_ext autoreload
%autoreload 2

import itertools

import pandas as pd
from tqdm import tqdm

from process_examples import load_examples, extract_terms_from_example
from utils import load_conceptnet, normalize_conceptnet, normalize_input
from find_shortest_path import find_word_path, render_path_verbose, search_shortest_path
examples = load_examples() # load examples.txt in machine-readable form
conceptnet = load_conceptnet() # load preprocessed ConceptNet pickle<jupyter_output><empty_output><jupyter_text>Extract Terms for Different Examples<jupyter_code>extract_terms_from_example(examples[0])
extract_terms_from_example(examples[2])<jupyter_output><empty_output><jupyter_text>Investige Path Search With Different Parametrs<jupyter_code>find_word_path("safe", "baggage", conceptnet)
# example for a path with length 4, but runs too long and is not really meaningful
find_word_path("safe", "baggage", conceptnet, max_path_len=4)
find_word_path("airport", "baggage", conceptnet)
find_word_path("airport", "baggage", conceptnet, renderer=render_path_verbose)
find_word_path("checked", "garbage", conceptnet, max_path_len=4)
find_word_path("safe", "baggage", conceptnet, max_path_len=4)
find_word_path("punish", "away", conceptnet, max_path_len=4)
find_word_path("excited", "apply", conceptnet, max_path_len=4)
find_word_path("harvard", "letter", conceptnet, max_path_len=4)<jupyter_output><empty_output><jupyter_text>Extract Paths for All Examples<jupyter_code>results = []

for example_idx, example in enumerate(tqdm(examples), start=1):
    question_context, choices = extract_terms_from_example(example)

    for tq in question_context:
        for tc in choices:
            p = find_word_path(tq, tc, conceptnet)

            results.append(
                {
                    "example": example_idx,
                    "question_context": tq,
                    "choices": tc,
                    "path": p,
                }
            )

path_df = pd.DataFrame(results)
path_df
path_df.index.name = "index"
path_df.to_csv("../data/processed/paths_for_examples_after_article_fix.csv")<jupyter_output><empty_output><jupyter_text>Additional EvaluationsThe following evaluations were not directly parts of the assignments tasks, but provided useful additional information.  Investigate Suffixes of Node Labels<jupyter_code>df = pd.read_csv("../data/processed/en_edges.csv")
"/c/en/0/n".split("/")
df.head()
set(parts[-1] for name in df.start.unique() if len(parts := name.split("/")) == 5)<jupyter_output><empty_output><jupyter_text>Create a List of All Unmatched Terms<jupyter_code>oov_list = []

for example_idx, example in enumerate(examples, start=1):
    question_context, choices = extract_terms_from_example(example)

    oov = [t for t in itertools.chain(question_context, choices) if normalize_input(t) not in conceptnet.nodes_name2idx]

    oov_list.append({"example": example_idx, "out_of_vocab": ",".join(oov)})



oov_df = pd.DataFrame(oov_list)
oov_df.to_csv("../data/processed/oov_after_article_fix.csv")
oov_df<jupyter_output><empty_output><jupyter_text>Create a List of the Most Common Relations<jupyter_code>df = pd.read_csv("../data/processed/en_edges.csv")
df.label.value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1048623315758641, 'fraction_numerical': 0.00497949619214997, 'mean_word_length': 5.735700197238659, 'pattern_counts': {'":': 6, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '273778', 'license': 'MIT', 'n_tokens_mistral': 1160, 'n_tokens_neox': 1106, 'n_words': 237, 'path': 'notebooks/Assignment1.ipynb', 'repo_name': 'felixbmuller/nlp-commonsense', 'size': '3414'}