<jupyter_start><jupyter_text>Notebook performs TF-IDF on BR24 data set to create a baseline system  /data/processed_data_extra_columns.pkl is used for this notebook  Results are not saved to a .pkl file<jupyter_code>import pickle as pk
import pandas as pd

from tqdm import tqdm
tqdm.pandas()
df = pd.read_pickle("../data/processed_data_extra_columns.pkl", compression='zip')
df.shape
df.head(1)
from sklearn.feature_extraction.text import TfidfVectorizer 
tfidf_vectorizer = TfidfVectorizer(use_idf=True, smooth_idf=True, lowercase=False) 
import difflib<jupyter_output><empty_output><jupyter_text>1. TDIDF stemmer -> all predictions<jupyter_code>def TFIDF_stemmer_all(row):
    tfidf_matrix = tfidf_vectorizer.fit_transform([row.clean_text_tfidf_nltk_stem])
    features_df = pd.DataFrame(tfidf_matrix.T.todense(), index=tfidf_vectorizer.get_feature_names(), columns=["tfidf"])
    return (features_df.sort_values(by=["tfidf"], ascending=False).index.values)
df['TFIDF_stemmer_all'] = df.progress_apply(TFIDF_stemmer_all, axis=1)<jupyter_output>100%|██████████| 40862/40862 [01:47<00:00, 381.36it/s]<jupyter_text>2. TFIDF lemmatizer -> all predictions<jupyter_code>def TFIDF_lemmatizer_all(row):
    tfidf_matrix = tfidf_vectorizer.fit_transform([row.clean_text_tfidf_spacy_lemma])
    features_df = pd.DataFrame(tfidf_matrix.T.todense(), index=tfidf_vectorizer.get_feature_names(), columns=["tfidf"])
    return (features_df.sort_values(by=["tfidf"], ascending=False).index.values)
df['TFIDF_lemmatizer_all'] = df.progress_apply(TFIDF_lemmatizer_all, axis=1)<jupyter_output>100%|██████████| 40862/40862 [01:45<00:00, 388.94it/s]<jupyter_text>3. TFIDF stemmer -> top 30 predictions<jupyter_code>def TFIDF_stemmer_30(row):
    tfidf_matrix = tfidf_vectorizer.fit_transform([row.clean_text_tfidf_nltk_stem])
    features_df = pd.DataFrame(tfidf_matrix.T.todense(), index=tfidf_vectorizer.get_feature_names(), columns=["tfidf"])
    return (features_df.sort_values(by=["tfidf"], ascending=False).index.values[:30])
df['TFIDF_stemmer_30'] = df.progress_apply(TFIDF_stemmer_30, axis=1)<jupyter_output>100%|██████████| 40862/40862 [01:45<00:00, 387.69it/s]<jupyter_text>4. TFIDF lemmatizer -> top 30 predictions<jupyter_code>def TFIDF_lemmatizer_30(row):
    tfidf_matrix = tfidf_vectorizer.fit_transform([row.clean_text_tfidf_spacy_lemma])
    features_df = pd.DataFrame(tfidf_matrix.T.todense(), index=tfidf_vectorizer.get_feature_names(), columns=["tfidf"])
    return (features_df.sort_values(by=["tfidf"], ascending=False).index.values[:30])
df['TFIDF_lemmatizer_30'] = df.progress_apply(TFIDF_lemmatizer_30, axis=1)<jupyter_output>100%|██████████| 40862/40862 [01:45<00:00, 388.59it/s]<jupyter_text>Finding matches<jupyter_code>def find_matches(truth, predictions):
    """
    Functions compares the truth value (editor tags) and predictions made by the system using the difflib library.
    1. A prediction is considered as a match if it a close match and scores at least 0.7 with an editor tag using the difflib.get_close_matches
    2. If no match is found then we look for a substring match.
    
    If both steps do not give a match, then there is no prediction for that particular editor tag.
    
    :param truth: editor tags
    :param predictions: predictions made by the system
    :return: those predictions which are correct, i.e., match with editor predictions
    """
    
    matches = []
    for tag in truth:
        predicted = difflib.get_close_matches(tag, predictions, 1, 0.7) #closest match
        if not predicted:
            predicted = [key for key in predictions if tag in key] #if no closest match then take substring match
        
        if predicted:
            matches.append(predicted[0])
        else:
            continue
    return matches<jupyter_output><empty_output><jupyter_text>Finding matches between all TF-IDF predictions and editor tags<jupyter_code>df['TFIDF_stemmer_all_x_tags'] = df.progress_apply(lambda x: find_matches(x['tags'], x['TFIDF_stemmer_all']), axis=1)
df['TFIDF_lemmatizer_all_x_tags'] = df.progress_apply(lambda x: find_matches(x['tags'], x['TFIDF_lemmatizer_all']), axis=1)
TFIDF_stemmer_all_x_tags_acc = df.apply(lambda row: int(((len(row.TFIDF_stemmer_all_x_tags)/len(row.tags) if len(row.tags) != 0 else 0 ))*100), axis=1)
TFIDF_lemmatizer_all_x_tags_acc = df.apply(lambda row: int(((len(row.TFIDF_lemmatizer_all_x_tags)/len(row.tags) if len(row.tags) != 0 else 0 ))*100), axis=1)
print(TFIDF_stemmer_all_x_tags_acc.mean())
print(TFIDF_lemmatizer_all_x_tags_acc.mean())<jupyter_output>55.79692623953796
66.98316284078116<jupyter_text>Finding matches between all TF-IDF predictions and in_text tags<jupyter_code>df['TFIDF_stemmer_all_x_in_text'] = df.progress_apply(lambda x: find_matches(x['in_text'], x['TFIDF_stemmer_all']), axis=1)
df['TFIDF_lemmatizer_all_x_in_text'] = df.progress_apply(lambda x: find_matches(x['in_text'], x['TFIDF_lemmatizer_all']), axis=1)
TFIDF_stemmer_all_x_in_text_acc = df.apply(lambda row: int(((len(row.TFIDF_stemmer_all_x_in_text)/len(row.in_text) if len(row.in_text) != 0 else -1 ))*100), axis=1)
TFIDF_lemmatizer_all_x_in_text_acc = df.apply(lambda row: int(((len(row.TFIDF_lemmatizer_all_x_in_text)/len(row.in_text) if len(row.in_text) != 0 else -1 ))*100), axis=1)
print(TFIDF_stemmer_all_x_in_text_acc[TFIDF_stemmer_all_x_in_text_acc >= 0].mean())
print(TFIDF_lemmatizer_all_x_in_text_acc[TFIDF_lemmatizer_all_x_in_text_acc >= 0].mean())<jupyter_output>76.7309516750667
90.401571301512<jupyter_text>Finding matches between top 30  TF-IDF predictions and editor tags<jupyter_code>df['TFIDF_stemmer_30_x_tags'] = df.progress_apply(lambda x: find_matches(x['tags'], x['TFIDF_stemmer_30']), axis=1)
df['TFIDF_lemmatizer_30_x_tags'] = df.progress_apply(lambda x: find_matches(x['tags'], x['TFIDF_lemmatizer_30']), axis=1)
TFIDF_stemmer_30_x_tags_acc = df.apply(lambda row: int(((len(row.TFIDF_stemmer_30_x_tags)/len(row.tags) if len(row.tags) != 0 else 0 ))*100), axis=1)
TFIDF_lemmatizer_30_x_tags_acc = df.apply(lambda row: int(((len(row.TFIDF_lemmatizer_30_x_tags)/len(row.tags) if len(row.tags) != 0 else 0 ))*100), axis=1)
print(TFIDF_stemmer_30_x_tags_acc.mean())
print(TFIDF_lemmatizer_30_x_tags_acc.mean())<jupyter_output>31.29151779159121
35.13039009348539<jupyter_text>Finding matches between top 30 TF-IDF predictions and in_text tags<jupyter_code>df['TFIDF_stemmer_30_x_in_text'] = df.progress_apply(lambda x: find_matches(x['in_text'], x['TFIDF_stemmer_30']), axis=1)
df['TFIDF_lemmatizer_30_x_in_text'] = df.progress_apply(lambda x: find_matches(x['in_text'], x['TFIDF_lemmatizer_30']), axis=1)
TFIDF_stemmer_30_x_in_text_acc = df.apply(lambda row: int(((len(row.TFIDF_stemmer_30_x_in_text)/len(row.in_text) if len(row.in_text) != 0 else -1 ))*100), axis=1)
TFIDF_lemmatizer_30_x_in_text_acc = df.apply(lambda row: int(((len(row.TFIDF_lemmatizer_30_x_in_text)/len(row.in_text) if len(row.in_text) != 0 else -1 ))*100), axis=1)
print(TFIDF_stemmer_30_x_in_text_acc[TFIDF_stemmer_30_x_in_text_acc >= 0].mean())
print(TFIDF_lemmatizer_30_x_in_text_acc[TFIDF_lemmatizer_30_x_in_text_acc >= 0].mean())<jupyter_output>45.15268307144975
49.81329676845538<jupyter_text>Sample article<jupyter_code>df.iloc[0].clean_text
df.iloc[0].TFIDF_lemmatizer_30
df.iloc[0].tags<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13028602709730394, 'fraction_numerical': 0.052141781853017655, 'mean_word_length': 7.4, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '537273', 'license': 'MIT', 'n_tokens_mistral': 3182, 'n_tokens_neox': 2902, 'n_words': 458, 'path': 'notebooks/3_tfidf.ipynb', 'repo_name': 'tiwari-jyoti1808/BR24-auto-tagging', 'size': '7307'}