<jupyter_start><jupyter_text>1. Baselines 2. LogReg with target + annotator 3. LogReg with context + annotator 4. LogReg with target + context + similarity + annotator 5. & 6. & 7.: kNN with 2., 3. and 4.  DONE: LogReg with Annotators - group by annotators -> take majority vote across contexts - add annotator (categorical variable) as a predictor   SVD Grid Search - reduce tf-idf space to > 100 dims (256?) = M \in V x 256 - loop over dimensions k and produce N \n V x k from M - fit LogReg on N    DONE: use Word2Vec embeddings DONE: split train-test by target words - measure accuracy per annotator, then average (happens by default) - prediction for target is average      DONE: adapt baseline - old: majority vote on the most common class *across all samples*  - new: majority vote on most common class *per annotator* -> annotator prior P(c|a)---=> P(c | w, a) = P(c | (x_1, ..., x_k), a)<jupyter_code>from tqdm import tqdm
import joblib
from collections import Counter

import pandas as pd
import numpy as np


import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.preprocessing import StandardScaler

from utils import get_DF, get_indices

from sklearn.linear_model import LogisticRegression, LogisticRegressionCV
from sklearn.model_selection import GridSearchCV, cross_validate

from sklearn.metrics import balanced_accuracy_score, classification_report, accuracy_score,\
                                    confusion_matrix, plot_roc_curve, precision_score, recall_score,\
                                    f1_score

df = get_DF()
tfidf = joblib.load(f'vectors/tfidf_0.15.pkl')
mat = pd.read_csv("vectors/svd_mat_0.15.tsv", delimiter="\t", header=None).to_numpy()
scaled = StandardScaler(with_mean=False).fit_transform(mat)
tfidf_words = tfidf.get_feature_names()
mat_inds = get_indices(df.target.unique(), tfidf_words)
df["mat_ind"] = df.target.apply(lambda w: mat_inds[w] if w in mat_inds else -1)
df = df[df.mat_ind > -1]
target_annotations = {x:list(subdf.y) for x, subdf in df.groupby(["annotator_x", "target"])}
xs, y = list(zip(*target_annotations.items()))
x_anno, x_target = list(zip(*xs))

mldf = pd.DataFrame([x_anno, x_target, y]).T
mldf.columns = ["x_anno", "target", "y_ls"]

mldf["ratio"] = mldf.y_ls.apply(lambda ls: sum(ls)/len(ls))
mldf = mldf[mldf.ratio != 0.5]
mldf["maj_vote"] = mldf.ratio.apply(lambda r: int(r > 0.5))

mldf = pd.concat([mldf, pd.get_dummies(mldf["x_anno"])], axis=1)<jupyter_output><empty_output><jupyter_text>---<jupyter_code>X = np.hstack([np.stack(mldf.target.apply(lambda w: scaled[mat_inds[w]])), 
               mldf.iloc[:, 5:].to_numpy()])
Y = mldf.maj_vote.to_numpy()
logregcv = LogisticRegression(penalty="elasticnet", max_iter=5000, solver="saga")
logregcv = LogisticRegression(penalty="none", max_iter=3000, solver="lbfgs")

prec = lambda x, y: precision_score(x, y, average=None)
rec = lambda x, y: recall_score(x, y, average=None)

scores = cross_validate(
            logregcv,
            X,
            Y,
            cv=30,
            n_jobs=8,
            scoring=['accuracy',
                     'balanced_accuracy',
                     'f1_weighted',
                     'precision_weighted',3
                     'recall_weighted', "roc_auc"],
            groups=mldf.target,
            verbose=1
        )  # parallel processing

scores
sns.distplot([0.71486957, 0.70836237, 0.70410114, 0.6949289 , 0.7141457 ])
sns.distplot([0.76886035, 0.74371322, 0.7693954 , 0.78491172, 0.76445396])
import scipy.stats as st

bal_acc = scores["test_balanced_accuracy"]
acc = scores["test_accuracy"]


print("balanced acc:", st.t.interval(0.95, len(bal_acc)-1, loc=np.mean(bal_acc), scale=st.sem(bal_acc)))
print("accuracy    :", st.t.interval(0.95, len(acc)-1, loc=np.mean(acc), scale=st.sem(acc)))
for k, v in scores.items():
    print(k, "\t", tuple(map(lambda x: round(x, 3), st.t.interval(0.95, len(v)-1, loc=np.mean(v), scale=st.sem(v)))))
import sklearn
sorted(sklearn.metrics.SCORERS.keys())<jupyter_output><empty_output><jupyter_text>---<jupyter_code># hand-made confidence intervals

bal_accs = []
accs = []

for _ in tqdm(range(20)):
    words_shuffled = np.random.permutation(mldf.target.unique())

    ratio = int(mldf.target.unique().shape[0]*0.8)
    train_words = set(words_shuffled[:ratio])
    msk = mldf.target.isin(train_words)

    # msk = np.random.rand(len(mldf)) < 0.7

    train_df = mldf[msk]
    test_df = mldf[~msk]


    target_train, anno_train, Y_train = (np.stack(train_df.target.apply(lambda w: scaled[mat_inds[w]])), 
                                         train_df.iloc[:, 5:].to_numpy(), 
                                         train_df.maj_vote.to_numpy())

    target_test, anno_test, Y_test = (np.stack(test_df.target.apply(lambda w: scaled[mat_inds[w]])),
                                        test_df.iloc[:, 5:].to_numpy(),
                                        test_df.maj_vote.to_numpy())

    X_train, X_test = np.hstack([target_train, anno_train]), np.hstack([target_test, anno_test])
    
    
    
    
    logreg = LogisticRegression(penalty="none", max_iter=5000, solver="lbfgs")
    logreg.fit(X_train, Y_train)
    
    
    
    # Logistic Regression


    preds = logreg.predict(X_test)

    print(classification_report(Y_test, preds))
    b = balanced_accuracy_score(Y_test, preds)
    a = accuracy_score(Y_test,
                       
                       preds)
    bal_accs.append(b)
    accs.append(a)


    cm = confusion_matrix(Y_test, preds)

    print(cm)

    plot_roc_curve(logreg, X_test, Y_test)
    _=plt.plot([0,1], [0,1], "--", c="r")<jupyter_output><empty_output><jupyter_text>---<jupyter_code># Logistic Regression

from sklearn.metrics import balanced_accuracy_score, classification_report, accuracy_score,\
                                confusion_matrix, plot_roc_curve

preds = logreg.predict(X_test)

print(classification_report(Y_test, preds))
print(balanced_accuracy_score(Y_test, preds))
print(accuracy_score(Y_test, preds))


cm = confusion_matrix(Y_test, preds)

print(cm)

plot_roc_curve(logreg, X_test, Y_test)
_=plt.plot([0,1], [0,1], "--", c="r")
sns.distplot(bal_accs)

sns.distplot(accs)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1346774193548387, 'fraction_numerical': 0.027580645161290322, 'mean_word_length': 3.3576950105411103, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '434771', 'license': 'CC-BY-2.0', 'n_tokens_mistral': 2386, 'n_tokens_neox': 2223, 'n_words': 458, 'path': 'K-Cap_2021/regressions/logreg.ipynb', 'repo_name': 'cultural-ai/ConConCor', 'size': '6200'}