<jupyter_start><jupyter_text>Evaluation Program<jupyter_code>import pandas as pd

def all_metrics(tt,pp):
    #prf = precision_recall_fscore_support(tt,pp, average='micro', labels=np.unique(tt))[0:3]
    acc = accuracy_score(tt,pp)
    return acc #np.hstack([prf, acc])

all_metrics_names = ["Acc"]
def make_classic_classifier():
    return make_pipeline(
    MaxAbsScaler(),
    LogisticRegression(C=1, dual=False, penalty="l2")
)

def make_highdim_classifier():
    return make_pipeline(
        StandardScaler(),
        sklearn.svm.SVC(C=1.0, probability=True)
    )


CL_mdl = lambda: MLCharacterSolver(make_classic_classifier(), nicknames2name_comb)
WE_mdl = lambda: MLCharacterSolver(make_highdim_classifier(), nicknames2name_comb, get_embedding_features)


FM_mdl = lambda: FirstMentionedSolver(nicknames2name_comb)
MC_mdl = lambda: MostMentionedSolver(nicknames2name_comb)

datasets = [("WOT", ann_WOT), ("ASOIAF", ann_ASOIAF), ("SOC", ann_SOC)]
supdatasets = [("SA", ann_SA)]
base_mdls = [("ML Classical Features", CL_mdl),
             ("ML Word Emb. Features", WE_mdl),
             ("First Mentioned", FM_mdl),
             ("Most Commonly Mentioned", MC_mdl)
       ]<jupyter_output><empty_output><jupyter_text>main eval<jupyter_code>def make_program(datasets, mdls):
    program = OrderedDict()    
    for (test_data_name, test_data),(mdl_name,mdl) in it.product(datasets, mdls):
        if mdl_name[0:2]=="ML":
            for (train_data_name, train_data) in datasets:
                if train_data_name==test_data_name:
                    continue
                program[(test_data_name,mdl_name, train_data_name)] = (
                    train_data,
                    test_data,
                    mdl()
                )
        else:
            program[(test_data_name, mdl_name, "---")] = ([], test_data, mdl())
    return program

program = make_program(datasets, base_mdls)


res = pd.DataFrame(index=pd.MultiIndex.from_tuples(program.keys()),
                   columns = all_metrics_names)
res.sort_index(inplace=True)

for ind in res.index:
    print(ind, end="")
    score = evaluate(*program[ind], metric=all_metrics)
    res.loc[ind,:] = score
    print(" ", score)
    res.to_csv("../resulthtos/maineval.csv", index_label=["Test Set", "Method", "Train Set"])
    
res
xs = [1,2,3]
np.random.shuffle(xs)
xs<jupyter_output><empty_output><jupyter_text>Combining training data<jupyter_code>def make_program(datasets, mdls):
    program = OrderedDict()    
    for (test_data_name, test_data),(mdl_name,mdl) in it.product(datasets, mdls):
        if mdl_name[0:2]!="ML":
            continue
        full_train_data = []
        full_train_data_names = []
        for (train_data_name, train_data) in datasets:
            if train_data_name==test_data_name:
                continue
            full_train_data_names.append(train_data_name)
            full_train_data.extend(train_data)
            
        np.random.shuffle(full_train_data)
        program[(test_data_name, mdl_name, "+".join(full_train_data_names))] = (
            full_train_data,
            test_data,
            mdl()
        )
    return program

program = make_program(datasets, base_mdls)


res_comb = pd.DataFrame(index=pd.MultiIndex.from_tuples(program.keys()),
                   columns = all_metrics_names)
res_comb.sort_index(inplace=True)

for ind in res_comb.index:
    print(ind, end="")
    score = evaluate(*program[ind], metric=all_metrics)
    res_comb.loc[ind,:] = score
    print(" ", score)
    res_comb.to_csv("../results/combeval.csv", index_label=["Test Set", "Method", "Train Set"])
    
res_comb<jupyter_output>('ASOIAF', 'ML Classical Features', 'WOT+SOC')  0.9765625
('ASOIAF', 'ML Word Emb. Features', 'WOT+SOC')  0.97265625
('SOC', 'ML Classical Features', 'WOT+ASOIAF')  0.9340659340659341
('SOC', 'ML Word Emb. Features', 'WOT+ASOIAF')<jupyter_text>Cross EvaluationTo test how much it effects things from different styles.<jupyter_code>def make_program(datasets, mdls):
    program = dict()    
    for (data_name, data),(mdl_name,mdl) in it.product(datasets, mdls):
        program[(data_name, mdl_name)] = (data, mdl())
    return program
program = make_program(datasets+[("Combined", ann_comb)], base_mdls)


res_xval = pd.DataFrame(index=pd.MultiIndex.from_tuples(program.keys()),
                        columns = all_metrics_names)
res_xval.sort_index(inplace=True)

for ind in res_xval.index:
    print(ind, end="")
    score = xval_evaluate(*program[ind], metric=all_metrics) 
    res_xval.loc[ind, :] = score
    print(" ", score)
    res_xval.to_csv("../results/crosseval.csv", index_label=["Dataset", "Method"])
    
res_xval<jupyter_output><empty_output><jupyter_text>Supp dataSA  ground truth is really weak.It is for a chapter which has maybe 4 scenses only 1-2 of which will actually be about that character<jupyter_code>def make_program(datasets, supdatasets, mdls):
    all_datasets = list(datasets)
    all_datasets.extend(supdatasets)
    
    program = OrderedDict()    
    for (test_data_name, test_data),(mdl_name,mdl) in it.product(datasets, mdls):
        if mdl_name[0:2]=="ML":
            combined_data = []
            combined_data_names = []
            for (train_data_name, train_data) in all_datasets:
                if train_data_name==test_data_name:
                    continue
                combined_data.append(train_data)
                combined_data_names.append(train_data_name)
            
            if len(combined_data) > 1:
                train_data_name = " and ".join(combined_data_names)
                program[(test_data_name, mdl_name, train_data_name)] = (
                    np.hstack(combined_data),
                    test_data,
                    mdl()
                )
    return program

program = make_program(datasets,supdatasets, base_mdls)


res = pd.DataFrame(index=pd.MultiIndex.from_tuples(program.keys()),
                   columns = all_metrics_names)
res.sort_index(inplace=True)

for ind in res.index:
    print(ind, end="")
    
    score = evaluate(*program[ind], metric=all_metrics)
    res.loc[ind,:] = score
    print(" ", score)
    res.to_csv("../results/extradata.csv", index_label=["Test Set", "Method", "Train Set"])
    
res<jupyter_output><empty_output><jupyter_text>Sanity check WOT<jupyter_code>the_CL_mdl = joblib.load("../trained_models/CL.pkl")
the_WE_mdl = joblib.load("../trained_models/WE.pkl")
texts, ref_chars = extract_texts_and_characters(ann_WOT)
out_chars =  np.asarray(list(the_WE_mdl.choose_characters(texts)))
np.asarray(list(zip(out_chars, ref_chars)))[out_chars != ref_chars]<jupyter_output><empty_output><jupyter_text>Training Set accurasy<jupyter_code>def make_program(datasets, mdls):
    program = OrderedDict()    
    for (test_data_name, test_data),(mdl_name,mdl) in it.product(datasets, mdls):
        if mdl_name[0:2]!="ML":
            continue
        program[(test_data_name,mdl_name, test_data_name)] = (
            test_data,
            test_data,
            mdl()
        )
    return program

program = make_program(datasets, base_mdls)

res_train = pd.DataFrame(index=pd.MultiIndex.from_tuples(program.keys()),
                   columns = all_metrics_names)
res_train.sort_index(inplace=True)

for ind in res_train.index:
    print(ind, end="")
    score = evaluate(*program[ind], metric=all_metrics)
    res_train.loc[ind,:] = score
    print(" ", score)
    res_train.to_csv("../results/traineval.csv", index_label=["Test Set", "Method", "Train Set"])
    
res_train<jupyter_output>('ASOIAF', 'ML Classical Features', 'ASOIAF')  0.98046875
('ASOIAF', 'ML Word Emb. Features', 'ASOIAF')  0.98828125
('SOC', 'ML Classical Features', 'SOC')  0.945054945054945
('SOC', 'ML Word Emb. Features', 'SOC')  0.9560439560439561
('WOT', 'ML Classical Features', 'WOT')  0.7847222222222222
('WOT', 'ML Word Emb. Features', 'WOT')  0.7939814814814815<jupyter_text>Feature importance stuff<jupyter_code>def feature_importance(mdl):
    _, _,vector_keys = get_feature_vectors(ann_comb[1]['text'])
    feature_weights = list(zip(mdl.classifier.feature_importances_,vector_keys))
    feature_weights.sort(reverse=True)
    non_zero_weights = [(weight,name) for weight, name in feature_weights if weight>0]
    print("Number of nonzeo weights: ", len(non_zero_weights))
    print("\n".join(", ".join(map(str,wt)) for wt in non_zero_weights))

feature_importance(CL_SOC)
feature_importance(HY_SOC)
feature_importance(CL_ASOIAF)
feature_importance(HY_ASOIAF)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12883220219985958, 'fraction_numerical': 0.01708401591387784, 'mean_word_length': 3.575481798715203, 'pattern_counts': {'":': 4, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '396205', 'license': 'MIT', 'n_tokens_mistral': 3071, 'n_tokens_neox': 2947, 'n_words': 510, 'path': 'proto/Evaluations.ipynb', 'repo_name': 'oxinabox/NovelPerspective', 'size': '8546'}