<jupyter_start><jupyter_text>Confirming accuracyThis notebook is a dead-simple way to check on accuracy figures reported in the article, assuming that you're willing to take the results of the modeling run itself on faith.If you want to reproduce the modeling ruin, you'll need to run **../logistic/main_experiment.py,** using the appropriate command for a given set of genre tags.<jupyter_code>import pandas as pd
def accuracy(probs, groundtruth):
    ''' We assume these two vectors are paired.
    '''
    fn = 0
    tp = 0
    fp = 0
    tn = 0
    for (p, g) in zip(probs, groundtruth):
        if p >= 0.5 and g >= 0.5:
            tp += 1
        elif p >= 0.5 and g < 0.5:
            fp += 1
        elif p < 0.5 and g >= 0.5:
            fn += 1
        elif p < 0.5 and g < 0.5:
            tn += 1
        else:
            print('hmmm ...')
    
    accuracy = (tp + tn) / (tp + tn + fp + fn)
    
    return accuracy

def check_file(filename):
    path = '../modeloutput/' + filename + '.csv'
    
    df = pd.read_csv(path)
    probs = df['probability']
    groundtruth = df['realclass']
    return accuracy(probs, groundtruth)
    
check_file('maxaccuracy_mudies')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1115702479338843, 'fraction_numerical': 0.019834710743801654, 'mean_word_length': 2.2293333333333334, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '25306', 'license': 'MIT', 'n_tokens_mistral': 408, 'n_tokens_neox': 379, 'n_words': 140, 'path': 'interpretations/confirm_accuracy.ipynb', 'repo_name': 'tedunderwood/measureperspective', 'size': '1210'}