<jupyter_start><jupyter_text>Bio vs. non-bio<jupyter_code>results = {}
for filename in ['predictions-without-mobius.xls', 'predictions-with-mobius.xls']:
    df = pd.read_excel(filename)
    gt_label_col = 'Actual_number (0 index)'
    pred_label_col = 'Predicted_number (0 index)'

    correctness_col = 'correctness'
    df[correctness_col] = df[gt_label_col] == df[pred_label_col]

    # print(df.head())

    # These classes in the mapping are from the column Actual labels category
    gt_label_name_col = 'Actual labels category'
    class2biomap =   { 
                        'aquatic mammals': True,
                        'fish': True,
                        'flowers': True, # 
                        'food containers': False, # 
                        'fruit and vegetables': True, # 
                        'household electrical device': False,
                        'household furniture': False,                    
                        'insects': True,
                        'large carnivores': True,
                        'large man-made outdoor things': False,
                        'large natural outdoor scenes': False, # 
                        'large omnivores and herbivores': True,
                        'non-insect invertebrates': True,
                        'medium-sized mammals': True,
                        'people': True,
                        'reptiles': True,
                        'small mammals': True,
                        'trees': True, # 
                        'vehicles 1': False,
                        'vehicles 2': False,
                     }

    correctness_dict = defaultdict(list)
    class_correctness_dict = defaultdict(list)
    for i, row in df.iterrows():
        cl = row[gt_label_name_col]
        corr = row[correctness_col]
        if class2biomap[cl]:
            # Biological
            correctness_dict['bio'].append(corr)
        else:
            # Non-biological
            correctness_dict['nonbio'].append(corr)
        class_correctness_dict[cl].append(corr)

    for b, corr_list in correctness_dict.items():
        print(f'{b}: {len(corr_list)} examples, with mean {np.mean(corr_list)} std dev {np.std(corr_list)}')

    for cls, corr_list in class_correctness_dict.items():
        print(f'{cls}: {len(corr_list)} examples, with mean {np.mean(corr_list)} std dev {np.std(corr_list)}')

    results[filename] = []
    results[filename].append(correctness_dict)
    results[filename].append(class_correctness_dict)
vanilla= results['predictions-without-mobius.xls'][1]
mobius = results['predictions-with-mobius.xls'][1]
mobius_acc = {k: np.mean(corr_list) for k, corr_list in mobius.items()}
vanilla_acc = {k: np.mean(corr_list) for k, corr_list in vanilla.items()}
print("category", "bio-increase?", "vanilla", "mobius")
class2biomap
bio_mobius_acc = 0
for k in sorted(mobius_acc, key=lambda x: x[1]):
    bio_mobius_acc += int((mobius_acc[k]-vanilla_acc[k] > 0) == class2biomap[k])
    print(k, (mobius_acc[k]-vanilla_acc[k] > 0) == class2biomap[k], vanilla_acc[k], mobius_acc[k])
    
print(bio_mobius_acc/20)<jupyter_output>category bio-increase? vanilla mobius
large natural outdoor scenes False 0.852 0.872
large omnivores and herbivores True 0.844 0.848
large carnivores True 0.838 0.844
large man-made outdoor things True 0.898 0.896
medium-sized mammals True 0.826 0.852
vehicles 2 True 0.918 0.902
reptiles True 0.738 0.772
vehicles 1 False 0.886 0.908
people True 0.646 0.668
fish True 0.818 0.824
flowers False 0.86 0.856
small mammals True 0.748 0.79
insects True 0.85 0.858
household electrical device False 0.864 0.87
food containers False 0.796 0.806
household furniture True 0.86 0.848
non-insect invertebrates True 0.814 0.82
aquatic mammals True 0.73 0.744
fruit and vegetables True 0.862 0.896
trees False 0.758 0.744
0.7
{'fraction_non_alphanumeric': 0.10879750130140552, 'fraction_numerical': 0.045288912024986985, 'mean_word_length': 2.347560975609756, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '99900', 'license': 'ECL-2.0', 'n_tokens_mistral': 1376, 'n_tokens_neox': 1177, 'n_words': 306, 'path': 'mobius_data_augmentation/notebooks/Mobius-analysis.ipynb', 'repo_name': 'trestripes-com/mobius', 'size': '3842'}