<jupyter_start><jupyter_text>SQuADv1<jupyter_code>SEEDS = [28, 29, 30]

squadv1_df = pd.DataFrame()

for seed, checkpoint in tqdm(
    list(itertools.product(SEEDS, SAVE_STEPS_SCHEDULE))
):
    
    df = pd.read_json(
        f'predictions/albert-xlarge-v2-squadv1-wu=100-lr=3e5-bs=32-msl=384-seed={seed}/checkpoint-{checkpoint}/eval_results.json',
        orient='index'
    ).T

    df['seed'] = seed
    df['checkpoint'] = checkpoint

    squadv1_df = squadv1_df.append(df, ignore_index=True)
squadv1_df.head()
squadv1_df.rename(columns={'eval_f1': 'F1'}, inplace=True)
squadv1_df['model'] = 'SQuADv1'
squadv1_df.head()<jupyter_output><empty_output><jupyter_text>SQuADv2<jupyter_code>SEEDS = [27, 29, 30]

squadv2_df = pd.DataFrame()

for seed, checkpoint in tqdm(
    list(itertools.product(SEEDS, SAVE_STEPS_SCHEDULE))
):
    
    df = pd.read_json(
        f'predictions/albert-xlarge-v2-squadv2-wu=100-lr=3e5-bs=32-msl=384-seed={seed}/checkpoint-{checkpoint}/eval_results.json',
        orient='index'
    ).T

    df['seed'] = seed
    df['checkpoint'] = checkpoint

    squadv2_df = squadv2_df.append(df, ignore_index=True)
squadv2_df.head()
squadv2_df.rename(columns={'eval_f1': 'F1'}, inplace=True)
squadv2_df['model'] = 'SQuADv2'
squadv2_df.head()<jupyter_output><empty_output><jupyter_text>SQuADv1-AdversarialQA<jupyter_code>SEEDS = [27, 28, 29]
DATASETS = ['squad', 'dbert', 'dbidaf', 'droberta']

squadv1_adqa_df = pd.DataFrame()

for seed, checkpoint, dataset in tqdm(
    list(itertools.product(SEEDS, SAVE_STEPS_SCHEDULE, DATASETS))
):
    
    df = pd.read_json(
        f'predictions/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-{dataset}/checkpoint-{checkpoint}/eval_results.json',
        orient='index'
    ).T

    df['seed'] = seed
    df['checkpoint'] = checkpoint
    df['dataset'] = dataset

    squadv1_adqa_df = squadv1_adqa_df.append(df, ignore_index=True)
squadv1_adqa_df.head()
squadv1_adqa_df.rename(columns={'eval_f1': 'F1'}, inplace=True)
squadv1_adqa_df['F1_sum'] = squadv1_adqa_df['F1'] * squadv1_adqa_df['eval_samples']
squadv1_adqa_df = squadv1_adqa_df[['checkpoint', 'seed', 'eval_samples', 'F1_sum']].groupby(['checkpoint', 'seed']).sum().reset_index()
squadv1_adqa_df.head()
squadv1_adqa_df['F1'] = squadv1_adqa_df['F1_sum'] / squadv1_adqa_df['eval_samples']
squadv1_adqa_df.head()
squadv1_adqa_df.shape
squadv1_adqa_df['model'] = 'SQuADv1-AdQA'
squadv1_adqa_df.head()<jupyter_output><empty_output><jupyter_text>AdversarialQA<jupyter_code>ADQA_CHKPTS = [1, 2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32, 36, 44, 52, 60, 68, 76, 92, 108, 124, 140, 156, 172, 188, 204, 220, 236, 252, 268, 284, 300, 316, 332, 348, 364, 380, 396, 428, 460, 492, 524, 556, 588, 620, 652, 684, 716, 748, 780, 812, 844, 876, 908, 940, 972, 1004, 1036, 1100, 1164, 1228, 1292, 1356, 1420, 1484, 1548, 1612, 1676, 1804, 1932, 2060, 2188, 2316, 2444, 2572, 2700, 2828, 2956, 3084, 3212]
SEEDS = [28, 29, 30]

adqa_df = pd.DataFrame()

for seed, checkpoint in tqdm(
    list(itertools.product(SEEDS, ADQA_CHKPTS))
):
    
    df = pd.read_json(
        f'predictions/albert-xlarge-v2-adversarial_qa_all-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-adversarialQA/checkpoint-{checkpoint}/eval_results.json',
        orient='index'
    ).T

    df['seed'] = seed
    df['checkpoint'] = checkpoint

    adqa_df = adqa_df.append(df, ignore_index=True)
adqa_df['f1_sum'] = adqa_df['eval_f1'] * adqa_df['eval_samples']
adqa_df.head()
adqa_df = adqa_df.groupby(['checkpoint', 'seed']).sum().reset_index()
adqa_df['F1'] = adqa_df['f1_sum'] / adqa_df['eval_samples']
adqa_df.head()
sns.lineplot(data=adqa_df, x='checkpoint', y='F1', hue='seed')
adqa_df = adqa_df[['checkpoint', 'seed', 'F1']]
adqa_df['model'] = 'AdQA'
adqa_df.head()<jupyter_output><empty_output><jupyter_text>MRQA<jupyter_code>BATCH_SIZE = 32
SEEDS = [27, 28, 30]
DATASETS = [
    ('hotpotqa', 'in_sample'),
    ('naturalquestions', 'in_sample'),
    ('newsqa', 'in_sample'),
    ('searchqa', 'in_sample'),
    ('squad', 'in_sample'),
    ('triviaqa', 'in_sample')
]
mrqa_df = pd.DataFrame()

for seed, dataset, checkpoint in tqdm(
    list(itertools.product(SEEDS, DATASETS, SAVE_STEPS_SCHEDULE))
):
    
    df = pd.read_json(
        f'predictions/albert-xlarge-v2-mrqa-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-dataset={dataset[0]}/checkpoint-{checkpoint}/predict_results.json',
        orient='index'
    ).T

    df['seed'] = seed
    df['dataset'] = dataset[0]
    df['checkpoint'] = checkpoint
#     df['sample'] = dataset[1]

    mrqa_df = mrqa_df.append(df, ignore_index=True)
mrqa_df['f1_sum'] = mrqa_df['f1'] * mrqa_df['predict_samples']
mrqa_df.head()
mrqa_df = mrqa_df.groupby(['checkpoint', 'seed']).sum().reset_index()
mrqa_df['F1'] = mrqa_df['f1_sum'] / mrqa_df['predict_samples']
mrqa_df.head()
sns.lineplot(data=mrqa_df, x='checkpoint', y='F1', hue='seed')
mrqa_df = mrqa_df[['checkpoint', 'seed', 'F1']]
mrqa_df['model'] = 'MRQA'
mrqa_df.head()
mrqa_df.shape<jupyter_output><empty_output><jupyter_text>Plot<jupyter_code>all_models_df = pd.concat(
    [df[['checkpoint', 'seed', 'F1', 'model']] for df in [squadv1_df, squadv2_df, squadv1_adqa_df, mrqa_df, adqa_df]]
)
print(all_models_df.shape)
all_models_df.head()
all_models_df['num_examples'] = all_models_df['checkpoint'] * BATCH_SIZE
all_models_df.groupby('model').count()
all_models_df['model'].unique()
fig, axs = plt.subplots(3, 2, sharey=True, figsize=(15, 15))

idx_x, idx_y = 0, 0

for i, model in enumerate(all_models_df['model'].unique()):
    
    plot_df = all_models_df.copy()[all_models_df['model'] == model]
    
    axs[idx_x, idx_y].set_ylim(0, 100)

    sns.lineplot(x='num_examples', y='F1', hue='seed', data=plot_df, ax=axs[idx_x, idx_y], palette='tab10', marker='o')
    
    axs[idx_x, idx_y].set_title(f'{model}')
    axs[idx_x, idx_y].set_xlabel('Number of training examples')
    axs[idx_x, idx_y].set_ylabel('Validation F1')
    axs[idx_x, idx_y].legend()

    if i % 2 == 0:
        idx_y += 1
    else:
        idx_y -= 1
        idx_x += 1

axs[idx_x, idx_y].axis('off')
plt.tight_layout()
plt.savefig('reports/figures/all_model_learning_curves.pdf')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17766661287719865, 'fraction_numerical': 0.06825883492012264, 'mean_word_length': 5.260606060606061, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '332376', 'license': 'MIT', 'n_tokens_mistral': 3057, 'n_tokens_neox': 2717, 'n_words': 394, 'path': 'notebooks/final_plots/13-All model learning curves.ipynb', 'repo_name': 'stevieg3/when-do-reading-comprehension-models-learn', 'size': '6197'}