<jupyter_start><jupyter_text>MRQA<jupyter_code>ALL_CHECKPOINTS = [1, 2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32, 36, 44, 52, 60, 68, 76, 92, 108, 124, 140, 156, 172, 188, 204, 220, 236, 252, 268, 284, 300, 316, 332, 348, 364, 380, 396, 428, 460, 492, 524, 556, 588, 620, 652, 684, 716, 748, 780, 812, 844, 876, 908, 940, 972, 1004, 1036, 1100, 1164, 1228, 1292, 1356, 1420, 1484, 1548, 1612, 1676, 1804, 1932, 2060, 2188, 2316, 2444, 2572, 2700, 2828, 2956, 3084, 3212, 3340, 3468, 3596, 3724, 3852, 3980, 4108, 4236, 4364, 4492, 4620, 4748, 4876, 5004, 5132, 5260, 5388, 5516, 5644, 5772, 5900, 6028, 6156, 6284, 6412, 6540, 6668, 6796, 6924, 7052, 7180, 7308, 7436, 7564, 7692, 7820, 7948]
len(ALL_CHECKPOINTS)
remaining_chkpts_dict = {}

for seed in [27, 28, 29]:
    for dataset in ['bioasq', 'drop', 'duorc', 'hotpotqa', 'naturalquestions', 'newsqa', 'race', 'relationextraction', 'searchqa', 'squad', 'textbookqa', 'triviaqa']: 
        chkpts = os.listdir(f'predictions/albert-xlarge-v2-mrqa-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-dataset={dataset}')
        chkpts = [int(i.split('-')[-1]) for i in chkpts]

        remaining_chkpts = set(ALL_CHECKPOINTS) - set(chkpts)

        remaining_chkpts_dict[(seed, dataset)] = np.sort(list(remaining_chkpts))
        print(seed, dataset, len(remaining_chkpts_dict[(seed, dataset)]))

    print('\n')
for k, v in remaining_chkpts_dict.items():
    if len(remaining_chkpts_dict[k]) != 0:
        print(k[0], '-', k[1], '-', v)
for seed in [27, 28, 29]:
    for dataset in ['bioasq', 'drop', 'duorc', 'hotpotqa', 'naturalquestions', 'newsqa', 'race', 'relationextraction', 'searchqa', 'squad', 'textbookqa', 'triviaqa']: 
        chkpts = os.listdir(f'predictions/albert-xlarge-v2-mrqa-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-dataset={dataset}')

        for chkpt in chkpts:
            if len(os.listdir(f'predictions/albert-xlarge-v2-mrqa-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-dataset={dataset}/{chkpt}')) < 4:
                print(seed, '-', dataset, '-', chkpt)
                print('\n')
', '.join([x.split('-')[1] for x in chpt_list])
string = ""
for num in remaining_chkpts_dict[30]:
    string += str(num)
    string += ', '
string
all_chpts = []
for dataset in ['bioasq', 'drop', 'duorc', 'hotpotqa', 'naturalquestions', 'newsqa', 'race', 'relationextraction', 'searchqa', 'squad', 'textbookqa', 'triviaqa']:
    all_chpts += list(remaining_chkpts_dict[(29, dataset)])
len(all_chpts)
len(set(all_chpts)) * 12
string = ""
for num in np.sort(list(set(all_chpts))):
    string += str(num)
    string += ', '
string<jupyter_output><empty_output><jupyter_text>SQuAD-AdversarialQA evaluation on training data<jupyter_code>ALL_CHECKPOINTS = [1, 2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32, 36, 44, 52, 60, 68, 76, 92, 108, 124, 140, 156, 172, 188, 204, 220, 236, 252, 268, 284, 300, 316, 332, 348, 364, 380, 396, 428, 460, 492, 524, 556, 588, 620, 652, 684, 716, 748, 780, 812, 844, 876, 908, 940, 972, 1004, 1036, 1100, 1164, 1228, 1292, 1356, 1420, 1484, 1548, 1612, 1676, 1804, 1932, 2060, 2188, 2316, 2444, 2572, 2700, 2828, 2956, 3084, 3212, 3340, 3468, 3596, 3724, 3852, 3980, 4108, 4236, 4364, 4492, 4620, 4748, 4876, 5004, 5132, 5260, 5388, 5516, 5644, 5772, 5900, 6028, 6156, 6284, 6412, 6540, 6668, 6796, 6924, 7052, 7180, 7308, 7436, 7564, 7692, 7820, 7948]
len(ALL_CHECKPOINTS)
remaining_chkpts_dict = {}

for seed in [27, 28, 29]:
    chkpts = os.listdir(f'predictions/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}')
    chkpts = [int(i.split('-')[-1]) for i in chkpts]

    remaining_chkpts = set(ALL_CHECKPOINTS) - set(chkpts)

    remaining_chkpts_dict[seed] = np.sort(list(remaining_chkpts))
    print(seed, len(remaining_chkpts_dict[seed]))

    print('\n')
chkpts = os.listdir(f'predictions/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=29')

for chkpt in chkpts:
    if len(os.listdir(f'predictions/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=29/{chkpt}')) < 2:
        print(chkpt)
string = ""
for num in remaining_chkpts_dict[28]:
    string += str(num)
    string += ', '

string
string = ""
for num in remaining_chkpts_dict[29]:
    string += str(num)
    string += ', '

string
seed = 29
for i, checkpoint in enumerate([1, 10, 14, 124]):
    print(f'test $SGE_TASK_ID -eq {i+1} && sleep 10 && python src/models/run_qa.py --model_name_or_path /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/models/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}/checkpoint-{checkpoint} --do_predict --test_file /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/data/processed/squad1_dadversarialQA_1_1_weighted_train.json --per_device_eval_batch_size 64 --output_dir /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/predictions/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}/checkpoint-{checkpoint} --overwrite_output_dir --overwrite_cache --report_to none > logs/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-checkpoint={checkpoint}.log 2>&1')<jupyter_output>test $SGE_TASK_ID -eq 1 && sleep 10 && python src/models/run_qa.py --model_name_or_path /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/models/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=29/checkpoint-1 --do_predict --test_file /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/data/processed/squad1_dadversarialQA_1_1_weighted_train.json --per_device_eval_batch_size 64 --output_dir /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/predictions/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=29/checkpoint-1 --overwrite_output_dir --overwrite_cache --report_to none > logs/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=29-checkpoint=1.log 2>&1
test $SGE_TASK_ID -eq 2 && sleep 10 && python src/models/run_qa.py --model_name_or_path /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/models/albert-xlarge-v2-squadv1-adversarialall-wu=100-l[...]<jupyter_text>SQuAD-AdversarialQA evaluation on dev data (full outputs)<jupyter_code>ALL_CHECKPOINTS = [1, 2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32, 36, 44, 52, 60, 68, 76, 92, 108, 124, 140, 156, 172, 188, 204, 220, 236, 252, 268, 284, 300, 316, 332, 348, 364, 380, 396, 428, 460, 492, 524, 556, 588, 620, 652, 684, 716, 748, 780, 812, 844, 876, 908, 940, 972, 1004, 1036, 1100, 1164, 1228, 1292, 1356, 1420, 1484, 1548, 1612, 1676, 1804, 1932, 2060, 2188, 2316, 2444, 2572, 2700, 2828, 2956, 3084, 3212, 3340, 3468, 3596, 3724, 3852, 3980, 4108, 4236, 4364, 4492, 4620, 4748, 4876, 5004, 5132, 5260, 5388, 5516, 5644, 5772, 5900, 6028, 6156, 6284, 6412, 6540, 6668, 6796, 6924, 7052, 7180, 7308, 7436, 7564, 7692, 7820, 7948]
len(ALL_CHECKPOINTS)
remaining_chkpts_dict = {}

for dataset in ['dbert', 'dbidaf', 'droberta', 'squad']:
    for seed in [27, 28, 29]:
        chkpts = os.listdir(f'predictions/full/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-{dataset}')
        chkpts = [int(i.split('-')[-1]) for i in chkpts]

        remaining_chkpts = set(ALL_CHECKPOINTS) - set(chkpts)

        remaining_chkpts_dict[(seed, dataset)] = np.sort(list(remaining_chkpts))
        print(seed, dataset, len(remaining_chkpts_dict[(seed, dataset)]))

        print('\n')
for k, v in remaining_chkpts_dict.items():
    print('#=================================#')
    print(k)
    print(','.join([str(i) for i in v]))
    print('Total: ', len(v))
    print('\n')
len(set(np.concatenate(list(remaining_chkpts_dict.values())))) * 12
len(np.concatenate(list(remaining_chkpts_dict.values())))
for dataset in ['dbert', 'dbidaf', 'droberta', 'squad']:
    for seed in [27, 28, 29]:
        chkpts = os.listdir(f'predictions/full/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-{dataset}')

        for chkpt in chkpts:
            if len(os.listdir(f'predictions/full/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-{dataset}/{chkpt}')) < 5:
                print(dataset, seed, chkpt)
string = ""
for num in remaining_chkpts_dict[28]:
    string += str(num)
    string += ', '

string
string = ""
for num in remaining_chkpts_dict[29]:
    string += str(num)
    string += ', '

string
seed = 29
for i, checkpoint in enumerate([1, 10, 14, 124]):
    print(f'test $SGE_TASK_ID -eq {i+1} && sleep 10 && python src/models/run_qa.py --model_name_or_path /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/models/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}/checkpoint-{checkpoint} --do_predict --test_file /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/data/processed/squad1_dadversarialQA_1_1_weighted_train.json --per_device_eval_batch_size 64 --output_dir /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/predictions/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}/checkpoint-{checkpoint} --overwrite_output_dir --overwrite_cache --report_to none > logs/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-checkpoint={checkpoint}.log 2>&1')<jupyter_output>test $SGE_TASK_ID -eq 1 && sleep 10 && python src/models/run_qa.py --model_name_or_path /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/models/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=29/checkpoint-1 --do_predict --test_file /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/data/processed/squad1_dadversarialQA_1_1_weighted_train.json --per_device_eval_batch_size 64 --output_dir /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/predictions/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=29/checkpoint-1 --overwrite_output_dir --overwrite_cache --report_to none > logs/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=29-checkpoint=1.log 2>&1
test $SGE_TASK_ID -eq 2 && sleep 10 && python src/models/run_qa.py --model_name_or_path /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/models/albert-xlarge-v2-squadv1-adversarialall-wu=100-l[...]<jupyter_text>AdversarialQA evaluation on CheckList<jupyter_code>ALL_CHECKPOINTS = [1, 2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32, 36, 44, 52, 60, 68, 76, 92, 108, 124, 140, 156, 172, 188, 204, 220, 236, 252, 268, 284, 300, 316, 332, 348, 364, 380, 396, 428, 460, 492, 524, 556, 588, 620, 652, 684, 716, 748, 780, 812, 844, 876, 908, 940, 972, 1004, 1036, 1100, 1164, 1228, 1292, 1356, 1420, 1484, 1548, 1612, 1676, 1804, 1932, 2060, 2188, 2316, 2444, 2572, 2700, 2828, 2956, 3084, 3212, 3340, 3468, 3596, 3724, 3852, 3980, 4108, 4236, 4364, 4492, 4620, 4748, 4876, 5004, 5132, 5260, 5388, 5516, 5644, 5772, 5900, 6028, 6156, 6284, 6412, 6540, 6668, 6796, 6924, 7052, 7180, 7308, 7436, 7564, 7692, 7820, 7948]
len(ALL_CHECKPOINTS)
remaining_chkpts_dict = {}

for seed in [28, 29, 30]:
    chkpts = os.listdir(f'predictions/checklist/albert-xlarge-v2-adversarial_qa_all-wu=100-lr=3e5-bs=32-msl=384-seed={seed}')
    chkpts = [int(i.split('-')[-1]) for i in chkpts]

    remaining_chkpts = set(ALL_CHECKPOINTS) - set(chkpts)

    remaining_chkpts_dict[seed] = np.sort(list(remaining_chkpts))
    print(seed, len(remaining_chkpts_dict[seed]))

    print('\n')
chkpts = os.listdir(f'predictions/checklist/albert-xlarge-v2-adversarial_qa_all-wu=100-lr=3e5-bs=32-msl=384-seed=30')

count = 0

for chkpt in chkpts:
    if len(os.listdir(f'predictions/checklist/albert-xlarge-v2-adversarial_qa_all-wu=100-lr=3e5-bs=32-msl=384-seed=30/{chkpt}')) < 4:
        count += 1
count
string = ""
for num in remaining_chkpts_dict[28]:
    string += str(num)
    string += ', '

string
string = ""
for num in remaining_chkpts_dict[29]:
    string += str(num)
    string += ', '

string
string = ""
for num in remaining_chkpts_dict[30]:
    string += str(num)
    string += ', '

string
seed = 29
for i, checkpoint in enumerate([8, 12, 28, 44, 76, 92, 108, 2316, 2444, 2572, 2700, 2828, 2956, 3084, 3212, 3340, 3468, 3596, 3724, 3852, 3980, 4108, 4236, 4364, 4492, 4620, 4748, 4876, 5004, 5132, 5260, 5388, 5516, 5644, 5772, 5900, 6028, 6156, 6284, 6412, 6540, 6668, 6796, 6924, 7052, 7180, 7308, 7436, 7564, 7692, 7820, 7948]):
    print(f'test $SGE_TASK_ID -eq {i+1} && sleep 10 && python src/models/run_qa.py --model_name_or_path /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/models/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}/checkpoint-{checkpoint} --do_predict --test_file /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/data/processed/squad1_dadversarialQA_1_1_weighted_train.json --per_device_eval_batch_size 64 --output_dir /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/predictions/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}/checkpoint-{checkpoint} --overwrite_output_dir --overwrite_cache --report_to none > logs/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-checkpoint={checkpoint}.log 2>&1')<jupyter_output>test $SGE_TASK_ID -eq 1 && sleep 10 && python src/models/run_qa.py --model_name_or_path /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/models/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=29/checkpoint-8 --do_predict --test_file /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/data/processed/squad1_dadversarialQA_1_1_weighted_train.json --per_device_eval_batch_size 64 --output_dir /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/predictions/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=29/checkpoint-8 --overwrite_output_dir --overwrite_cache --report_to none > logs/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=29-checkpoint=8.log 2>&1
test $SGE_TASK_ID -eq 2 && sleep 10 && python src/models/run_qa.py --model_name_or_path /SAN/intelsys/rclearn/when-do-reading-comprehension-models-learn/models/albert-xlarge-v2-squadv1-adversarialall-wu=100-l[...]<jupyter_text>END SQuAD v2<jupyter_code>for seed in [27, 29, 30]:
    chkpts = os.listdir(f'predictions/albert-xlarge-v2-squadv2-wu=100-lr=3e5-bs=32-msl=384-seed={seed}')
c
    for chkpt in chkpts:
        if len(os.listdir(f'predictions/albert-xlarge-v2-squadv2-wu=100-lr=3e5-bs=32-msl=384-seed={seed}/{chkpt}')) != 5:
            print(seed, chkpt)
SQ_ADQA_CHECKPOINTS = [1, 2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32, 36, 44, 52, 60, 68, 76, 92, 108, 124, 140, 156, 172, 188, 204, 220, 236, 252, 268, 284, 300, 316, 332, 348, 364, 380, 396, 428, 460, 492, 524, 556, 588, 620, 652, 684, 716, 748, 780, 812]
len(SQ_ADQA_CHECKPOINTS)
remaining_chkpts_list = []

for seed in [27, 28, 29]:
    chkpts = os.listdir(f'predictions/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}')
    chkpts = [int(i.split('-')[-1]) for i in chkpts]
    
    remaining_chkpts = set(SQ_ADQA_CHECKPOINTS) - set(chkpts)
    remaining_chkpts_list.append(remaining_chkpts)
    
#     chkpts = np.sort(chkpts)
#     print(seed, chkpts[-1])
    print(seed, remaining_chkpts)
chkpts = os.listdir(f'predictions/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=29')

for chkpt in chkpts:
    if len(os.listdir(f'predictions/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=29/{chkpt}')) < 2:
        print(chkpt)
len(remaining_chkpts)<jupyter_output><empty_output><jupyter_text>SQuAD v1<jupyter_code>for model in ['dbert', 'dbidaf', 'droberta']:
    for seed in [28, 29, 30]:
        chkpts = os.listdir(f'predictions/albert-xlarge-v2-squadv1-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-{model}')

        for chkpt in chkpts:
            if len(os.listdir(f'predictions/albert-xlarge-v2-squadv1-wu=100-lr=3e5-bs=32-msl=384-seed={seed}/{chkpt}')) != 4:
                print(model, seed, chkpt)
for model in ['dbert', 'dbidaf', 'droberta']:
    for seed in [28, 29, 30]:
        chkpts = os.listdir(f'predictions/albert-xlarge-v2-squadv1-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-{model}')
        chkpts = [int(i.split('-')[-1]) for i in chkpts]

        remaining_chkpts = set(ALL_CHECKPOINTS) - set(chkpts)
        
        if len(remaining_chkpts) != 0:
            print(model, seed, remaining_chkpts)
remaining_chkpts_list
remaining_chkpts_list_union = list(set().union(*remaining_chkpts_list))
remaining_chkpts_list_union
len(remaining_chkpts_list_union)
108 in remaining_chkpts_list_union
remaining_chkpts_list_union.append(108)
remaining_chkpts_list_union
len(remaining_chkpts_list_union)  * 6
remaining_chkpts_list = []

for seed in [27, 28, 29]:
    chkpts = os.listdir(f'predictions/on-train/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=27/checkpoint-1')
    chkpts = [int(i.split('-')[-1]) for i in chkpts]
    
    print(len(chkpts))
string = ""
for num in (set(ALL_CHECKPOINTS) - set(chkpts)):
    string += str(num)
    string += ', '
string
chkpts = os.listdir(f'predictions/checklist/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=28')

for chkpt in chkpts:
    if len(os.listdir(f'predictions/checklist/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=28/{chkpt}')) < 4:
        print(chkpt)
60, 32, 68, 36, 44, 76, 92, 2572, 4748<jupyter_output><empty_output><jupyter_text>seed 30 - checkpoint 16seed 29 - 6028, 6412<jupyter_code>import pandas as pd
remaining_chkpts_list = []
missing_checkpoints = []

for dataset in ['dbert', 'dbidaf', 'droberta', 'squad']:
    for seed in [27, 28, 29]:
        chkpts = os.listdir(f'predictions/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-{dataset}')
        chkpts = [int(i.split('-')[-1]) for i in chkpts]
        
        missing = list(set(ALL_CHECKPOINTS) - set(chkpts))
        missing_checkpoints += missing
        
        print(f'{seed}-{dataset}')
        print(len(chkpts))
        print('\n')
len(set(missing_checkpoints))
string = ""
for num in (set(missing_checkpoints)):
    string += str(num)
    string += ', '
string
string = ""
for num in (set(ALL_CHECKPOINTS) - set(chkpts)):
    string += str(num)
    string += ', '
string
remaining_chkpts_list = []

for seed in [27, 28, 29]:
    for model in ['dbert', 'dbidaf', 'droberta', 'squad']:
        chkpts = os.listdir(f'predictions/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-{model}')
        chkpts = [int(i.split('-')[-1]) for i in chkpts]
    
        print(len(chkpts))
string = ""
for num in (set(ALL_CHECKPOINTS) - set(chkpts)):
    string += str(num)
    string += ', '
string
for seed in [27, 28, 29]:
    for model in ['dbert', 'dbidaf', 'droberta', 'squad']:
        chkpts = os.listdir(f'predictions/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-{model}')

        for chkpt in chkpts:
            if len(os.listdir(f'predictions/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-{model}/{chkpt}')) < 4:
                print(f'predictions/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-{model}/{chkpt}')
                print('\n')
1484, 1548, 1612
for seed in [27, 28, 29]:
    for chkpt in ALL_CHECKPOINTS:
        try:
            pd.read_csv(f'predictions/checklist/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}/checkpoint-{chkpt}/checklist_results_summary.txt')
        except FileNotFoundError:
            pass

        for chkpt in chkpts:
            if len(os.listdir(f'predictions/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-{model}/{chkpt}')) < 4:
                print(f'predictions/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}-{model}/{chkpt}')
                print('\n')<jupyter_output><empty_output><jupyter_text>Context prediction checks 1<jupyter_code>import argparse
import re
import os
os.getcwd()
for seed in [27, 28, 29]:
    for chkpt in ALL_CHECKPOINTS:
        with open(f'predictions/checklist/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}/checkpoint-{chkpt}/checklist_results.txt', 'r') as f:
            summary = f.read()

        contexts = re.findall(r"[\n\r].*C:\s*([^\n\r]*)", summary)
        preds = re.findall(r"[\n\r].*P:\s*([^\n\r]*)", summary)
#         assert len(contexts) == len(preds)

        count = 0
        affected_checkpoints = []
        for c, p in zip(contexts, preds):
            if p =='empty':
                continue
            else:
                try:
                    assert p in c
#                     assert '\n' not in p
                except:
                    count += 1 
                    print("Context:", c)
                    print("P:", p)
                    print('\n')

        if count > 0:
            print(seed, '-', chkpt, '-', count)
[w for w in context_list if 'Oxygen gas (' in w][0]
len(contexts), len(preds)
preds
for seed in [27, 28, 29]:
    for chkpt in ALL_CHECKPOINTS:
        num_lines = sum(1 for line in open(f'predictions/checklist/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}/checkpoint-{chkpt}/predict_predictions_flat.txt'))
        assert num_lines == 71293<jupyter_output><empty_output><jupyter_text>Check if predictions in context<jupyter_code>import os
import json
import ast

import pandas as pd
import numpy as np
import checklist
from checklist.test_suite import TestSuite
from tqdm.contrib.itertools import product
os.chdir('../..')
SEEDS = [27, 28, 29]
ALL_CHECKPOINTS = [1, 2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32, 36, 44, 52, 60, 68, 76, 92, 108, 124, 140, 156, 172, 188, 204, 220, 236, 252, 268, 284, 300, 316, 332, 348, 364, 380, 396, 428, 460, 492, 524, 556, 588, 620, 652, 684, 716, 748, 780, 812, 844, 876, 908, 940, 972, 1004, 1036, 1100, 1164, 1228, 1292, 1356, 1420, 1484, 1548, 1612, 1676, 1804, 1932, 2060, 2188, 2316, 2444, 2572, 2700, 2828, 2956, 3084, 3212, 3340, 3468, 3596, 3724, 3852, 3980, 4108, 4236, 4364, 4492, 4620, 4748, 4876, 5004, 5132, 5260, 5388, 5516, 5644, 5772, 5900, 6028, 6156, 6284, 6412, 6540, 6668, 6796, 6924, 7052, 7180, 7308, 7436, 7564, 7692, 7820, 7948]
len(ALL_CHECKPOINTS)
suite_path = '/home/sgeorge/Github/checklist/release_data/squad/squad_suite.pkl'
suite = TestSuite.from_file(suite_path)
context_list = [x[0] for x in [ast.literal_eval(x) for x in suite.get_raw_examples()]]
question_list = [x[1] for x in [ast.literal_eval(x) for x in suite.get_raw_examples()]]
len(context_list)
context_list[:5]
question_list[:5]
count = 0

for seed, chkpt in product(SEEDS, ALL_CHECKPOINTS):
    
    with open(f'predictions/checklist/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed={seed}/checkpoint-{chkpt}/predict_predictions.json') as json_file:
        predictions = json.load(json_file)

    for context, pred in zip(context_list, predictions.values()):
        if pred == 'empty':
            continue
        else:
            count += 1
            assert pred in context
suite.run_from_file('predictions/checklist/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=27/checkpoint-8/predict_predictions.json', overwrite=True, file_format='pred_only')
# suite.summary()
with open(f'predictions/checklist/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=27/checkpoint-8/predict_predictions.json') as json_file:
    predictions = json.load(json_file)
flat_preds = list(predictions.values())
len(flat_preds)
flat_preds_df = pd.DataFrame(flat_preds)
print(flat_preds_df.shape)
flat_preds_df.head()
flat_preds_df = pd.DataFrame([x + '\r\n' for x in flat_preds])
print(flat_preds_df.shape)
flat_preds_df.head()
print('of silicon (silica SiO\\n2, as found in granite and quartz), aluminium')
with open('predictions/checklist/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=27/checkpoint-8/predict_predictions_flat_new.txt', 'w') as f:
    f.writelines(s + '\n' for s in [x.replace('\n', '\\n') for x in flat_preds])
[x.replace('\n', '\\n') for x in flat_preds]
flat_preds_df.to_csv('predictions/checklist/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=27/checkpoint-8/predict_predictions_flat_new.csv', header=False, index=False, line_terminator='\r\n')
with open('predictions/checklist/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=27/checkpoint-8/predict_predictions_flat_new.txt','w') as log:
    for value in flat_preds:
        log.write('{}\n'.format(value))
[x for x in flat_preds if '\n' in x]
context_list[30426]
pd.read_csv('predictions/checklist/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=27/checkpoint-8/predict_predictions_flat.txt', delimiter = "\t", header=None).shape
len(predictions)
pd.read_csv('predictions/checklist/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=27/checkpoint-8/predict_predictions_flat_new.txt', delimiter = "\t", header=None).shape
pd.read_csv('predictions/checklist/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=27/checkpoint-8/predict_predictions_flat_new.txt', delimiter = "\t", header=None).shape
np.testing.assert_array_equal(np.array(list(predictions.values())), pd.read_csv('predictions/checklist/albert-xlarge-v2-squadv1-adversarialall-wu=100-lr=3e5-bs=32-msl=384-seed=27/checkpoint-8/predict_predictions_flat.txt', delimiter = "\t", header=None)[0])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16697452474409297, 'fraction_numerical': 0.1319941506965289, 'mean_word_length': 4.371434477056635, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 49, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '415810', 'license': 'MIT', 'n_tokens_mistral': 13486, 'n_tokens_neox': 10899, 'n_words': 1863, 'path': 'notebooks/exploration/00-Check job completion.ipynb', 'repo_name': 'stevieg3/when-do-reading-comprehension-models-learn', 'size': '25986'}