<jupyter_start><jupyter_text>Notebook to merge Jesse's augmentations (shear and rotate).<jupyter_code>import os
import pickle
tokens_dir = os.path.join(os.getcwd(), 'data', 'tokens')
def merge_aug_train_pickles(folder):
    merged_records = []
    
    for file in ['train.pickle', 'transformed_train.pickle']:
        with open(os.path.join(folder, file), 'rb') as f:
            data = pickle.load(f)
            merged_records.extend(data)
            
    with open(os.path.join(folder, 'train.pickle'), 'wb') as f:
        pickle.dump(merged_records, f)
        
    print(f'{len(merged_records)} examples total.')
%%time
merge_aug_train_pickles(os.path.join(tokens_dir, 'b=96_train=2011,2013_test=2012_c=all_t=5_rotate'))
%%time
merge_aug_train_pickles(os.path.join(tokens_dir, 'b=96_train=2011,2013_test=2012_c=all_t=5_shear'))
%%time
merge_aug_train_pickles(os.path.join(tokens_dir, 'b=96_train=2011,2013_test=2012_c=all_t=3,5,7_rotate'))
%%time
merge_aug_train_pickles(os.path.join(tokens_dir, 'b=96_train=2011,2013_test=2012_c=all_t=3,5,7_shear'))
%%time
merge_aug_train_pickles(os.path.join(tokens_dir, 'b=96_train=2011,2013_test=2012_c=all_t=1,3,5,7,9_rotate'))
%%time
merge_aug_train_pickles(os.path.join(tokens_dir, 'b=96_train=2011,2013_test=2012_c=all_t=1,3,5,7,9_shear'))<jupyter_output>619656 examples total.
Wall time: 2min 43s
{'fraction_non_alphanumeric': 0.15081723625557206, 'fraction_numerical': 0.0824665676077266, 'mean_word_length': 6.4010989010989015, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '55422', 'license': 'MIT', 'n_tokens_mistral': 618, 'n_tokens_neox': 546, 'n_words': 67, 'path': 'merge_jesses_augmentations.ipynb', 'repo_name': 'Jesse-mk/10617_Project', 'size': '1346'}