<commit_before>import cPickle as pickle
from preprocess import *


# load annotations
with open('./train/train.annotations.pkl', 'rb') as f:
    train_annotations = pickle.load(f)
    
    
with open('./val/val.annotations.pkl', 'rb') as f:
    val_annotations = pickle.load(f)

    
with open('./test/test.annotations.pkl', 'rb') as f:
    test_annotations = pickle.load(f)

    
with open('./train/word_to_idx.pkl', 'rb') as f:
    word_to_idx = pickle.load(f)
    

train_captions = build_caption_vectors(train_annotations, word_to_idx, 15)
with open('./train/train.captions.pkl', 'wb') as f:
    pickle.dump(train_captions, f, pickle.HIGHEST_PROTOCOL)
    print 'generated train.captions.pkl'
    

train_file_names, id_to_idx = get_file_names(train_annotations)
with open('./train/train.file.names.pkl', 'wb') as f:
    pickle.dump(train_file_names, f, pickle.HIGHEST_PROTOCOL)
    print 'generated train.file.names.pkl'
    
    
train_image_idxs = get_image_idxs(train_annotations, id_to_idx)
with open('./train/train.captions.pkl', 'wb') as f:
    pickle.dump(train_captions, f, pickle.HIGHEST_PROTOCOL)
    print 'generated train.captions.pkl'

    
val_file_names, _ = get_file_names(val_annotations)
with open('./val/val.file.names.pkl', 'wb') as f:
    pickle.dump(val_file_names, f, pickle.HIGHEST_PROTOCOL)
    print 'generated val.file.names.pkl'

    
test_file_names, _ = get_file_names(test_annotations)
with open('./test/test.file.names.pkl', 'wb') as f:
    pickle.dump(test_file_names, f, pickle.HIGHEST_PROTOCOL)
    print 'generated test.file.names.pkl'
    
    
    
    
    
    
    
    
    
    
    
    <commit_msg>data generation file<commit_after>import cPickle as pickle
from utils import *


def main():
    splits = ['train', 'val', 'test']
    for split in splits:
        annotations = load_pickle('./%s/%s.annotations.pkl' %(split, split))

        if split == 'train':
            word_to_idx = load_pickle('./train/word_to_idx.pkl')
            captions = build_caption_vectors(annotations, word_to_idx, 15)
            save_pickle(captions, './train/train.captions.pkl')

        file_names, id_to_idx = get_file_names(annotations)
        save_pickle(file_names, './%s/%s.file.names.pkl' %(split, split))

        if split == 'train':
            image_idxs = get_image_idxs(annotations, id_to_idx)
            save_pickle(image_idxs, './train/train.image.idxs.pkl')


if __name__ == "__main__":
   main()


    
    
    
    
    
    
    
    
    
    
    
{'fraction_non_alphanumeric': 0.13003589948145194, 'fraction_numerical': 0.0015955325089748703, 'mean_word_length': 2.980952380952381, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7284127', 'n_tokens_mistral': 935, 'n_tokens_neox': 909, 'n_words': 159}