def saveTokensToFiles(TOKEN_DATA_PATH, train_data_tokenized, train_attention_mask, valid_data_tokenized, valid_attention_mask, test_data_tokenized, test_attention_mask):
    with open((TOKEN_DATA_PATH + '/train_data_tokenized.txt'), 'wb') as fp:
        pickle.dump(train_data_tokenized, fp)
    with open((TOKEN_DATA_PATH + '/train_attention_mask.txt'), 'wb') as fp:
        pickle.dump(train_attention_mask, fp)
    with open((TOKEN_DATA_PATH + '/valid_data_tokenized.txt'), 'wb') as fp:
        pickle.dump(valid_data_tokenized, fp)
    with open((TOKEN_DATA_PATH + '/valid_attention_mask.txt'), 'wb') as fp:
        pickle.dump(valid_attention_mask, fp)
    with open((TOKEN_DATA_PATH + '/test_data_tokenized.txt'), 'wb') as fp:
        pickle.dump(test_data_tokenized, fp)
    with open((TOKEN_DATA_PATH + '/test_attention_mask.txt'), 'wb') as fp:
        pickle.dump(test_attention_mask, fp)
def saveTokensToFiles(TOKEN_DATA_PATH, train_data_tokenized, train_attention_mask, valid_data_tokenized, valid_attention_mask, test_data_tokenized, test_attention_mask):
    with open((TOKEN_DATA_PATH + '/train_data_tokenized.txt'), 'wb') as fp:
        pickle.dump(train_data_tokenized, fp)
    with open((TOKEN_DATA_PATH + '/train_attention_mask.txt'), 'wb') as fp:
        pickle.dump(train_attention_mask, fp)
    with open((TOKEN_DATA_PATH + '/valid_data_tokenized.txt'), 'wb') as fp:
        pickle.dump(valid_data_tokenized, fp)
    with open((TOKEN_DATA_PATH + '/valid_attention_mask.txt'), 'wb') as fp:
        pickle.dump(valid_attention_mask, fp)
    with open((TOKEN_DATA_PATH + '/test_data_tokenized.txt'), 'wb') as fp:
        pickle.dump(test_data_tokenized, fp)
    with open((TOKEN_DATA_PATH + '/test_attention_mask.txt'), 'wb') as fp:
        pickle.dump(test_attention_mask, fp)