<jupyter_start><jupyter_text>Testing new generator<jupyter_code># generators
W_HYP = False
BATCH_SIZE = 64
shuffle = True
training_generator = generator.Keras_DataGenerator(data_dir='', dataset='train_new', 
                                                  w_hyp=W_HYP, batch_size=BATCH_SIZE,
                                                  shuffle=shuffle)
validation_generator = generator.Keras_DataGenerator(data_dir='', dataset='valid_new', 
                                                    w_hyp=W_HYP, batch_size=BATCH_SIZE,
                                                    shuffle=shuffle)
print(len(validation_generator))
print((validation_generator.n))
print((validation_generator.batches_per_partition))
print((validation_generator.batch_size))
validation_generator.batch_index
for i, (x, y) in enumerate(validation_generator):
    print(i, x.shape, y.shape)
    print()
for i, (x, y) in enumerate(training_generator):
    print(i, x.shape, y.shape)
    print()
1<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09683794466403162, 'fraction_numerical': 0.0029644268774703555, 'mean_word_length': 2.5543859649122806, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '230779', 'license': 'Apache-2.0', 'n_tokens_mistral': 290, 'n_tokens_neox': 278, 'n_words': 48, 'path': 'deepmath/deephol/train/Other_notuseful/building_new_generator.ipynb', 'repo_name': 'magualas/deepmath', 'size': '1012'}