<jupyter_start><jupyter_text>Data collected by me<jupyter_code>generator1 = DataGenerator('./my_data/', epochs, batch_size, debug=False)
dtale.show(generator1.df)
generator1.show_stats(logarithmic=False, bins=100)
generator1.show_stats(logarithmic=True, bins=100)
generator1.balance_dataset()
dtale.show(generator1.df)
generator1.show_stats(logarithmic=False, bins=100)
generator1.show_stats(logarithmic=True, bins=100)<jupyter_output><empty_output><jupyter_text>Data provided by Udacity<jupyter_code>generator2 = DataGenerator('./udacity_data/', epochs, batch_size, debug=False)
dtale.show(generator2.df)
generator2.show_stats(logarithmic=False, bins=100)
generator2.show_stats(logarithmic=True, bins=100)
generator2.balance_dataset()
dtale.show(generator2.df)
rng = (-1., 1.)
generator2.show_stats(logarithmic=False, bins=100, range=rng) 
generator2.show_stats(logarithmic=True, bins=100, range=rng)<jupyter_output><empty_output><jupyter_text>Data merged (my + udacity)<jupyter_code>generator = generator1.merge(generator2)
dtale.show(generator.df)
train, valid = generator.split(factor=0.1)
dtale.show(train.df)
dtale.show(valid.df)
rng = (-1., 1.)
train.show_stats(logarithmic=False, bins=100, range=rng)
rng = (-1., 1.)
valid.show_stats(logarithmic=False, bins=100, range=rng)
plt.imshow(generator.mean_of_all())<jupyter_output><empty_output><jupyter_text>Let's check loading time<jupyter_code>import time
start_time = time.time()
for i in range(batch_size):
    x, y = train.__getitem__(i)
    x, y = valid.__getitem__(i)
end_time = time.time()
print(end_time - start_time)<jupyter_output>1.3782048225402832
{'fraction_non_alphanumeric': 0.13949163050216987, 'fraction_numerical': 0.04525728456292622, 'mean_word_length': 9.150943396226415, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '589226', 'license': 'MIT', 'n_tokens_mistral': 656, 'n_tokens_neox': 605, 'n_words': 85, 'path': 'data_visualize.ipynb', 'repo_name': 'gregtyminski/Behavioral-Clonning', 'size': '1613'}