<jupyter_start><jupyter_text>How to use Preprocessing:<jupyter_code>#lays all images ontop of each other --> the result image can be used to determine the interesting area of the
#pictures and determine the cutting borders
path = '/Users/Dino/Desktop/1'
maximum = preprocessing.find_overlap(path) # this function uses cutting borders whitch are (ATTENTION:) set constant in the code
plt.imshow(maximum)
#create individual images.npy and collects them to leafs/data/...
root_path = '/Users/Dino/Documents/Academia/2016_ArtificialIntelligence/Data-Leaf-Coded/'
save_path = '/Users/Dino/Dropbox/Academia2016/academia_AI/academia_ai/leafs/data/'
preprocessing.collectData(root_path, save_path, cfactor = 20, overwrite = False)
# die resultierende Grösse ist 2000 / cfactor 
#create a training and a testing list
path = '/Users/Dino/Dropbox/Academia2016/academia_AI/academia_ai/leafs/data/80x80'
data = preprocessing.createTrainingAndTestingList(path, shuffle = False)
trainset = data[0]
testset = data[1]
#preprocessing.normalize(trainset)
#preprocessing.normalize(testset)

#take from training and testing only the matrixes, not the whole object leaf
training_images = [np.array(leaf.image) for leaf in trainset]
training_solutions = [preprocessing.desired_output(leaf.label) for leaf in trainset]
test_images = [np.array(leaf.image) for leaf in testset]
test_solutions = [preprocessing.desired_output(leaf.label) for leaf in testset]
#test if the saved thing is useful
plt.imshow(test_images[550], interpolation = 'none', cmap ='Greys')
plt.colorbar
savedstuff = np.load('/Users/Dino/Dropbox/Academia2016/academia_AI/academia_ai/leafs/data/80x80/0.npy')
print('Shape of savedstuff:', savedstuff.shape)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10237131289762869, 'fraction_numerical': 0.021399652978600348, 'mean_word_length': 6.26890756302521, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '228911', 'license': 'MIT', 'n_tokens_mistral': 604, 'n_tokens_neox': 557, 'n_words': 146, 'path': 'academia_ai/examples/HowToUse-Preprocessing-2017-01-03.ipynb', 'repo_name': 'Knuppknou/academia_AI', 'size': '1729'}