<commit_before>#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Thu Apr 27 11:09:41 2017

@author: daniele
"""

# Import relevant packages
import numpy as np
import matplotlib.pyplot as plt

from os.path import isfile

import scipy.ndimage
from sklearn.preprocessing import LabelBinarizer
from tqdm import tqdm

from modules.path_munging import (all_image_paths, batch_list, create_folder,
                                  count_batches)
from modules.visualization import display_images
from modules.image_preprocessing import (get_Type, array_all_images,
                                         array_all_labels)


class DataPreprocessor(object):
    """
    Preprocessing object. Loads image files, transforms them to a preferred
    format, and saves them to disk.
    """

    def __init__(self, training_folders=[], testing_folder="", **kwargs):
        self.training_folders = training_folders
        self.testing_folder = testing_folder
        # Now go through the training and testing folders to make a list of
        # paths (strings) to the image files.
        # We first get all path-names for the training and testing images
        training_pathnames = sum([all_image_paths(folder)
                                  for folder in training_folders], [])
        testing_pathnames = all_image_paths(testing_folder)
        # In each folder all images depict the same cervical type
        all_Types = np.sort([get_Type(pathname[0])
                             for pathname in training_pathnames])
        # We may now make the function that one-hot-encodes Types into arrays.
        # This will be necessary later when we make arrays with one-hot-encoded
        # labels.
        self.alltypes = all_Types
        enc = LabelBinarizer()
        enc.fit(all_Types)
        self.encoder = enc
        # We now flatten the lists of path names
        training_pathnames = np.array(sum(training_pathnames, []))
        testing_pathnames = np.array(sum(testing_pathnames, []))
        # When training, we don't want the images to be ordered. Therefore, we
        # take a random permutation of their order. For reproducibility we fix
        # the random seed.
        np.random.seed(42)<commit_msg>fix: all_Types should be wrapped by np.unique to avoid duplicates<commit_after>#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Thu Apr 27 11:09:41 2017

@author: daniele
"""

# Import relevant packages
import numpy as np
import matplotlib.pyplot as plt

from os.path import isfile

import scipy.ndimage
from sklearn.preprocessing import LabelBinarizer
from tqdm import tqdm

from modules.path_munging import (all_image_paths, batch_list, create_folder,
                                  count_batches)
from modules.visualization import display_images
from modules.image_preprocessing import (get_Type, array_all_images,
                                         array_all_labels)


class DataPreprocessor(object):
    """
    Preprocessing object. Loads image files, transforms them to a preferred
    format, and saves them to disk.
    """

    def __init__(self, training_folders=[], testing_folder="", **kwargs):
        self.training_folders = training_folders
        self.testing_folder = testing_folder
        # Now go through the training and testing folders to make a list of
        # paths (strings) to the image files.
        # We first get all path-names for the training and testing images
        training_pathnames = sum([all_image_paths(folder)
                                  for folder in training_folders], [])
        testing_pathnames = all_image_paths(testing_folder)
        # In each folder all images depict the same cervical type
        all_Types = np.sort(np.unique([get_Type(pathname[0])
                                       for pathname in training_pathnames]))
        # We may now make the function that one-hot-encodes Types into arrays.
        # This will be necessary later when we make arrays with one-hot-encoded
        # labels.
        self.alltypes = all_Types
        enc = LabelBinarizer()
        enc.fit(all_Types)
        self.encoder = enc
        # We now flatten the lists of path names
        training_pathnames = np.array(sum(training_pathnames, []))
        testing_pathnames = np.array(sum(testing_pathnames, []))
        # When training, we don't want the images to be ordered. Therefore, we
        # take a random permutation of their order. For reproducibility we fix
        # the random seed.
        np.random.seed(42)
{'fraction_non_alphanumeric': 0.07043497460808125, 'fraction_numerical': 0.007507175977036874, 'mean_word_length': 2.5669291338582676, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5072670', 'n_tokens_mistral': 1250, 'n_tokens_neox': 1185, 'n_words': 418}