<commit_before>import numpy as np
import copy
from sklearn.base import BaseEstimator

def load_images(filenames):
    '''Expects filenames to be a list of .fits file locations'''
    from astropy.io.fits import getdata
    return [ normalized_positive_image(getdata(filename)) for filename in filenames ]

def normalized_positive_image(image) :
    '''Images need to be numpy arrays between -1 and 1 for median and
    possibly HOG, but also should be log normalized so contrast is
    maintained.'''
    
    pos_def = image-image.min()+1.0
    return np.log(pos_def) / np.log(pos_def.max())


class MedianSmooth(BaseEstimator):
    def __init__(self, radius = 3):
        self.radius = radius

    def fit(self, images, y = None):
        return self

    def transform(self, images):
        from skimage.filters.rank import median
        from skimage.morphology import disk
        return np.array([median(image, disk(self.radius))
                         for image in images])

    def fit_transform(self, images, y = None):
        return self.transform(images)

class HOG(BaseEstimator):
    def __init__(self, orientations = 9, pixels_per_cell = (8, 8),
                 cells_per_block = (3, 3) ):
        self.orientations = orientations
        self.pixels_per_cell = pixels_per_cell
        self.cells_per_block = cells_per_block

    def fit(self, images, y = None):
        return self

    def transform(self, images):<commit_msg>Normalization incorrect - parenthesis in wrong place...<commit_after>import numpy as np
import copy
from sklearn.base import BaseEstimator

def load_images(filenames):
    '''Expects filenames to be a list of .fits file locations'''
    from astropy.io.fits import getdata

    return [ normalized_positive_image(getdata(filename)) for filename in filenames ]

def normalized_positive_image(image) :
    '''Images need to be numpy arrays between -1 and 1 for median and
    possibly HOG, but also should be log normalized so contrast is
    maintained.'''
    
    pos_def = image-image.min()+1.0
    return np.log(pos_def) / np.log(pos_def).max()


class MedianSmooth(BaseEstimator):
    def __init__(self, radius = 3):
        self.radius = radius

    def fit(self, images, y = None):
        return self

    def transform(self, images):
        from skimage.filters.rank import median
        from skimage.morphology import disk
        return np.array([median(image, disk(self.radius))
                         for image in images])

    def fit_transform(self, images, y = None):
        return self.transform(images)

class HOG(BaseEstimator):
    def __init__(self, orientations = 9, pixels_per_cell = (8, 8),
                 cells_per_block = (3, 3) ):
        self.orientations = orientations
        self.pixels_per_cell = pixels_per_cell
        self.cells_per_block = cells_per_block

    def fit(self, images, y = None):
        return self

    def transform(self, images):
{'fraction_non_alphanumeric': 0.08852570650323459, 'fraction_numerical': 0.0068096697310180455, 'mean_word_length': 2.927807486631016, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3489221', 'n_tokens_mistral': 891, 'n_tokens_neox': 841, 'n_words': 268}