<commit_before>import seldon.pipeline.pipelines as pl
from sklearn import preprocessing
from dateutil.parser import parse
import datetime

class Auto_transform(pl.Feature_transform):
    """Automatically transform a set of features into normalzied numeric or categorical features or dates

    Args:
        exclude (list):list of features to not include

        max_values_numeric_categorical (int):max number of unique values for numeric feature to treat as categorical

        custom_date_formats (list(str)): list of custom date formats to try

        ignore_vals (list(str)): list of feature values to treat as NA/ignored values
    """
    def __init__(self,exclude=[],max_values_numeric_categorical=20,custom_date_formats=None,ignore_vals=None):
        super(Auto_transform, self).__init__()
        self.exclude = exclude
        self.max_values_numeric_categorical = max_values_numeric_categorical
        self.scalers = {}
        self.custom_date_formats = custom_date_formats
        if ignore_vals:
            self.ignore_vals = ignore_vals
        else:
            self.ignore_vals = ["NA",""]
        self.transforms = {}

    def get_models(self):
        return [(self.exclude,self.custom_date_formats,self.max_values_numeric_categorical),self.transforms,self.scalers]
    
    def set_models(self,models):
        (self.exclude,self.custom_date_formats,self.max_values_numeric_categorical) = models[0]
        self.transforms = models[1]
        self.scalers = models[2]

    @staticmethod
    def is_number(s):
        try:
            float(s)
            return True
        except ValueError:
            return False

    @staticmethod
    def isBoolean(v):
        v = str(v)
        return v.lower() == "true" or v.lower() == "false" or v == "1" or v == "0"

    @staticmethod
    def toBoolean(f,v):
        v = str(v)
        if v.lower() == "true" or v == "1":
            return 1
        else:
            return 0

    def fit_scalers(self,objs,features):
        """fit numeric scalers on all numeric features

        requires enough memory to run sklearn standard scaler on all values for a feature
        """
        print "creating ",len(features),"features scalers"
        Xs = {}
        for f in features:
            Xs[f] = []
        for j in objs:
            for f in features:
                if f in j and self.is_number(j[f]):
                    Xs[f].append(float(j[f]))
        c = 1
        for f in Xs:
            print "creating feature scaler",c," for ",f
            self.scalers[f] = preprocessing.StandardScaler(with_mean=True, with_std=True).fit(Xs[f])
            c += 1

    def scale(self,f,v):
        if self.is_number(v):
            return self.scalers[f].transform([v])[0]
        else:
            return 0

    @staticmethod
    def make_categorical_token(f,v):
        """make a ctaegorical feature from feature and its value
        """
        v = str(v).lower().replace(" ","_")
        if Auto_transform.is_number(v):
            return "t_"+v
        else:
            return v

    def is_date(self,v):
        """is this feature a date
        """
        try:
            parse(v)
            return True
        except:
            if self.custom_date_formats:
                for f in self.custom_date_formats:
                    try:
                        datetime.datetime.strptime( v, f )
                        return True
                    except:
                        pass
            return False

    def unix_time(self,dt):
        """transform a date into a unix day number
        """
        epoch = datetime.datetime.utcfromtimestamp(0)
        delta = dt - epoch
        return delta.total_seconds()

    def to_date(self,f,v):
        d = None
        try:
            d = parse(v)
        except:
            for f in self.custom_date_formats:
                try:
                    d = datetime.datetime.strptime( v, f )
                except:
                    pass
        return "t_"+str(int(self.unix_time(d)/86400))

    def fit(self,objs):
        """try to guess a transform to apply to each feature
        """
        values = {}
        c = 1
        for j in objs:
            for f in j:
                if f in self.exclude or j[f] in self.ignore_vals:
                    pass
                else:
                    cur = values.get(f,set())
                    if len(cur) < (self.max_values_numeric_categorical + 1):
                        cur.add(j[f])
                        values[f] = cur
        featuresToScale = []
        for f in values:
            if all(self.isBoolean(x) for x in values[f]):
                self.transforms[f] = self.toBoolean.__name__
            else:
                if len(values[f]) > self.max_values_numeric_categorical:
                    if all(self.is_number(x) for x in values[f]):
                       featuresToScale.append(f)
                       self.transforms[f] = self.scale.__name__
                    elif all(self.is_date(x) for x in values[f]):
                        self.transforms[f] = self.to_date.__name__
                    else:
                        self.transforms[f] = self.make_categorical_token.__name__
                else:
                    self.transforms[f] = self.make_categorical_token.__name__
        self.fit_scalers(objs,featuresToScale)

    def transform(self,j):
        """Apply learnt transforms on each feature
        """
        jNew = {}
        for f in j:
            if not f in self.transforms:
                jNew[f] = j[f]
            else:
                if not j[f] in self.ignore_vals:
                    jNew[f] = getattr(self,self.transforms[f])(f,j[f])
        return jNew

<commit_msg>handle bad dates in auto transform<commit_after>import seldon.pipeline.pipelines as pl
from sklearn import preprocessing
from dateutil.parser import parse
import datetime

class Auto_transform(pl.Feature_transform):
    """Automatically transform a set of features into normalzied numeric or categorical features or dates

    Args:
        exclude (list):list of features to not include

        max_values_numeric_categorical (int):max number of unique values for numeric feature to treat as categorical

        custom_date_formats (list(str)): list of custom date formats to try

        ignore_vals (list(str)): list of feature values to treat as NA/ignored values
    """
    def __init__(self,exclude=[],max_values_numeric_categorical=20,custom_date_formats=None,ignore_vals=None):
        super(Auto_transform, self).__init__()
        self.exclude = exclude
        self.max_values_numeric_categorical = max_values_numeric_categorical
        self.scalers = {}
        self.custom_date_formats = custom_date_formats
        if ignore_vals:
            self.ignore_vals = ignore_vals
        else:
            self.ignore_vals = ["NA",""]
        self.transforms = {}

    def get_models(self):
        return [(self.exclude,self.custom_date_formats,self.max_values_numeric_categorical),self.transforms,self.scalers]
    
    def set_models(self,models):
        (self.exclude,self.custom_date_formats,self.max_values_numeric_categorical) = models[0]
        self.transforms = models[1]
        self.scalers = models[2]

    @staticmethod
    def is_number(s):
        try:
            float(s)
            return True
        except ValueError:
            return False

    @staticmethod
    def isBoolean(v):
        v = str(v)
        return v.lower() == "true" or v.lower() == "false" or v == "1" or v == "0"

    @staticmethod
    def toBoolean(f,v):
        v = str(v)
        if v.lower() == "true" or v == "1":
            return 1
        else:
            return 0

    def fit_scalers(self,objs,features):
        """fit numeric scalers on all numeric features

        requires enough memory to run sklearn standard scaler on all values for a feature
        """
        print "creating ",len(features),"features scalers"
        Xs = {}
        for f in features:
            Xs[f] = []
        for j in objs:
            for f in features:
                if f in j and self.is_number(j[f]):
                    Xs[f].append(float(j[f]))
        c = 1
        for f in Xs:
            print "creating feature scaler",c," for ",f
            self.scalers[f] = preprocessing.StandardScaler(with_mean=True, with_std=True).fit(Xs[f])
            c += 1

    def scale(self,f,v):
        if self.is_number(v):
            return self.scalers[f].transform([v])[0]
        else:
            return 0

    @staticmethod
    def make_categorical_token(f,v):
        """make a ctaegorical feature from feature and its value
        """
        v = str(v).lower().replace(" ","_")
        if Auto_transform.is_number(v):
            return "t_"+v
        else:
            return v

    def is_date(self,v):
        """is this feature a date
        """
        try:
            parse(v)
            return True
        except:
            if self.custom_date_formats:
                for f in self.custom_date_formats:
                    try:
                        datetime.datetime.strptime( v, f )
                        return True
                    except:
                        pass
            return False

    def unix_time(self,dt):
        """transform a date into a unix day number
        """
        epoch = datetime.datetime.utcfromtimestamp(0)
        delta = dt - epoch
        return delta.total_seconds()

    def to_date(self,f,v):
        d = None
        try:
            d = parse(v)
        except:
            for f in self.custom_date_formats:
                try:
                    d = datetime.datetime.strptime( v, f )
                except:
                    pass
        if d:
            return "t_"+str(int(self.unix_time(d)/86400))
        else:
            return None

    def fit(self,objs):
        """try to guess a transform to apply to each feature
        """
        values = {}
        c = 1
        for j in objs:
            for f in j:
                if f in self.exclude or j[f] in self.ignore_vals:
                    pass
                else:
                    cur = values.get(f,set())
                    if len(cur) < (self.max_values_numeric_categorical + 1):
                        cur.add(j[f])
                        values[f] = cur
        featuresToScale = []
        for f in values:
            if all(self.isBoolean(x) for x in values[f]):
                self.transforms[f] = self.toBoolean.__name__
            else:
                if len(values[f]) > self.max_values_numeric_categorical:
                    if all(self.is_number(x) for x in values[f]):
                       featuresToScale.append(f)
                       self.transforms[f] = self.scale.__name__
                    elif all(self.is_date(x) for x in values[f]):
                        self.transforms[f] = self.to_date.__name__
                    else:
                        self.transforms[f] = self.make_categorical_token.__name__
                else:
                    self.transforms[f] = self.make_categorical_token.__name__
        self.fit_scalers(objs,featuresToScale)

    def transform(self,j):
        """Apply learnt transforms on each feature
        """
        jNew = {}
        for f in j:
            if not f in self.transforms:
                jNew[f] = j[f]
            else:
                if not j[f] in self.ignore_vals:
                    vNew = getattr(self,self.transforms[f])(f,j[f])
                    if vNew:
                        jNew[f] = vNew
        return jNew


{'fraction_non_alphanumeric': 0.09281901153779921, 'fraction_numerical': 0.0037885310831754777, 'mean_word_length': 1.6278280542986425, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3748492', 'n_tokens_mistral': 3432, 'n_tokens_neox': 3270, 'n_words': 854}