<commit_before>from datetime import datetime


class Cleaner(object):
    """
    A cleaner object for csv files
    """
    def __init__(self):
        self.TYPES = {
            'parcel': self.clean_parcel_row
        }

    def clean(self, _type, row):
        """
        Reads type and determines which function to use to clean the row based on type
        :param _type: type of cleaning function that should be used.
        :param row: Row that is being cleaned.
        :return:
        """
        _func = self.TYPES[_type]
        return _func(row)

    def clean_parcel_row(self, row):
        # There should be twelve keys
        if len(row) is not 12:
            print("Missing column(s) in row: {}".format(row))
            return 'DIRTY', row

        # Iterate through each column
        for k, v in row.iteritems():
            # Format Dates to ISO 8601 standards...
            if k == 'Parcel_Info_Source_Date':
                if format_date(row['Parcel_Info_Source_Date']) == 'N':
                    return 'DIRTY', row
                else:
                    row['Parcel_Info_Source_Date'] = format_date(row['Parcel_Info_Source_Date'])
            if k == 'Parcel_owner_date':
                if format_date(row['Parcel_owner_date']) == 'N':
                    return 'DIRTY', row
                else:
                    row['Parcel_owner_date'] = format_date(row['Parcel_owner_date'])

            # Format parcel owners if there are more than one into a comma separated list.
            if '+' in row['Parcel_owner_name']:
                row['Parcel_owner_name'] = row['Parcel_owner_name'].replace(' +', ',')

            # TODO Will need more heuristics on what we expect we will encounter and will need to be cleaned.
        return 'CLEAN', row


def format_date(date_string):
    if date_string == 'N':
        return 'N'
    else:
        _date = datetime.strptime(date_string, '%m/%d/%Y')
        return datetime.strftime(_date, '%Y-%m-%d')


def trim_whitespace(value):
    return value.strip()<commit_msg>add clean method for reac_score csv<commit_after>from datetime import datetime


class Cleaner(object):
    """
    A cleaner object for csv files
    """
    def __init__(self):
        self.TYPES = {
            'parcel': self.clean_parcel_row,
            'reac_score': self.clean_reac_score_row
        }

    def clean(self, _type, row):
        """
        Reads type and determines which function to use to clean the row based on type
        :param _type: type of cleaning function that should be used.
        :param row: Row that is being cleaned.
        :return:
        """
        _func = self.TYPES[_type]
        return _func(row)

    def clean_parcel_row(self, row):
        # There should be twelve keys
        if len(row) is not 12:
            print("Missing column(s) in row: {}".format(row))
            return 'DIRTY', row

        # Iterate through each column
        for k, v in row.iteritems():
            # Format Dates to ISO 8601 standards...
            if k == 'Parcel_Info_Source_Date':
                if format_date(row['Parcel_Info_Source_Date']) == 'N':
                    return 'DIRTY', row
                else:
                    row['Parcel_Info_Source_Date'] = format_date(row['Parcel_Info_Source_Date'])
            if k == 'Parcel_owner_date':
                if format_date(row['Parcel_owner_date']) == 'N':
                    return 'DIRTY', row
                else:
                    row['Parcel_owner_date'] = format_date(row['Parcel_owner_date'])

            # Format parcel owners if there are more than one into a comma separated list.
            if '+' in row['Parcel_owner_name']:
                row['Parcel_owner_name'] = row['Parcel_owner_name'].replace(' +', ',')

            # TODO Will need more heuristics on what we expect we will encounter and will need to be cleaned.
        return 'CLEAN', row

    def clean_reac_score_row(self, row):
        if len(row) is not 6:
            print("Missing column(s) in row: {}".format(row))
            return 'DIRTY', row
        for k, v in row.iteritems():
            row[k] = trim_whitespace(v)
            if k == 'REAC_date':
                if format_date(row['REAC_date']) == 'N':
                    return 'DIRTY', row
                else:
                    row['REAC_date'] = format_date(row['REAC_date'])
        return 'CLEAN', row

def format_date(date_string):
    if date_string == 'N':
        return 'N'
    else:
        _date = datetime.strptime(date_string, '%m/%d/%Y')
        return datetime.strftime(_date, '%Y-%m-%d')


def trim_whitespace(value):
    return value.strip()
{'fraction_non_alphanumeric': 0.10822974710672953, 'fraction_numerical': 0.002786112301757394, 'mean_word_length': 1.9707192870782941, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2755986', 'n_tokens_mistral': 1390, 'n_tokens_neox': 1332, 'n_words': 396}