<commit_before>import pandas as pd
import re
import yaml


class UnknownActionError(Exception):
    pass


class ActionList:
    def __init__(self, list_of_actions):
        self.actions = []
        for step in list_of_actions:
            try:
                action_name = list(step.keys())[0]
                instruction = step[action_name]
            except AttributeError:
                action_name = step
                instruction = []
            action = Action.factory(action_name, instruction)
            self.actions.append(action)

    def perform_instructions(self, input_data):
        output_data = input_data
        for action in self.actions:
            output_data = action.perform_instructions(output_data)
        return output_data

    @classmethod
    def from_yaml(cls, filepath_or_buffer):
        if isinstance(filepath_or_buffer, str):
            with open(filepath_or_buffer) as f:
                actions = yaml.safe_load(f)
        else:
            actions = yaml.safe_load(filepath_or_buffer)
        return cls(actions)


# abstract, never used.
class Action:
    def __init__(self, instructions):
        self.instructions = instructions

    @staticmethod
    def factory(action, instruction):
        # just calls different constructors based on passed action
        action_classes = {
            'change_date_or_time_format': ChangeDateFormat,
            'copy_column': CopyAction,
            'rename_column': RenameAction,
            'extract_query_string': ExtractQueryStringAction,
            'extract_text': ExtractTextAction,
            'only_keep_these_columns': FilterColumnAction,
            'only_keep_rows_where': FilterRowAction,
            'only_edit_rows_where': EditSpecificRowsAction,
            'run_these_formula': FormulaAction,
            'remove_columns': RemoveColumnAction,
            'remove_duplicates': RemoveDuplicatesAction,
            'replace_text': ReplaceTextAction,
            'add_text_at_end': AppendTextAction,
            'add_text_at_start': PrependTextAction,
            'sum_up_by': GroupBySumAction,
            'make_column_names_lowercase': LowerCaseColumnNamesAction,
            'make_column_names_alphanumeric': AlphaNumColumnNamesAction,
            'ensure_column_is_in_this_format': ChangeColumnFormatAction,
        }
        try:
            action_class = action_classes[action]
            return action_class(instruction)
        except KeyError:
            msg = 'action {} unknown'.format(action)
            raise UnknownActionError(msg)


class GroupBySumAction(Action):
    """
    self.instructions: list of columns to group by.
    """
    def perform_instructions(self, input_data):
        columns_to_group_by = self.instructions
        grouped = input_data.groupby(columns_to_group_by, as_index=False)
        output_data = grouped.sum()
        return output_data


class ChangeDateFormat(Action):
    """
    self.instructions: list of dicts
        keys:
            target_column
            result_column
            date_format
    """
    def perform_instructions(self, input_data):
        for instruction in self.instructions:
            result_col = instruction['result_column']
            date_format = instruction['date_format']
            date_format_strings = (
                # TODO: hour/minute/second support.
                ('YYYY', '%Y'),
                ('YY', '%y'),
                ('MM', '%m'),
                ('DD', '%d'),
            )
            for date_str, strformat in date_format_strings:
                date_format = date_format.replace(date_str, strformat)
                date_format = date_format.replace(date_str.lower(), strformat)
            format_date = lambda x: x.strftime(date_format)
            dates = input_data[instruction['target_column']].dropna()
            input_data[result_col] = dates.map(format_date)
        return input_data


class AlphaNumColumnNamesAction(Action):
    """
    """
    def perform_instructions(self, input_data):
        spaces_to_underscore = lambda x: x.replace(' ', '_')
        output_data = input_data.rename(columns=spaces_to_underscore)
        strip_punctuation = lambda x: re.sub('[^\w_]', '', x)
        output_data = output_data.rename(columns=strip_punctuation)
        return output_data


class LowerCaseColumnNamesAction(Action):
    """
    """
    def perform_instructions(self, input_data):
        output_data = input_data.rename(columns=str.lower)
        return output_data


class ChangeColumnFormatAction(Action):
    """
    self.instructions: list of dicts
    """
    def perform_instructions(self, input_data):
        output_data = input_data
        for instruction in self.instructions:
            column_format = list(instruction.values())[0]
            column = list(instruction.keys())[0]
            if column_format == 'date':
                output_data[column] = pd.to_datetime(output_data[column])
            if column_format == 'text':
                output_data[column] = output_data[column].astype(int)
                output_data[column] = output_data[column].astype(str)
        return output_data


class AppendTextAction(Action):
    """
    self.instructions: list of dicts
        keys:
            target_column
            result_column
            text
    """
    def perform_instructions(self, input_data):
        output_data = input_data
        for instruction in self.instructions:
            col = instruction['target_column']
            text = instruction['text']
            result_col = instruction['result_column']
            # TODO: factor out commonality between Prepend/Append
            output_data[result_col] = output_data[col] + text
        return output_data


class PrependTextAction(Action):<commit_msg>handle int -> string conversion better<commit_after>import pandas as pd
import re
import yaml


class UnknownActionError(Exception):
    pass


class ActionList:
    def __init__(self, list_of_actions):
        self.actions = []
        for step in list_of_actions:
            try:
                action_name = list(step.keys())[0]
                instruction = step[action_name]
            except AttributeError:
                action_name = step
                instruction = []
            action = Action.factory(action_name, instruction)
            self.actions.append(action)

    def perform_instructions(self, input_data):
        output_data = input_data
        for action in self.actions:
            output_data = action.perform_instructions(output_data)
        return output_data

    @classmethod
    def from_yaml(cls, filepath_or_buffer):
        if isinstance(filepath_or_buffer, str):
            with open(filepath_or_buffer) as f:
                actions = yaml.safe_load(f)
        else:
            actions = yaml.safe_load(filepath_or_buffer)
        return cls(actions)


# abstract, never used.
class Action:
    def __init__(self, instructions):
        self.instructions = instructions

    @staticmethod
    def factory(action, instruction):
        # just calls different constructors based on passed action
        action_classes = {
            'change_date_or_time_format': ChangeDateFormat,
            'copy_column': CopyAction,
            'rename_column': RenameAction,
            'extract_query_string': ExtractQueryStringAction,
            'extract_text': ExtractTextAction,
            'only_keep_these_columns': FilterColumnAction,
            'only_keep_rows_where': FilterRowAction,
            'only_edit_rows_where': EditSpecificRowsAction,
            'run_these_formula': FormulaAction,
            'remove_columns': RemoveColumnAction,
            'remove_duplicates': RemoveDuplicatesAction,
            'replace_text': ReplaceTextAction,
            'add_text_at_end': AppendTextAction,
            'add_text_at_start': PrependTextAction,
            'sum_up_by': GroupBySumAction,
            'make_column_names_lowercase': LowerCaseColumnNamesAction,
            'make_column_names_alphanumeric': AlphaNumColumnNamesAction,
            'ensure_column_is_in_this_format': ChangeColumnFormatAction,
        }
        try:
            action_class = action_classes[action]
            return action_class(instruction)
        except KeyError:
            msg = 'action {} unknown'.format(action)
            raise UnknownActionError(msg)


class GroupBySumAction(Action):
    """
    self.instructions: list of columns to group by.
    """
    def perform_instructions(self, input_data):
        columns_to_group_by = self.instructions
        grouped = input_data.groupby(columns_to_group_by, as_index=False)
        output_data = grouped.sum()
        return output_data


class ChangeDateFormat(Action):
    """
    self.instructions: list of dicts
        keys:
            target_column
            result_column
            date_format
    """
    def perform_instructions(self, input_data):
        for instruction in self.instructions:
            result_col = instruction['result_column']
            date_format = instruction['date_format']
            date_format_strings = (
                # TODO: hour/minute/second support.
                ('YYYY', '%Y'),
                ('YY', '%y'),
                ('MM', '%m'),
                ('DD', '%d'),
            )
            for date_str, strformat in date_format_strings:
                date_format = date_format.replace(date_str, strformat)
                date_format = date_format.replace(date_str.lower(), strformat)
            format_date = lambda x: x.strftime(date_format)
            dates = input_data[instruction['target_column']].dropna()
            input_data[result_col] = dates.map(format_date)
        return input_data


class AlphaNumColumnNamesAction(Action):
    """
    """
    def perform_instructions(self, input_data):
        spaces_to_underscore = lambda x: x.replace(' ', '_')
        output_data = input_data.rename(columns=spaces_to_underscore)
        strip_punctuation = lambda x: re.sub('[^\w_]', '', x)
        output_data = output_data.rename(columns=strip_punctuation)
        return output_data


class LowerCaseColumnNamesAction(Action):
    """
    """
    def perform_instructions(self, input_data):
        output_data = input_data.rename(columns=str.lower)
        return output_data


class ChangeColumnFormatAction(Action):
    """
    self.instructions: list of dicts
    """
    def perform_instructions(self, input_data):
        output_data = input_data
        for instruction in self.instructions:
            column_format = list(instruction.values())[0]
            column = list(instruction.keys())[0]
            if column_format == 'date':
                output_data[column] = pd.to_datetime(output_data[column])
            if column_format == 'text':
                output_data[column] = output_data[column].dropna().astype(int)
                output_data[column] = output_data[column].dropna().astype(str)
                output_data[column] = output_data[column].fillna('')
        return output_data


class AppendTextAction(Action):
    """
    self.instructions: list of dicts
        keys:
            target_column
            result_column
            text
    """
    def perform_instructions(self, input_data):
        output_data = input_data
        for instruction in self.instructions:
            col = instruction['target_column']
            text = instruction['text']
            result_col = instruction['result_column']
            # TODO: factor out commonality between Prepend/Append
            output_data[result_col] = output_data[col] + text
        return output_data


class PrependTextAction(Action):
{'fraction_non_alphanumeric': 0.08713480266529984, 'fraction_numerical': 0.0005125576627370579, 'mean_word_length': 2.388422575976845, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2443867', 'n_tokens_mistral': 3204, 'n_tokens_neox': 3116, 'n_words': 629}