<commit_before>from harpoon.errors import BadConfiguration, BadTask, BadYaml
from harpoon.formatter import MergedOptionStringFormatter
from harpoon.option_spec.harpoon_specs import HarpoonSpec
from harpoon.processes import command_output
from option_merge.storage import Converter
from harpoon.tasks import available_tasks
from harpoon.option_spec.objs import Task

from option_merge import MergedOptions
from input_algorithms.meta import Meta
import logging
import yaml
import os

log = logging.getLogger("harpoon.executor")

class Harpoon(object):
    def __init__(self, configuration_file, docker_context, logging_handler=None):
        self.docker_context = docker_context
        self.logging_handler = logging_handler

        self.configuration = self.collect_configuration(configuration_file)
        self.configuration_folder = os.path.dirname(os.path.abspath(configuration_file))
        self.setup_logging_theme()

    def start(self, cli_args):
        """Do the harpooning"""
        if not self.configuration.get("images"):
            raise BadConfiguration("Didn't find any images in the configuration")

        self.configuration.update(
            { "$@": cli_args["harpoon"].get("extra", "")
            , "config_root" : self.configuration_folder
            }
        )

        tasks = self.find_tasks()
        task = cli_args["harpoon"]["chosen_task"]
        if task not in tasks:
            raise BadTask("Unknown task", task=task, available=tasks.keys())

        tasks[task].run(self, cli_args)

    ########################
    ###   THEME
    ########################

    def setup_logging_theme(self):
        """Setup a logging theme"""
        if "term_colors" not in self.configuration:
            return

        if not getattr(self, "logging_handler", None):
            log.warning("Told to set term_colors but don't have a logging_handler to change")
            return

        colors = self.configuration.get("term_colors")
        if not colors:
            return

        if colors not in ("light", "dark"):
            log.warning("Told to set colors to a theme we don't have\tgot=%s\thave=[light, dark]", colors)
            return

        # Haven't put much effort into actually working out more than just the message colour
        if colors == "light":
            self.logging_handler._column_color['%(message)s'][logging.INFO] = ('cyan', None, False)
        else:
            self.logging_handler._column_color['%(message)s'][logging.INFO] = ('blue', None, False)

    ########################
    ###   CONFIG
    ########################

    def home_dir_configuration(self):
        """Return a dictionary from ~/.harpoon.yml"""
        location = os.path.expanduser("~/.harpoon.yml")
        if not os.path.exists(location):
            return None, {}

        result = self.read_yaml(location)
        if not result:
            result = {}
        if not isinstance(result, dict):
            raise BadYaml("Expected yaml file to declare a dictionary", location=location, got=type(result))

        result["__mtime__"] = self.get_committime_or_mtime(location)
        return location, result

    def read_yaml(self, filepath):
        """Read in a yaml file"""
        try:
            if os.stat(filepath).st_size == 0:
                return {}
            return yaml.load(open(filepath))
        except yaml.parser.ParserError as error:
            raise BadYaml("Failed to read yaml", location=filepath, error_type=error.__class__.__name__, error=error.problem)

    def get_committime_or_mtime(self, location):
        """Get the commit time of some file or the modified time of of it if can't get from git"""
        date, status = command_output("git show -s --format=%at -n1 -- {0}".format(os.path.basename(location)), cwd=os.path.dirname(location))
        if status == 0 and date:
            return int(date[0])
        else:
            return os.path.getmtime(location)

    def collect_configuration(self, configuration_file):
        """Return us a MergedOptions with this configuration and any collected configurations"""
        errors = []
        result = self.read_yaml(configuration_file)
        result["__mtime__"] = self.get_committime_or_mtime(configuration_file)

        images = {}
        if "images" in result:
            images = result.pop("images")

        configuration = MergedOptions.using(result, source=configuration_file)
        configuration_dir = os.path.dirname(os.path.abspath(configuration_file))
        configuration["images"] = MergedOptions()

        source, conf = self.home_dir_configuration()
        if conf:
            configuration.update(conf, source=source)

        harpoon_spec = HarpoonSpec()

        if isinstance(images, dict):
            for image, result in images.items():
                tasks = {}
                if 'tasks' in result:
                    tasks = result.pop("tasks")

                def convert_image(val):
                    return harpoon_spec.image_spec.normalise(Meta(configuration, [("images", ""), (image, "")]), val)
                configuration["images"].update({image: result}, source=configuration_file, converter=Converter(convert=convert_image, convert_path=image))

                def convert_task(val):
                    spec = harpoon_spec.tasks_spec(available_tasks)
                    meta = Meta(configuration, [('images', ''), (image, ""), ('tasks', ""), (image, "")])
                    return spec.normalise(meta, val)

                configuration["images"].update({image: {"tasks": tasks}}
                    , source=configuration_file
                    , converter=Converter(convert=convert_task, convert_path = [image, "tasks"])
                    )

        if "images.__images_from__" in configuration:
            images_from = MergedOptionStringFormatter(configuration, "images.__images_from__").format()
            del configuration["images.__images_from__"]

            if not images_from.startswith("/"):
                images_from = os.path.join(configuration_dir, images_from)

            if not os.path.exists(images_from) or not os.path.isdir(images_from):
                raise BadConfiguration("Specified folder for other configuration files points to a folder that doesn't exist", path="images.__images_from__", value=images_from)

            for root, dirs, files in os.walk(images_from):
                for fle in files:
                    if fle.endswith(".yml") or fle.endswith(".yaml"):
                        location = os.path.join(root, fle)
                        try:
                            name = os.path.splitext(fle)[0]
                            result = self.read_yaml(location)
                            result["__mtime__"] = self.get_committime_or_mtime(location)

                            images = {}
                            if 'images' in result:
                                images = result.pop("images")

                            if isinstance(images, dict):
                                for image, result in images.items():
                                    tasks = {}
                                    if 'tasks' in result:
                                        tasks = result.pop("tasks")

                                    def convert_image(val):
                                        return harpoon_spec.image_spec.normalise(Meta(configuration, [("images", ""), (image, "")]), val)
                                    configuration["images"].update({image: result}, source=location, converter=Converter(convert=convert_image, convert_path=image))

                                    def convert_task(val):
                                        spec = harpoon_spec.tasks_spec(available_tasks)
                                        meta = Meta(configuration, [('images', ''), (image, ""), ('tasks', ""), (image, "")])
                                        return spec.normalise(meta, val)

                                    configuration["images"].update({image: {"tasks": tasks}}
                                        , source=location
                                        , converter=Converter(convert=convert_task, convert_path = [image, "tasks"])
                                        )

                            configuration["images"].update({name: result}, source=location)
                        except BadYaml as error:
                            errors.append(error)<commit_msg>Images need to be added after tasks<commit_after>from harpoon.errors import BadConfiguration, BadTask, BadYaml
from harpoon.formatter import MergedOptionStringFormatter
from harpoon.option_spec.harpoon_specs import HarpoonSpec
from harpoon.processes import command_output
from option_merge.storage import Converter
from harpoon.tasks import available_tasks
from harpoon.option_spec.objs import Task

from option_merge import MergedOptions
from input_algorithms.meta import Meta
import logging
import yaml
import os

log = logging.getLogger("harpoon.executor")

class Harpoon(object):
    def __init__(self, configuration_file, docker_context, logging_handler=None):
        self.docker_context = docker_context
        self.logging_handler = logging_handler

        self.configuration = self.collect_configuration(configuration_file)
        self.configuration_folder = os.path.dirname(os.path.abspath(configuration_file))
        self.setup_logging_theme()

    def start(self, cli_args):
        """Do the harpooning"""
        if not self.configuration.get("images"):
            raise BadConfiguration("Didn't find any images in the configuration")

        self.configuration.update(
            { "$@": cli_args["harpoon"].get("extra", "")
            , "config_root" : self.configuration_folder
            }
        )

        tasks = self.find_tasks()
        task = cli_args["harpoon"]["chosen_task"]
        if task not in tasks:
            raise BadTask("Unknown task", task=task, available=tasks.keys())

        tasks[task].run(self, cli_args)

    ########################
    ###   THEME
    ########################

    def setup_logging_theme(self):
        """Setup a logging theme"""
        if "term_colors" not in self.configuration:
            return

        if not getattr(self, "logging_handler", None):
            log.warning("Told to set term_colors but don't have a logging_handler to change")
            return

        colors = self.configuration.get("term_colors")
        if not colors:
            return

        if colors not in ("light", "dark"):
            log.warning("Told to set colors to a theme we don't have\tgot=%s\thave=[light, dark]", colors)
            return

        # Haven't put much effort into actually working out more than just the message colour
        if colors == "light":
            self.logging_handler._column_color['%(message)s'][logging.INFO] = ('cyan', None, False)
        else:
            self.logging_handler._column_color['%(message)s'][logging.INFO] = ('blue', None, False)

    ########################
    ###   CONFIG
    ########################

    def home_dir_configuration(self):
        """Return a dictionary from ~/.harpoon.yml"""
        location = os.path.expanduser("~/.harpoon.yml")
        if not os.path.exists(location):
            return None, {}

        result = self.read_yaml(location)
        if not result:
            result = {}
        if not isinstance(result, dict):
            raise BadYaml("Expected yaml file to declare a dictionary", location=location, got=type(result))

        result["__mtime__"] = self.get_committime_or_mtime(location)
        return location, result

    def read_yaml(self, filepath):
        """Read in a yaml file"""
        try:
            if os.stat(filepath).st_size == 0:
                return {}
            return yaml.load(open(filepath))
        except yaml.parser.ParserError as error:
            raise BadYaml("Failed to read yaml", location=filepath, error_type=error.__class__.__name__, error=error.problem)

    def get_committime_or_mtime(self, location):
        """Get the commit time of some file or the modified time of of it if can't get from git"""
        date, status = command_output("git show -s --format=%at -n1 -- {0}".format(os.path.basename(location)), cwd=os.path.dirname(location))
        if status == 0 and date:
            return int(date[0])
        else:
            return os.path.getmtime(location)

    def collect_configuration(self, configuration_file):
        """Return us a MergedOptions with this configuration and any collected configurations"""
        errors = []
        result = self.read_yaml(configuration_file)
        result["__mtime__"] = self.get_committime_or_mtime(configuration_file)

        images = {}
        if "images" in result:
            images = result.pop("images")

        configuration = MergedOptions.using(result, source=configuration_file)
        configuration_dir = os.path.dirname(os.path.abspath(configuration_file))
        configuration["images"] = MergedOptions()

        source, conf = self.home_dir_configuration()
        if conf:
            configuration.update(conf, source=source)

        harpoon_spec = HarpoonSpec()

        if isinstance(images, dict):
            for image, result in images.items():
                tasks = {}
                if 'tasks' in result:
                    tasks = result.pop("tasks")

                def convert_task(val):
                    spec = harpoon_spec.tasks_spec(available_tasks)
                    meta = Meta(configuration, [('images', ''), (image, ""), ('tasks', ""), (image, "")])
                    return spec.normalise(meta, val)

                configuration["images"].update({image: {"tasks": tasks}}
                    , source=configuration_file
                    , converter=Converter(convert=convert_task, convert_path = [image, "tasks"])
                    )

                def convert_image(val):
                    return harpoon_spec.image_spec.normalise(Meta(configuration, [("images", ""), (image, "")]), val)
                configuration["images"].update({image: result}, source=configuration_file, converter=Converter(convert=convert_image, convert_path=image))

        if "images.__images_from__" in configuration:
            images_from = MergedOptionStringFormatter(configuration, "images.__images_from__").format()
            del configuration["images.__images_from__"]

            if not images_from.startswith("/"):
                images_from = os.path.join(configuration_dir, images_from)

            if not os.path.exists(images_from) or not os.path.isdir(images_from):
                raise BadConfiguration("Specified folder for other configuration files points to a folder that doesn't exist", path="images.__images_from__", value=images_from)

            for root, dirs, files in os.walk(images_from):
                for fle in files:
                    if fle.endswith(".yml") or fle.endswith(".yaml"):
                        location = os.path.join(root, fle)
                        try:
                            name = os.path.splitext(fle)[0]
                            result = self.read_yaml(location)
                            result["__mtime__"] = self.get_committime_or_mtime(location)

                            images = {}
                            if 'images' in result:
                                images = result.pop("images")

                            if isinstance(images, dict):
                                for image, result in images.items():
                                    tasks = {}
                                    if 'tasks' in result:
                                        tasks = result.pop("tasks")

                                    def convert_task(val):
                                        spec = harpoon_spec.tasks_spec(available_tasks)
                                        meta = Meta(configuration, [('images', ''), (image, ""), ('tasks', ""), (image, "")])
                                        return spec.normalise(meta, val)

                                    configuration["images"].update({image: {"tasks": tasks}}
                                        , source=location
                                        , converter=Converter(convert=convert_task, convert_path = [image, "tasks"])
                                        )

                                    def convert_image(val):
                                        return harpoon_spec.image_spec.normalise(Meta(configuration, [("images", ""), (image, "")]), val)
                                    configuration["images"].update({image: result}, source=location, converter=Converter(convert=convert_image, convert_path=image))

                            configuration["images"].update({name: result}, source=location)
                        except BadYaml as error:
                            errors.append(error)
{'fraction_non_alphanumeric': 0.11462869247969872, 'fraction_numerical': 0.0007061315758503001, 'mean_word_length': 2.0256364607441695, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6496288', 'n_tokens_mistral': 4303, 'n_tokens_neox': 4127, 'n_words': 1009}