<commit_before># Copyright 2017-2020 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import copy
import errno
import logging
import os
import re
import sys

import six
import yaml

from guild import config
from guild import opref
from guild import resourcedef
from guild import util

log = logging.getLogger("guild")

NAME = "guild.yml"

ALL_TYPES = [
    "config",
    "include",
    "model",
    "package",
]

MODEL_TYPES = ["model", "config"]

STRING_SRC_P = re.compile(r"<.*>$")
INCLUDE_REF_P = re.compile(r"([^:#]+)(?::([^#]+))?(?:#(.+))?")
INCLUDE_REF_P1 = re.compile(r"([^:#]*):([^#]+)(?:#(.+))?")
INCLUDE_REF_P2 = re.compile(r"([^:#]+)(?:#(.+))?")
INCLUDE_REF_P_DESC = "CONFIG[#ATTRS] or MODEL:OPERATION[#ATTRS]"
PARAM_P = re.compile(r"({{.*?}})")

DEFAULT_PKG_VERSION = "0.0.0"

_cache = {}

###################################################################
# Exceptions
###################################################################


class GuildfileError(Exception):
    def __init__(self, guildfile_or_path, msg):
        super(GuildfileError, self).__init__(guildfile_or_path, msg)
        if isinstance(guildfile_or_path, Guildfile):
            self.path = guildfile_or_path.src
        else:
            self.path = guildfile_or_path
        self.msg = msg

    def __str__(self):
        return "error in %s: %s" % (self.path or "<generated>", self.msg)


class NoModels(GuildfileError):
    def __init__(self, guildfile_or_path):
        super(NoModels, self).__init__(guildfile_or_path, "no models")

    def __str__(self):
        return "no models in %s" % self.path


class GuildfileReferenceError(GuildfileError):
    pass


class GuildfileCycleError(GuildfileError):
    def __init__(self, guildfile_or_path, desc, cycle):
        msg = "%s (%s)" % (desc, " -> ".join(cycle))
        super(GuildfileCycleError, self).__init__(guildfile_or_path, msg)


class GuildfileIncludeError(GuildfileError):
    def __init__(self, guildfile_or_path, include):
        msg = (
            "cannot find include '%s' "
            "(includes must be local to including Guild file or a "
            "Guild package on the system path)" % include
        )
        super(GuildfileIncludeError, self).__init__(guildfile_or_path, msg)


class GuildfileMissing(Exception):
    pass


###################################################################
# Helpers
###################################################################


def _required(name, data, guildfile, pop=False):
    try:
        if pop:
            return data.pop(name)
        else:
            return data[name]
    except KeyError:
        raise GuildfileError(
            guildfile, "missing required '%s' attribute in %r" % (name, data)
        )


def _string_source(src):
    return STRING_SRC_P.match(src)


###################################################################
# Guildfile
###################################################################


class Guildfile(object):
    def __init__(self, data, src=None, dir=None, included=None, extends_seen=None):
        if not dir and src and not _string_source(src):
            dir = os.path.dirname(src)
        if src is None and dir is None:
            raise ValueError("either src or dir must be specified")
        self.src = src
        self.dir = dir
        self.models = {}
        self.package = None
        coerced = _coerce_guildfile_data(data, self)
        self.data = self._expand_data_includes(coerced, included or [])
        try:
            self._apply_data(extends_seen or [])
        except (GuildfileError, resourcedef.ResourceFormatError):
            raise
        except Exception as e:
            log.error("loading %s: %r", self.src, e)
            raise

    def _expand_data_includes(self, data, included):
        i = 0
        while i < len(data):
            item = data[i]
            try:
                includes = item["include"]
            except KeyError:
                i += 1
            else:
                new_items = self._include_data(includes, included)
                data[i : i + 1] = new_items
                i += len(new_items)
        return data

    def _include_data(self, includes, included):
        if not _string_source(self.src):
            included.append(os.path.abspath(self.src))
        include_data = []
        for path in includes:
            path = self._find_include(path)
            if path in included:
                raise GuildfileCycleError(
                    "cycle in 'includes'", included[0], included + [path]
                )
            data = yaml.safe_load(open(path, "r"))
            guildfile = Guildfile(data, path, included=included)
            include_data.extend(guildfile.data)
        return include_data

    def _find_include(self, include):
        path = util.find_apply(
            [self._local_include, self._sys_path_include, self._gpkg_include], include
        )
        if path:
            return path
        raise GuildfileIncludeError(self, include)

    def _local_include(self, path):
        log.debug("looking for include '%s' in %s", path, self.dir)
        full_path = os.path.abspath(os.path.join(self.dir or "", path))
        if os.path.exists(full_path):
            log.debug("found include %s", full_path)
            return full_path
        return None

    @staticmethod
    def _sys_path_include(include):
        include_path = include.replace(".", os.path.sep)
        for path in sys.path:
            log.debug("looking for include '%s' in %s", include, path)
            guildfile = guildfile_path(path, include_path)
            if os.path.exists(guildfile):
                log.debug("found include %s", guildfile)
                return guildfile
        return None

    def _gpkg_include(self, include):
        return self._sys_path_include("gpkg." + include)

    def _apply_data(self, extends_seen):
        for item in self.data:
            item_type, name = self._validated_item_type(item)
            if item_type == "model":
                self._apply_model(name, item, extends_seen)
            elif item_type == "package":
                self._apply_package(name, item)

    def _validated_item_type(self, item):
        used = [name for name in ALL_TYPES if name in item]
        if not used:
            raise GuildfileError(
                self,
                (
                    "missing required type (one of: %s) in %r"
                    % (", ".join(ALL_TYPES), item)
                ),
            )
        elif len(used) > 1:
            raise GuildfileError(
                self, ("multiple types (%s) in %r" % (", ".join(used), item))
            )
        validated_type = used[0]
        name = item[validated_type]
        if not isinstance(name, six.string_types):
            raise GuildfileError(
                self, "invalid %s name %r: expected a string" % (validated_type, name)
            )
        return validated_type, name

    def _apply_model(self, name, data, extends_seen):
        if name in self.models:
            raise GuildfileError(self, "duplicate model '%s'" % name)
        model = ModelDef(name, data, self, extends_seen)
        self.models[name] = model

    def _apply_package(self, name, data):
        if self.package:
            raise GuildfileError(self, "mutiple package definitions")
        self.package = PackageDef(name, data, self)

    @property
    def default_model(self):
        models = list(self.models.values())
        if len(models) == 1:
            return models[0]
        for m in models:
            if m.default:
                return m
        return None

    @property
    def default_operation(self):
        model = self.default_model
        if not model:
            return None
        return model.default_operation

    def __repr__(self):
        return "<guild.guildfile.Guildfile '%s'>" % self

    def __str__(self):
        return self.src or self.dir

    def __eq__(self, _x):
        raise AssertionError()


###################################################################
# Coercion rules
###################################################################


def _coerce_guildfile_data(data, guildfile):
    if data is None:
        return []
    elif isinstance(data, dict):
        data = [_anonymous_model_data(data)]
    if not isinstance(data, list):
        raise GuildfileError(
            guildfile, "invalid guildfile data %r: expected a mapping" % data
        )
    return [_coerce_guildfile_item_data(item_data, guildfile) for item_data in data]


def _anonymous_model_data(ops_data):
    """Returns a dict representing an anonymous model.

    ops_data must be a dict representing the model operations. It will
    be used unmodified for the model `operations` attribute.
    """
    return {"model": "", "operations": ops_data}


def _coerce_guildfile_item_data(data, guildfile):
    """Coerces top-level object attributes based on name.

    Refer to _coerce_top_level_attr for coerced attributes.
    """
    if not isinstance(data, dict):
        return data
    coerced = {
        name: _coerce_top_level_attr(name, val, guildfile) for name, val in data.items()
    }
    _maybe_apply_anonymous_model(coerced)
    return coerced


def _coerce_top_level_attr(name, val, guildfile):
    """Coerces top-level object attributes by name.

    This function is unaware of context so an attribute "flags" under
    an operation is coerced using the same rules as a "flags"
    attribute under "config". This scheme relies on common rules
    applied to object attributes with the same name. This is not
    necessarily the case (e.g. "flags" for a step defines values, not
    flag defs) and so is a limitation of this approach.
    """
    if name == "include":
        return _coerce_include(val, guildfile)
    elif name == "extends":
        return _coerce_extends(val, guildfile)
    elif name == "operations":
        return _coerce_operations(val, guildfile)
    elif name == "flags":
        return _coerce_flags(val, guildfile)
    elif name == "sourcecode":
        return _coerce_select_files(val, guildfile)
    else:
        return val


def _coerce_include(data, guildfile):
    return _coerce_str_to_list(data, guildfile, "include")


def _coerce_extends(data, guildfile):
    return _coerce_str_to_list(data, guildfile, "extends")


def _coerce_operations(data, guildfile):
    if not isinstance(data, dict):
        raise GuildfileError(
            guildfile, "invalid operations value %r: expected a mapping" % data
        )
    return {
        op_name: _coerce_operation(op_name, op, guildfile)
        for op_name, op in data.items()
    }


def _coerce_operation(name, data, guildfile):
    if name == "$include":
        return data
    elif isinstance(data, six.string_types):
        return {"main": data}
    elif isinstance(data, dict):
        return {
            name: _coerce_operation_attr(name, val, guildfile)
            for name, val in data.items()
        }
    else:
        raise GuildfileError(
            guildfile,
            "invalid value for operation '%s' %r: expected a string or a mapping"
            % (data, name),
        )


def _coerce_operation_attr(name, val, guildfile):
    if name == "flags":
        return _coerce_flags(val, guildfile)
    elif name == "flags-import":
        return _coerce_flags_import(val, guildfile)
    elif name == "publish":
        return _coerce_publish(val, guildfile)
    elif name == "python-path":
        return _coerce_op_python_path(val, guildfile)
    elif name == "output-scalars":
        return _coerce_output_scalars(val, guildfile)
    elif name == "sourcecode":
        return _coerce_select_files(val, guildfile)
    else:
        return val


def _coerce_flags(data, guildfile):
    if not isinstance(data, dict):
        raise GuildfileError(
            guildfile, "invalid flags value %r: expected a mapping" % data
        )
    return {name: coerce_flag_data(name, val, guildfile) for name, val in data.items()}


def coerce_flag_data(name, data, guildfile):
    if name == "$include":
        return data
    elif isinstance(data, dict):
        return data
    elif isinstance(data, six.string_types + (int, float, bool, list)):
        return {"default": data}
    elif data is None:
        return {"default": None}
    else:
        raise GuildfileError(
            guildfile,
            "invalid value for %s flag %r: expected a mapping of "
            "flag attributes or default flag value" % (name, data),
        )


def _coerce_flags_import(data, gf):
    if data in (True, "all"):
        return True
    elif data is False:
        return []
    elif isinstance(data, list):
        return data
    else:
        raise GuildfileError(
            gf,
            "invalid flags-import value %r: expected yes/all, "
            "no, or a list of flag names" % data,
        )


def _coerce_op_python_path(data, guildfile):
    if data is None:
        return None
    return _coerce_str_to_list(data, guildfile, "python-path")


def _coerce_output_scalars(data, guildfile):
    if data is None:
        return None
    elif data is False:
        return []
    elif isinstance(data, six.string_types):
        return [data]
    elif isinstance(data, dict):
        return [data]
    elif isinstance(data, list):
        return data
    else:
        raise GuildfileError(
            guildfile,
            "invalid output-scalars %r: expected a mapping, list, or string" % data,
        )


def _coerce_publish(data, guildfile):
    try:
        files = data["files"]
    except KeyError:<commit_msg>Tweak to error message for invalid output-scalars attr<commit_after># Copyright 2017-2020 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import copy
import errno
import logging
import os
import re
import sys

import six
import yaml

from guild import config
from guild import opref
from guild import resourcedef
from guild import util

log = logging.getLogger("guild")

NAME = "guild.yml"

ALL_TYPES = [
    "config",
    "include",
    "model",
    "package",
]

MODEL_TYPES = ["model", "config"]

STRING_SRC_P = re.compile(r"<.*>$")
INCLUDE_REF_P = re.compile(r"([^:#]+)(?::([^#]+))?(?:#(.+))?")
INCLUDE_REF_P1 = re.compile(r"([^:#]*):([^#]+)(?:#(.+))?")
INCLUDE_REF_P2 = re.compile(r"([^:#]+)(?:#(.+))?")
INCLUDE_REF_P_DESC = "CONFIG[#ATTRS] or MODEL:OPERATION[#ATTRS]"
PARAM_P = re.compile(r"({{.*?}})")

DEFAULT_PKG_VERSION = "0.0.0"

_cache = {}

###################################################################
# Exceptions
###################################################################


class GuildfileError(Exception):
    def __init__(self, guildfile_or_path, msg):
        super(GuildfileError, self).__init__(guildfile_or_path, msg)
        if isinstance(guildfile_or_path, Guildfile):
            self.path = guildfile_or_path.src
        else:
            self.path = guildfile_or_path
        self.msg = msg

    def __str__(self):
        return "error in %s: %s" % (self.path or "<generated>", self.msg)


class NoModels(GuildfileError):
    def __init__(self, guildfile_or_path):
        super(NoModels, self).__init__(guildfile_or_path, "no models")

    def __str__(self):
        return "no models in %s" % self.path


class GuildfileReferenceError(GuildfileError):
    pass


class GuildfileCycleError(GuildfileError):
    def __init__(self, guildfile_or_path, desc, cycle):
        msg = "%s (%s)" % (desc, " -> ".join(cycle))
        super(GuildfileCycleError, self).__init__(guildfile_or_path, msg)


class GuildfileIncludeError(GuildfileError):
    def __init__(self, guildfile_or_path, include):
        msg = (
            "cannot find include '%s' "
            "(includes must be local to including Guild file or a "
            "Guild package on the system path)" % include
        )
        super(GuildfileIncludeError, self).__init__(guildfile_or_path, msg)


class GuildfileMissing(Exception):
    pass


###################################################################
# Helpers
###################################################################


def _required(name, data, guildfile, pop=False):
    try:
        if pop:
            return data.pop(name)
        else:
            return data[name]
    except KeyError:
        raise GuildfileError(
            guildfile, "missing required '%s' attribute in %r" % (name, data)
        )


def _string_source(src):
    return STRING_SRC_P.match(src)


###################################################################
# Guildfile
###################################################################


class Guildfile(object):
    def __init__(self, data, src=None, dir=None, included=None, extends_seen=None):
        if not dir and src and not _string_source(src):
            dir = os.path.dirname(src)
        if src is None and dir is None:
            raise ValueError("either src or dir must be specified")
        self.src = src
        self.dir = dir
        self.models = {}
        self.package = None
        coerced = _coerce_guildfile_data(data, self)
        self.data = self._expand_data_includes(coerced, included or [])
        try:
            self._apply_data(extends_seen or [])
        except (GuildfileError, resourcedef.ResourceFormatError):
            raise
        except Exception as e:
            log.error("loading %s: %r", self.src, e)
            raise

    def _expand_data_includes(self, data, included):
        i = 0
        while i < len(data):
            item = data[i]
            try:
                includes = item["include"]
            except KeyError:
                i += 1
            else:
                new_items = self._include_data(includes, included)
                data[i : i + 1] = new_items
                i += len(new_items)
        return data

    def _include_data(self, includes, included):
        if not _string_source(self.src):
            included.append(os.path.abspath(self.src))
        include_data = []
        for path in includes:
            path = self._find_include(path)
            if path in included:
                raise GuildfileCycleError(
                    "cycle in 'includes'", included[0], included + [path]
                )
            data = yaml.safe_load(open(path, "r"))
            guildfile = Guildfile(data, path, included=included)
            include_data.extend(guildfile.data)
        return include_data

    def _find_include(self, include):
        path = util.find_apply(
            [self._local_include, self._sys_path_include, self._gpkg_include], include
        )
        if path:
            return path
        raise GuildfileIncludeError(self, include)

    def _local_include(self, path):
        log.debug("looking for include '%s' in %s", path, self.dir)
        full_path = os.path.abspath(os.path.join(self.dir or "", path))
        if os.path.exists(full_path):
            log.debug("found include %s", full_path)
            return full_path
        return None

    @staticmethod
    def _sys_path_include(include):
        include_path = include.replace(".", os.path.sep)
        for path in sys.path:
            log.debug("looking for include '%s' in %s", include, path)
            guildfile = guildfile_path(path, include_path)
            if os.path.exists(guildfile):
                log.debug("found include %s", guildfile)
                return guildfile
        return None

    def _gpkg_include(self, include):
        return self._sys_path_include("gpkg." + include)

    def _apply_data(self, extends_seen):
        for item in self.data:
            item_type, name = self._validated_item_type(item)
            if item_type == "model":
                self._apply_model(name, item, extends_seen)
            elif item_type == "package":
                self._apply_package(name, item)

    def _validated_item_type(self, item):
        used = [name for name in ALL_TYPES if name in item]
        if not used:
            raise GuildfileError(
                self,
                (
                    "missing required type (one of: %s) in %r"
                    % (", ".join(ALL_TYPES), item)
                ),
            )
        elif len(used) > 1:
            raise GuildfileError(
                self, ("multiple types (%s) in %r" % (", ".join(used), item))
            )
        validated_type = used[0]
        name = item[validated_type]
        if not isinstance(name, six.string_types):
            raise GuildfileError(
                self, "invalid %s name %r: expected a string" % (validated_type, name)
            )
        return validated_type, name

    def _apply_model(self, name, data, extends_seen):
        if name in self.models:
            raise GuildfileError(self, "duplicate model '%s'" % name)
        model = ModelDef(name, data, self, extends_seen)
        self.models[name] = model

    def _apply_package(self, name, data):
        if self.package:
            raise GuildfileError(self, "mutiple package definitions")
        self.package = PackageDef(name, data, self)

    @property
    def default_model(self):
        models = list(self.models.values())
        if len(models) == 1:
            return models[0]
        for m in models:
            if m.default:
                return m
        return None

    @property
    def default_operation(self):
        model = self.default_model
        if not model:
            return None
        return model.default_operation

    def __repr__(self):
        return "<guild.guildfile.Guildfile '%s'>" % self

    def __str__(self):
        return self.src or self.dir

    def __eq__(self, _x):
        raise AssertionError()


###################################################################
# Coercion rules
###################################################################


def _coerce_guildfile_data(data, guildfile):
    if data is None:
        return []
    elif isinstance(data, dict):
        data = [_anonymous_model_data(data)]
    if not isinstance(data, list):
        raise GuildfileError(
            guildfile, "invalid guildfile data %r: expected a mapping" % data
        )
    return [_coerce_guildfile_item_data(item_data, guildfile) for item_data in data]


def _anonymous_model_data(ops_data):
    """Returns a dict representing an anonymous model.

    ops_data must be a dict representing the model operations. It will
    be used unmodified for the model `operations` attribute.
    """
    return {"model": "", "operations": ops_data}


def _coerce_guildfile_item_data(data, guildfile):
    """Coerces top-level object attributes based on name.

    Refer to _coerce_top_level_attr for coerced attributes.
    """
    if not isinstance(data, dict):
        return data
    coerced = {
        name: _coerce_top_level_attr(name, val, guildfile) for name, val in data.items()
    }
    _maybe_apply_anonymous_model(coerced)
    return coerced


def _coerce_top_level_attr(name, val, guildfile):
    """Coerces top-level object attributes by name.

    This function is unaware of context so an attribute "flags" under
    an operation is coerced using the same rules as a "flags"
    attribute under "config". This scheme relies on common rules
    applied to object attributes with the same name. This is not
    necessarily the case (e.g. "flags" for a step defines values, not
    flag defs) and so is a limitation of this approach.
    """
    if name == "include":
        return _coerce_include(val, guildfile)
    elif name == "extends":
        return _coerce_extends(val, guildfile)
    elif name == "operations":
        return _coerce_operations(val, guildfile)
    elif name == "flags":
        return _coerce_flags(val, guildfile)
    elif name == "sourcecode":
        return _coerce_select_files(val, guildfile)
    else:
        return val


def _coerce_include(data, guildfile):
    return _coerce_str_to_list(data, guildfile, "include")


def _coerce_extends(data, guildfile):
    return _coerce_str_to_list(data, guildfile, "extends")


def _coerce_operations(data, guildfile):
    if not isinstance(data, dict):
        raise GuildfileError(
            guildfile, "invalid operations value %r: expected a mapping" % data
        )
    return {
        op_name: _coerce_operation(op_name, op, guildfile)
        for op_name, op in data.items()
    }


def _coerce_operation(name, data, guildfile):
    if name == "$include":
        return data
    elif isinstance(data, six.string_types):
        return {"main": data}
    elif isinstance(data, dict):
        return {
            name: _coerce_operation_attr(name, val, guildfile)
            for name, val in data.items()
        }
    else:
        raise GuildfileError(
            guildfile,
            "invalid value for operation '%s' %r: expected a string or a mapping"
            % (data, name),
        )


def _coerce_operation_attr(name, val, guildfile):
    if name == "flags":
        return _coerce_flags(val, guildfile)
    elif name == "flags-import":
        return _coerce_flags_import(val, guildfile)
    elif name == "publish":
        return _coerce_publish(val, guildfile)
    elif name == "python-path":
        return _coerce_op_python_path(val, guildfile)
    elif name == "output-scalars":
        return _coerce_output_scalars(val, guildfile)
    elif name == "sourcecode":
        return _coerce_select_files(val, guildfile)
    else:
        return val


def _coerce_flags(data, guildfile):
    if not isinstance(data, dict):
        raise GuildfileError(
            guildfile, "invalid flags value %r: expected a mapping" % data
        )
    return {name: coerce_flag_data(name, val, guildfile) for name, val in data.items()}


def coerce_flag_data(name, data, guildfile):
    if name == "$include":
        return data
    elif isinstance(data, dict):
        return data
    elif isinstance(data, six.string_types + (int, float, bool, list)):
        return {"default": data}
    elif data is None:
        return {"default": None}
    else:
        raise GuildfileError(
            guildfile,
            "invalid value for %s flag %r: expected a mapping of "
            "flag attributes or default flag value" % (name, data),
        )


def _coerce_flags_import(data, gf):
    if data in (True, "all"):
        return True
    elif data is False:
        return []
    elif isinstance(data, list):
        return data
    else:
        raise GuildfileError(
            gf,
            "invalid flags-import value %r: expected yes/all, "
            "no, or a list of flag names" % data,
        )


def _coerce_op_python_path(data, guildfile):
    if data is None:
        return None
    return _coerce_str_to_list(data, guildfile, "python-path")


def _coerce_output_scalars(data, guildfile):
    if data is None:
        return None
    elif data is False:
        return []
    elif isinstance(data, six.string_types):
        return [data]
    elif isinstance(data, dict):
        return [data]
    elif isinstance(data, list):
        return data
    else:
        raise GuildfileError(
            guildfile,
            "invalid output-scalars %r: expected a mapping, list, string, or false"
            % data,
        )


def _coerce_publish(data, guildfile):
    try:
        files = data["files"]
    except KeyError:
{'fraction_non_alphanumeric': 0.133716448244446, 'fraction_numerical': 0.001752049898381106, 'mean_word_length': 2.5522778192681104, 'pattern_counts': {'":': 40, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1411823', 'n_tokens_mistral': 8584, 'n_tokens_neox': 8121, 'n_words': 2391}