<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import re
import os
from collections import namedtuple


class DiffiosConfig(object):

    """DiffiosConfig prepares a Cisco IOS Config to diff.

    DiffiosConfig takes a Cisco IOS config, as a file or a
    list, extracts the device hostname, removes any invalid
    lines, such as comments, breaks the config into a
    hierarchical block structure and partitions the config
    according to a list of lines to ignore.

    Attributes:
        ignores (list): List of lines to ignore
        config (list): List of valid config lines

    Args:
        config (str|list): Path to config file, or list
            containing lines of config

    Kwargs:
        ignores (str|list): Path to ignores file, or list
            containing lines to ignore. Defaults to ignores
            file in current working directory.

    >>> config = [
    ... '!',
    ... 'hostname ROUTER',
    ... '!',
    ... 'interface FastEthernet0/1',
    ... ' description **Link to Core**',
    ... ' ip address 192.168.0.1 255.255.255.0']
    >>> ignores = [
    ... 'hostname',
    ... '^ description']
    >>> conf = DiffiosConfig(config, ignores=ignores)
    >>> conf.config
    ['hostname ROUTER', \
'interface FastEthernet0/1', \
' description **Link to Core**', \
' ip address 192.168.0.1 255.255.255.0']
    >>> conf.ignores
    ['hostname', '^ description']
    >>> conf.config_blocks
    [['hostname ROUTER'], \
['interface FastEthernet0/1', \
' description **Link to Core**', \
' ip address 192.168.0.1 255.255.255.0']]
    >>> conf.ignored
    [['hostname ROUTER'], [' description **Link to Core**']]
    >>> conf.recorded
    [['interface FastEthernet0/1', \
' ip address 192.168.0.1 255.255.255.0']]

    """

    def __init__(self, config, ignores=None):
        if ignores is None and os.path.exists(os.path.join(os.getcwd(), 'diffios_ignore')):
            ignores = os.path.join(os.getcwd(), "diffios_ignore")
        elif ignores is None:
            ignores = []
        self.config = self._config(config)
        self.ignores = self._ignores(ignores)

    @property
    def contains_variables(self, delimiter=None):
        """TODO: Docstring for contains_variables.

        Kwargs:
            delimiter (TODO): TODO

        Returns: TODO

        """
        if delimiter is None:
            delimiter = re.compile(r'{{(.+)}}')
        return any(re.search(delimiter, line) for line in self.config)

    def _config(self, data):
        """Transforms given config data into usable format,
            with only valid lines.

        Args:
            data (list|file): config as either list or file.

        Returns:
            list: The config as a list of valid lines.

        """
        return self._remove_invalid_lines(self._check_data(data))

    def _ignores(self, data):
        """Transforms given ignores data into usable format.

        Args:
            ignores (list|file): ignores as either list or file.

        Returns:
            list: Lines to ignore.

        """
        return [l.strip().lower() for l in self._check_data(data)]

    @staticmethod
    def _check_data(data):
        """Check type of data and convert it if necessary.

        Args:
            data (list|file): input data as either list or file.

        Returns:
            list: The input data as a list.

        Raises:
            RuntimeError: If file cannot be opened or given data
                is not a list or file.

        """
        if isinstance(data, list):
            return data
        elif os.path.isfile(data):
            try:
                with open(data) as fin:
                    return fin.readlines()
            except IOError:
                raise RuntimeError(("DiffiosConfig could not open '{}'".format(data)))
        else:
            raise RuntimeError(("[FATAL] DiffiosConfig() received an "
                                "invalid argument: config={}\n").format(data))

    def _remove_invalid_lines(self, config):
        """Remove invalid lines from a given config.

        Args:
            config (list): Config as a list of lines.

        Returns:
            list: Only valid lines in given config.

        """
        return [l.rstrip() for l in config if self._valid_line(l)]

    @staticmethod
    def _valid_line(line):
        """Assert whether a given line is valid.

        Lines considered invalid are those which are empty
        or begin with a '!' to indicate a comment.

        Args:
            line (str): A single line from a config.

        Returns:
            bool: True if line is valid, False if not.

        """
        lstrip = line.strip()
        return len(lstrip) > 0 and not lstrip.startswith("!")

    @staticmethod
    def _group_into_blocks(config):
        """Group config into hierarchical blocks.

        Blocks are defined as a parent/child relationship,
        where children lines start with a ' ', and parents
        are the immediately preceding line that doesn't start
        with a ' '. Blocks can be single lines where there
        is no children. Returns the blocks sorted.

        Example Blocks:
            interface FastEthernet0/1
             ip address 192.168.0.1 255.255.255.0
             no shutdown

            hostname ROUTER

        Args:
            config (list): config as a list of lines.

        Returns:
            list: config as a sorted list of lists, each list
                representing a hierarchical block of config.

        """
        current_group, groups = [], []
        for line in config:
            if not line.startswith(' ') and current_group:
                groups.append(current_group)
                current_group = [line]
            else:
                current_group.append(line)
        if current_group:
            groups.append(current_group)
        return sorted(groups)

    @property
    def config_blocks(self):
        """The config as sorted list of hierarchical blocks.

        Returns:
            list: self.config as a sorted list of lists, each list
                representing a hierarchical block of config.

        """
        return self._group_into_blocks(self.config)

    @property
    def hostname(self):
        """The hostname of the given config.

        Returns:
            str: hostname of the given config
                or None if not found.

        """
        for line in self.config:
            if "hostname" in line.lower():
                return line.split()[1]
        return None

    def _ignore_line(self, line):
        """Check if a line should be ignored.

        Args:
            line (str): line to check

        Returns:
            bool: True if line should be ignored,
                False otherwise.

        """
        for line_to_ignore in self.ignores:
            if re.search(line_to_ignore, line.lower()):
                return True

    def _check_block(self, block):
        """Check for any lines to be ignored in a given block.

        Args:
            block (list): A single hierarchical block of config

        Returns:
            tuple: ignored and recorded (not ignored) lines in block

        """
        ignored, recorded = [], []
        for i, line in enumerate(block):
            if self._ignore_line(line) and i == 0:
                return (block, recorded)
            elif self._ignore_line(line):
                ignored.append(line)
            else:
                recorded.append(line)
        return (ignored, recorded)

    def _partition(self):
        """Partition out lines to ignore from the config.

        Returns:
            namedtuple: list of ignored lines and list of
                recorded (not ignored) lines

        """
        Partition = namedtuple("Partition", "ignored recorded")
        config_blocks = self._group_into_blocks(self.config)
        ignored, recorded = [], []
        for block in config_blocks:
            ignore, record = self._check_block(block)
            if ignore:
                ignored.append(ignore)
            if record:
                recorded.append(record)
        return Partition(ignored, recorded)

    @property
    def ignored(self):
        """Lines in the config that are being ignored.

        Returns:
            list: ignored lines

        """
        return self._partition().ignored

    @property
    def recorded(self):
        """Lines in the config that are being recorded (not ignored).

        Returns:
            list: recorded (not ignored) lines

        """
        return self._partition().recorded


class CheckConfig(object):

    """Docstring for CheckConfig. """

    def __init__(self, block, config):
        """TODO: to be defined1.

        Args:
            block (TODO): TODO
            config (TODO): TODO

        """
        self._block = block
        self._config = config
        self.present = []
        self.not_present = []
        self.parent = self._block[0]
        self.children = self._block[1:]
        self.check()

    def _parent_child_config_dict(self):
        return {el[0]: el[1:] for el in self._config}

    def check(self):
        if not self.children and self._block in self._config:
            self.present.append(self.parent)
        elif not self.children:
            self.not_present.append(self.parent)
        elif self.parent in self._parent_child_config_dict().keys():
                self.present.append(self.parent)
                config_children = self._parent_child_config_dict()[self.parent]
                for i, child in enumerate(self.children):
                    if child in config_children:
                        self.present.append(child)
                    elif i == 0:
                        self.not_present.append(self.parent)
                        self.not_present.append(child)
                    else:
                        self.not_present.append(child)


class DiffiosDiff(object):

    """Docstring for DiffiosDiff. """

    def __init__(self, baseline=None, comparison=None, ignore_file=None):
        """TODO: Docstring for __init__.

        Kwargs:
            baseline (TODO): TODO
            comparison (TODO): TODO
            ignore_file (TODO): TODO

        """
        # TODO: make it so DiffiosConf objects can be passed in also
        # TODO: confirm existence of files
        self.baseline = DiffiosConfig(baseline, ignore_file)
        self.comparison = DiffiosConfig(comparison, ignore_file)
        self.delimiter = r'{{(.+)}}'

    def _translate(self, with_vars, without_vars):
        """TODO: Docstring for _translate.
        Returns: TODO

        """
        result = []
        for wv_block in with_vars:
            for wov_block in without_vars:
                result.append(self._translated_block(wv_block, wov_block))
        return result

    def _translated_block(self, wv_block, wov_block):
        """TODO: Docstring for _translated_block.

        Args:
            wv_block (TODO): TODO
            wov_block (TODO): TODO

        Returns: TODO

        """
        block = []
        delimiter = self.delimiter
        for wv_line in wv_block:
            for wov_line in wov_block:
                match = re.search(delimiter, wv_line)
                if match:
                    var = match.group()
                    start = wv_line.index(delimiter[0])
                    end = (wv_line.index(delimiter[-1]) + 2) - len(wv_line)
                    translated_line = wov_line.replace(wov_line[start:end], var)
                    block.append(translated_line)
                else:
                    block.append(wov_line)
        return block

    def _translated(self, with_vars, without_vars):
        """TODO: Docstring for _translated.

        Args:
            baseline (TODO): TODO
            comparison (TODO): TODO

        Returns: TODO

        """
        result = self._translate(with_vars, without_vars)
        return result

    def _changes(self, dynamic, static):
        """TODO: Docstring for _changes.

        Args:
            dynamic (TODO): TODO
            static (TODO): TODO

        Returns: TODO

        """
        translated_static = None
        translated_dynamic = None
        if dynamic.contains_variables:
            with_vars = dynamic.recorded
            without_vars = static.recorded
            translated_static = self._translated(with_vars, without_vars)
            translated_dynamic = dynamic.recorded
        else:
            with_vars = static.recorded
            without_vars = dynamic.recorded
            translated_dynamic = self._translated(with_vars, without_vars)
            translated_static = static.recorded
        head = [line[0] for line in static.recorded]
        changes = []
        for dynamic_index, dynamic_block in enumerate(translated_dynamic):
            if len(dynamic_block) == 1 and dynamic_block not in translated_static:
                changes.append(dynamic.recorded[dynamic_index])
            elif len(dynamic_block) > 1:
                first_line = dynamic_block[0]
                if first_line in head:
                    static_block = translated_static[head.index(first_line)]
                    additional = [first_line]
                    for dynamic_block_index, line in enumerate(dynamic_block):
                        if line not in static_block:
                            additional.append(
                                dynamic.recorded[dynamic_index][dynamic_block_index])
                    if len(additional) > 1:
                        changes.append(additional)
                else:
                    changes.append(dynamic.recorded[dynamic_index])
        return sorted(changes)

    @staticmethod
    def _format_changes(data):
        """TODO: Docstring for _format_changes.

        Args:
            data (TODO): TODO

        Returns: TODO

        """
        return "\n\n".join("\n".join(lines) for lines in data)

    @property
    def additional(self):
        """TODO: Docstring for additional.

        Returns: TODO

        """
        # comparison = self.comparison.recorded
        # baseline = self.baseline.recorded
        additional = self._changes(self.comparison, self.baseline)
        return additional

    @property
    def missing(self):
        """TODO: Docstring for missing.

        Returns: TODO

        """
        # comparison = self.comparison.recorded
        # baseline = self.baseline.recorded
        missing = self._changes(self.baseline, self.comparison)
        return missing

    @property
    def pprint_additional(self):
        """TODO: Docstring for pprint_additional.

        Returns: TODO

        """
        return self._format_changes(self.additional)

    @property
    def pprint_missing(self):
        """TODO: Docstring for pprint_missing.

        Returns: TODO

        """
        return self._format_changes(self.missing)

    def diff(self):
        """TODO: Docstring for diff.
<commit_msg>Progress???<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import re
import os
from collections import namedtuple


class DiffiosConfig(object):

    """DiffiosConfig prepares a Cisco IOS Config to diff.

    DiffiosConfig takes a Cisco IOS config, as a file or a
    list, extracts the device hostname, removes any invalid
    lines, such as comments, breaks the config into a
    hierarchical block structure and partitions the config
    according to a list of lines to ignore.

    Attributes:
        ignores (list): List of lines to ignore
        config (list): List of valid config lines

    Args:
        config (str|list): Path to config file, or list
            containing lines of config

    Kwargs:
        ignores (str|list): Path to ignores file, or list
            containing lines to ignore. Defaults to ignores
            file in current working directory.

    >>> config = [
    ... '!',
    ... 'hostname ROUTER',
    ... '!',
    ... 'interface FastEthernet0/1',
    ... ' description **Link to Core**',
    ... ' ip address 192.168.0.1 2192.168.127.12']
    >>> ignores = [
    ... 'hostname',
    ... '^ description']
    >>> conf = DiffiosConfig(config, ignores=ignores)
    >>> conf.config
    ['hostname ROUTER', \
'interface FastEthernet0/1', \
' description **Link to Core**', \
' ip address 192.168.0.1 255.255.255.0']
    >>> conf.ignores
    ['hostname', '^ description']
    >>> conf.config_blocks
    [['hostname ROUTER'], \
['interface FastEthernet0/1', \
' description **Link to Core**', \
' ip address 192.168.0.1 255.255.255.0']]
    >>> conf.ignored
    [['hostname ROUTER'], [' description **Link to Core**']]
    >>> conf.recorded
    [['interface FastEthernet0/1', \
' ip address 192.168.0.1 255.255.255.0']]

    """

    def __init__(self, config, ignores=None):
        if ignores is None and os.path.exists(os.path.join(os.getcwd(), 'diffios_ignore')):
            ignores = os.path.join(os.getcwd(), "diffios_ignore")
        elif ignores is None:
            ignores = []
        self.config = self._config(config)
        self.ignores = self._ignores(ignores)

    @property
    def contains_variables(self, delimiter=None):
        """TODO: Docstring for contains_variables.

        Kwargs:
            delimiter (TODO): TODO

        Returns: TODO

        """
        if delimiter is None:
            delimiter = re.compile(r'{{(.+)}}')
        return any(re.search(delimiter, line) for line in self.config)

    def _config(self, data):
        """Transforms given config data into usable format,
            with only valid lines.

        Args:
            data (list|file): config as either list or file.

        Returns:
            list: The config as a list of valid lines.

        """
        return self._remove_invalid_lines(self._check_data(data))

    def _ignores(self, data):
        """Transforms given ignores data into usable format.

        Args:
            ignores (list|file): ignores as either list or file.

        Returns:
            list: Lines to ignore.

        """
        return [l.strip().lower() for l in self._check_data(data)]

    @staticmethod
    def _check_data(data):
        """Check type of data and convert it if necessary.

        Args:
            data (list|file): input data as either list or file.

        Returns:
            list: The input data as a list.

        Raises:
            RuntimeError: If file cannot be opened or given data
                is not a list or file.

        """
        if isinstance(data, list):
            return data
        elif os.path.isfile(data):
            try:
                with open(data) as fin:
                    return fin.readlines()
            except IOError:
                raise RuntimeError(("DiffiosConfig could not open '{}'".format(data)))
        else:
            raise RuntimeError(("[FATAL] DiffiosConfig() received an "
                                "invalid argument: config={}\n").format(data))

    def _remove_invalid_lines(self, config):
        """Remove invalid lines from a given config.

        Args:
            config (list): Config as a list of lines.

        Returns:
            list: Only valid lines in given config.

        """
        return [l.rstrip() for l in config if self._valid_line(l)]

    @staticmethod
    def _valid_line(line):
        """Assert whether a given line is valid.

        Lines considered invalid are those which are empty
        or begin with a '!' to indicate a comment.

        Args:
            line (str): A single line from a config.

        Returns:
            bool: True if line is valid, False if not.

        """
        lstrip = line.strip()
        return len(lstrip) > 0 and not lstrip.startswith("!")

    @staticmethod
    def _group_into_blocks(config):
        """Group config into hierarchical blocks.

        Blocks are defined as a parent/child relationship,
        where children lines start with a ' ', and parents
        are the immediately preceding line that doesn't start
        with a ' '. Blocks can be single lines where there
        is no children. Returns the blocks sorted.

        Example Blocks:
            interface FastEthernet0/1
             ip address 192.168.0.1 255.255.255.0
             no shutdown

            hostname ROUTER

        Args:
            config (list): config as a list of lines.

        Returns:
            list: config as a sorted list of lists, each list
                representing a hierarchical block of config.

        """
        current_group, groups = [], []
        for line in config:
            if not line.startswith(' ') and current_group:
                groups.append(current_group)
                current_group = [line]
            else:
                current_group.append(line)
        if current_group:
            groups.append(current_group)
        return sorted(groups)

    @property
    def config_blocks(self):
        """The config as sorted list of hierarchical blocks.

        Returns:
            list: self.config as a sorted list of lists, each list
                representing a hierarchical block of config.

        """
        return self._group_into_blocks(self.config)

    @property
    def hostname(self):
        """The hostname of the given config.

        Returns:
            str: hostname of the given config
                or None if not found.

        """
        for line in self.config:
            if "hostname" in line.lower():
                return line.split()[1]
        return None

    def _ignore_line(self, line):
        """Check if a line should be ignored.

        Args:
            line (str): line to check

        Returns:
            bool: True if line should be ignored,
                False otherwise.

        """
        for line_to_ignore in self.ignores:
            if re.search(line_to_ignore, line.lower()):
                return True

    def _check_block(self, block):
        """Check for any lines to be ignored in a given block.

        Args:
            block (list): A single hierarchical block of config

        Returns:
            tuple: ignored and recorded (not ignored) lines in block

        """
        ignored, recorded = [], []
        for i, line in enumerate(block):
            if self._ignore_line(line) and i == 0:
                return (block, recorded)
            elif self._ignore_line(line):
                ignored.append(line)
            else:
                recorded.append(line)
        return (ignored, recorded)

    def _partition(self):
        """Partition out lines to ignore from the config.

        Returns:
            namedtuple: list of ignored lines and list of
                recorded (not ignored) lines

        """
        Partition = namedtuple("Partition", "ignored recorded")
        config_blocks = self._group_into_blocks(self.config)
        ignored, recorded = [], []
        for block in config_blocks:
            ignore, record = self._check_block(block)
            if ignore:
                ignored.append(ignore)
            if record:
                recorded.append(record)
        return Partition(ignored, recorded)

    @property
    def ignored(self):
        """Lines in the config that are being ignored.

        Returns:
            list: ignored lines

        """
        return self._partition().ignored

    @property
    def recorded(self):
        """Lines in the config that are being recorded (not ignored).

        Returns:
            list: recorded (not ignored) lines

        """
        return self._partition().recorded


class CheckConfig(object):

    """Docstring for CheckConfig. """

    def __init__(self, block, config):
        """TODO: to be defined1.

        Args:
            block (TODO): TODO
            config (TODO): TODO

        """
        self._block = block
        self._config = config
        self.present = []
        self.not_present = []
        self.parent = self._block[0]
        self.children = self._block[1:]
        self.check()

    def _parent_child_config_dict(self):
        return {el[0]: el[1:] for el in self._config}

    def check(self):
        if not self.children and self._block in self._config:
            self.present.append(self.parent)
        elif not self.children:
            self.not_present.append(self.parent)
        elif self.parent in self._parent_child_config_dict().keys():
                self.present.append(self.parent)
                config_children = self._parent_child_config_dict()[self.parent]
                for i, child in enumerate(self.children):
                    if child in config_children:
                        self.present.append(child)
                    elif i == 0:
                        self.not_present.append(self.parent)
                        self.not_present.append(child)
                    else:
                        self.not_present.append(child)


class DiffiosDiff(object):

    """Docstring for DiffiosDiff. """

    def __init__(self, baseline=None, comparison=None, ignore_file=None):
        """TODO: Docstring for __init__.

        Kwargs:
            baseline (TODO): TODO
            comparison (TODO): TODO
            ignore_file (TODO): TODO

        """
        # TODO: make it so DiffiosConf objects can be passed in also
        # TODO: confirm existence of files
        self.baseline = DiffiosConfig(baseline, ignore_file)
        self.comparison = DiffiosConfig(comparison, ignore_file)
        self.delimiter = r'{{(.+)}}'

    def _translate(self, with_vars, without_vars):
        """TODO: Docstring for _translate.
        Returns: TODO

        """
        var_blocks = []
        for block in with_vars:
            for line in block:
                if re.search(self.delimiter, line):
                    var_blocks.append(block)
        # result = []
        for vb in var_blocks:
            for wvb in without_vars:
                self._compare_blocks(vb, wvb)
        return None

    def _compare_blocks(self, wv_block, wov_block):
        """TODO: Docstring for _translated_block.

        Args:
            wv_block (TODO): TODO
            wov_block (TODO): TODO

        Returns: TODO

        """
        block = []
        delimiter = self.delimiter
        for wv_line in wv_block:
            match = re.search(delimiter, wv_line)
            if match:
                var = match.group()
                start = wv_line.index(delimiter[0])
                end = (wv_line.index(delimiter[-1]) + 2) - len(wv_line)
                if end == 0:
                    end = len(wv_line)
                for wov_line in wov_block:
                    before = wov_line[:start]
                    after = wov_line[end:]
                    translated_line = before + var + after
                    if wv_line == translated_line:
                        block.append(translated_line)
                    else:
                        block.append(wov_line)
        print(block)
        return None

    def _changes(self, dynamic, static):
        """TODO: Docstring for _changes.

        Args:
            dynamic (TODO): TODO
            static (TODO): TODO

        Returns: TODO

        """
        translated_static = self._translate(static)
        translated_dynamic = self._translate(dynamic)
        head = [line[0] for line in static]
        changes = []
        for dynamic_index, dynamic_block in enumerate(translated_dynamic):
            if len(dynamic_block) == 1 and dynamic_block not in translated_static:
                changes.append(dynamic[dynamic_index])
            elif len(dynamic_block) > 1:
                first_line = dynamic_block[0]
                if first_line in head:
                    static_block = translated_static[head.index(first_line)]
                    additional = [first_line]
                    for dynamic_block_index, line in enumerate(dynamic_block):
                        if line not in static_block:
                            additional.append(
                                dynamic[dynamic_index][dynamic_block_index])
                    if len(additional) > 1:
                        changes.append(additional)
                else:
                    changes.append(dynamic[dynamic_index])
        return sorted(changes)

    @staticmethod
    def _format_changes(data):
        """TODO: Docstring for _format_changes.

        Args:
            data (TODO): TODO

        Returns: TODO

        """
        return "\n\n".join("\n".join(lines) for lines in data)

    @property
    def additional(self):
        """TODO: Docstring for additional.

        Returns: TODO

        """
        return self._changes(self.comparison.recorded, self.baseline.recorded)

    @property
    def missing(self):
        """TODO: Docstring for missing.

        Returns: TODO

        """
        return self._changes(self.baseline.recorded, self.comparison.recorded)

    @property
    def pprint_additional(self):
        """TODO: Docstring for pprint_additional.

        Returns: TODO

        """
        return self._format_changes(self.additional)

    @property
    def pprint_missing(self):
        """TODO: Docstring for pprint_missing.

        Returns: TODO

        """
        return self._format_changes(self.missing)

    def diff(self):
        """TODO: Docstring for diff.

{'fraction_non_alphanumeric': 0.09327939626746438, 'fraction_numerical': 0.008192541727572493, 'mean_word_length': 1.7671902925406828, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 20, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '281560', 'n_tokens_mistral': 8164, 'n_tokens_neox': 7577, 'n_words': 2374}