<commit_before># -*- coding: utf-8 -*-

"""
checkfiles.py: comparison mechanism for text files

Source repository: http://github.com/tdda/tdda

License: MIT

Copyright (c) Stochastic Solutions Limited 2016-2018
"""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
# from __future__ import unicode_literals

import os
import re
import sys
import tempfile
from collections import namedtuple

from tdda.referencetest.basecomparison import (BaseComparison, Diffs,
                                               Reconstruction, copycmd)


BinaryInfo = namedtuple('BinaryInfo',
                        ('byteoffset', 'actualLen', 'expectedLen'))


class FilesComparison(BaseComparison):

    def check_strings(self, actual, expected,
                      actual_path=None, expected_path=None,
                      lstrip=False, rstrip=False,
                      ignore_substrings=None, ignore_patterns=None,
                      remove_lines=None,
                      preprocess=None, max_permutation_cases=0, msgs=None):
        """
        Compare two lists of strings (actual and expected), one-by-one.

            *actual*
                                is a list of strings.
            *expected*
                                is the expected list of strings.
            *actual_path*
                                is the path of the file where the actual
                                string originated (used for reporting errors).
            *expected_path*
                                is the path from which expected was read; it is
                                used to suggest a diff command to use on files
                                for strings failing the check.
            *lstrip*
                                if set to true, both strings are left-stripped
                                before the comparison is carried out.
                                Note: the stripping on a per-line basis.
            *rstrip*
                                if set to true, both strings are right-stripped
                                before the comparison is carried out.
                                Note: the stripping on a per-line basis.
            *ignore_substrings*
                                is an optional list of substrings; lines
                                containing any of these substrings will be
                                ignored in the comparison.
            *ignore_patterns*
                                is an optional list of regular expressions;
                                lines will be considered to be the same if
                                they only differ in substrings that match one
                                of these regular expressions. The expressions
                                must not contain parenthesised groups, and
                                should only include explicit anchors if they
                                need refer to the whole line.
            *remove_lines*
                                is an optional list of substrings; lines
                                containing any of these substrings will be
                                completely removed before carrying out the
                                comparison. This is the means by which you
                                would exclude 'optional' content.
            *preprocess*
                                is an optional function that takes a list of
                                strings and preprocesses it in some way; this
                                function will be applied to both the actual
                                and expected.
            *max_permutation_cases*
                                is an optional number specifying the maximum
                                number of permutations allowed; if the actual
                                and expected lists differ only in that their
                                lines are permutations of each other, and
                                the number of such permutations does not
                                exceed this limit, then the two are considered
                                to be identical.
            *msgs*
                                is an optional Diffs object, where information
                                about differences will be appended; if not
                                specified, a new object will be created and
                                returned.

        Returns a tuple (failures, msgs), where failures is 1 if the lists
        differ and 0 if they are the same. The returned msgs is a list
        containing information about how they differed.
        """

        if msgs is None:
            msgs = Diffs()
        first_error = None
        first_error_line = None
        failure_cases = []
        reconstruction = None

        if preprocess:
            expected = preprocess(expected)
            actual = preprocess(actual)

        if actual and len(actual[-1]) == 0:
            actual = actual[:-1]
        if expected and len(expected[-1]) == 0:
            expected = expected[:-1]

        original_actual = actual
        original_expected = expected
        actual_map = {i: i for i in range(len(original_actual))}
        expected_map = {i: i for i in range(len(original_expected))}
        actual_removals = set()
        expected_removals = set()

        if remove_lines:
            actual_removals = set([i for i, a in enumerate(original_actual)
                                   if any(r in a for r in remove_lines)])
            expected_removals = set([i for i, a in enumerate(original_expected)
                                     if any(r in a for r in remove_lines)])
            actual = [a for i, a in enumerate(original_actual)
                      if i not in actual_removals]
            expected = [a for i, a in enumerate(original_expected)
                        if i not in expected_removals]

            # now build mappings from the after-removal line-numbers back to
            # the original line numbers, for both actual and expected
            iactual = 0
            for i in range(len(original_actual)):
                if i not in actual_removals:
                    actual_map[iactual] = i
                    iactual += 1
            iexpected = 0
            for i in range(len(original_expected)):
                if i not in expected_removals:
                    actual_map[iexpected] = i
                    iexpected += 1

        if len(actual) == len(expected):
            actual_ignored = set()
            expected_ignored = set()
            normalize = self.normalize_function(lstrip, rstrip)
            diffs = [i for i in range(len(actual))
                     if normalize(actual[i]) != normalize(expected[i])]
            ndiffs = len(diffs)

            if ndiffs > 0:
                ignore_substrings = ignore_substrings or []
                ignore_patterns = ignore_patterns or []
                anchored_patterns = [('' if p.startswith('^') else '^(.*)')
                                      + ('(%s)' % p)
                                      + ('' if p.endswith('$') else '(.*)$')
                                     for p in ignore_patterns]
                cPatterns = [re.compile(p) for p in anchored_patterns]
                if any(cp.groups > 3 for cp in cPatterns):
                    raise Exception('Invalid patterns: %s' % ignore_patterns)
                for i in diffs:
                    # 'i' is a line-number in the after-removals datasets
                    for substr in ignore_substrings:
                        if substr in actual[i] or substr in expected[i]:
                            # ignored a line, so there is one fewer to report
                            #
                            # TODO: This ignores a line even if it only
                            #       matches on ONE side and NOT on the other!
                            #       That doesn't seem right at all, but to
                            #       change 'or' to 'and' would change the
                            #       semantics in a way that will break some
                            #       tests in other software that is depending
                            #       on this behaviour. We should deprecate
                            #       this, and publish a timetable for changing
                            #       it.
                            ndiffs -= 1
                            actual_ignored.add(actual_map[i])
                            expected_ignored.add(expected_map[i])
                            break
                    else:
                        # not an ignorable substring line, so try patterns
                        if self.check_patterns(cPatterns, actual, expected, i):
                            # ignored a line, so there is one fewer to report
                            ndiffs -= 1
                            actual_ignored.add(actual_map[i])
                            expected_ignored.add(expected_map[i])
                        else:
                            # difference can't be ignored
                            if first_error_line is None:
                                first_error_line = i + 1<commit_msg>comment<commit_after># -*- coding: utf-8 -*-

"""
checkfiles.py: comparison mechanism for text files

Source repository: http://github.com/tdda/tdda

License: MIT

Copyright (c) Stochastic Solutions Limited 2016-2018
"""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
# from __future__ import unicode_literals

import os
import re
import sys
import tempfile
from collections import namedtuple

from tdda.referencetest.basecomparison import (BaseComparison, Diffs,
                                               Reconstruction, copycmd)


BinaryInfo = namedtuple('BinaryInfo',
                        ('byteoffset', 'actualLen', 'expectedLen'))


class FilesComparison(BaseComparison):

    def check_strings(self, actual, expected,
                      actual_path=None, expected_path=None,
                      lstrip=False, rstrip=False,
                      ignore_substrings=None, ignore_patterns=None,
                      remove_lines=None,
                      preprocess=None, max_permutation_cases=0, msgs=None):
        """
        Compare two lists of strings (actual and expected), one-by-one.

            *actual*
                                is a list of strings.
            *expected*
                                is the expected list of strings.
            *actual_path*
                                is the path of the file where the actual
                                string originated (used for reporting errors).
            *expected_path*
                                is the path from which expected was read; it is
                                used to suggest a diff command to use on files
                                for strings failing the check.
            *lstrip*
                                if set to true, both strings are left-stripped
                                before the comparison is carried out.
                                Note: the stripping on a per-line basis.
            *rstrip*
                                if set to true, both strings are right-stripped
                                before the comparison is carried out.
                                Note: the stripping on a per-line basis.
            *ignore_substrings*
                                is an optional list of substrings; lines
                                containing any of these substrings will be
                                ignored in the comparison.
            *ignore_patterns*
                                is an optional list of regular expressions;
                                lines will be considered to be the same if
                                they only differ in substrings that match one
                                of these regular expressions. The expressions
                                must not contain parenthesised groups, and
                                should only include explicit anchors if they
                                need refer to the whole line.
            *remove_lines*
                                is an optional list of substrings; lines
                                containing any of these substrings will be
                                completely removed before carrying out the
                                comparison. This is the means by which you
                                would exclude 'optional' content.
            *preprocess*
                                is an optional function that takes a list of
                                strings and preprocesses it in some way; this
                                function will be applied to both the actual
                                and expected.
            *max_permutation_cases*
                                is an optional number specifying the maximum
                                number of permutations allowed; if the actual
                                and expected lists differ only in that their
                                lines are permutations of each other, and
                                the number of such permutations does not
                                exceed this limit, then the two are considered
                                to be identical.
            *msgs*
                                is an optional Diffs object, where information
                                about differences will be appended; if not
                                specified, a new object will be created and
                                returned.

        Returns a tuple (failures, msgs), where failures is 1 if the lists
        differ and 0 if they are the same. The returned msgs is a list
        containing information about how they differed.
        """

        if msgs is None:
            msgs = Diffs()
        first_error = None
        first_error_line = None
        failure_cases = []
        reconstruction = None

        if preprocess:
            expected = preprocess(expected)
            actual = preprocess(actual)

        if actual and len(actual[-1]) == 0:
            actual = actual[:-1]
        if expected and len(expected[-1]) == 0:
            expected = expected[:-1]

        original_actual = actual
        original_expected = expected
        actual_map = {i: i for i in range(len(original_actual))}
        expected_map = {i: i for i in range(len(original_expected))}
        actual_removals = set()
        expected_removals = set()

        if remove_lines:
            actual_removals = set([i for i, a in enumerate(original_actual)
                                   if any(r in a for r in remove_lines)])
            expected_removals = set([i for i, a in enumerate(original_expected)
                                     if any(r in a for r in remove_lines)])
            actual = [a for i, a in enumerate(original_actual)
                      if i not in actual_removals]
            expected = [a for i, a in enumerate(original_expected)
                        if i not in expected_removals]

            # now build mappings from the after-removal line-numbers back to
            # the original line numbers, for both actual and expected
            iactual = 0
            for i in range(len(original_actual)):
                if i not in actual_removals:
                    actual_map[iactual] = i
                    iactual += 1
            iexpected = 0
            for i in range(len(original_expected)):
                if i not in expected_removals:
                    actual_map[iexpected] = i
                    iexpected += 1

        if len(actual) == len(expected):
            actual_ignored = set()
            expected_ignored = set()
            normalize = self.normalize_function(lstrip, rstrip)
            diffs = [i for i in range(len(actual))
                     if normalize(actual[i]) != normalize(expected[i])]
            ndiffs = len(diffs)

            if ndiffs > 0:
                ignore_substrings = ignore_substrings or []
                ignore_patterns = ignore_patterns or []
                anchored_patterns = [('' if p.startswith('^') else '^(.*)')
                                      + ('(%s)' % p)
                                      + ('' if p.endswith('$') else '(.*)$')
                                     for p in ignore_patterns]
                cPatterns = [re.compile(p) for p in anchored_patterns]
                if any(cp.groups > 3 for cp in cPatterns):
                    raise Exception('Invalid patterns: %s' % ignore_patterns)
                for i in diffs:
                    # 'i' is a line-number in the after-removals datasets
                    for substr in ignore_substrings:
                        if substr in actual[i] or substr in expected[i]:
                            # ignored a line, so there is one fewer to report
                            #
                            # Note that this ignores a line even if it only
                            # matches on ONE side and NOT on the other. This
                            # is deliberate, to cover the case where the
                            # expected reference file contains concrete
                            # representations of things that need to be
                            # ignored (like usernames, machine names etc),
                            # where you can't express that more generally
                            # as any kind of pattern.
                            #
                            ndiffs -= 1
                            actual_ignored.add(actual_map[i])
                            expected_ignored.add(expected_map[i])
                            break
                    else:
                        # not an ignorable substring line, so try patterns
                        if self.check_patterns(cPatterns, actual, expected, i):
                            # ignored a line, so there is one fewer to report
                            ndiffs -= 1
                            actual_ignored.add(actual_map[i])
                            expected_ignored.add(expected_map[i])
                        else:
                            # difference can't be ignored
                            if first_error_line is None:
                                first_error_line = i + 1
{'fraction_non_alphanumeric': 0.05130258381379458, 'fraction_numerical': 0.0028827674567584883, 'mean_word_length': 1.1509932253990125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 66, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '647439', 'n_tokens_mistral': 4219, 'n_tokens_neox': 3939, 'n_words': 1553}