<commit_before># -*- coding: utf-8  -*-
"""Diff module."""
#
# (C) Pywikibot team, 2014-2015
#
# Distributed under the terms of the MIT license.
#
from __future__ import absolute_import, unicode_literals

__version__ = '$Id$'


import difflib
import math
import sys

from collections import Sequence
if sys.version_info[0] > 2:
    from itertools import zip_longest
else:
    from itertools import izip_longest as zip_longest

try:
    from bs4 import BeautifulSoup
except ImportError as bserror:
    BeautifulSoup = False

import pywikibot
from pywikibot.tools import chars

from pywikibot.backports import format_range_unified  # introduced in 2.7.2
from pywikibot.tools import deprecated_args
from pywikibot.tools.formatter import color_format


class Hunk(object):

    """One change hunk between a and b.

    Note: parts of this code are taken from by difflib.get_grouped_opcodes().

    """

    APPR = 1
    NOT_APPR = -1
    PENDING = 0

    def __init__(self, a, b, grouped_opcode):
        """
        Constructor.

        @param a: sequence of lines
        @param b: sequence of lines
        @param grouped_opcode: list of 5-tuples describing how to turn a into b.
            it has the same format as returned by difflib.get_opcodes().

        """
        self.a = a
        self.b = b
        self.group = grouped_opcode
        self.header = u''
        self.colors = {
            '+': 'lightgreen',
            '-': 'lightred',
        }

        self.diff = list(self.create_diff())
        self.diff_plain_text = u''.join(self.diff)
        self.diff_text = u''.join(self.format_diff())

        first, last = self.group[0], self.group[-1]
        self.a_rng = (first[1], last[2])
        self.b_rng = (first[3], last[4])

        self.header = self.get_header()
        self.diff_plain_text = u'%s\n%s' % (self.header, self.diff_plain_text)
        self.diff_text = u'%s' % self.diff_text

        self.reviewed = self.PENDING

    def get_header(self):
        """Provide header of unified diff."""
        return self.get_header_text(self.a_rng, self.b_rng) + '\n'

    @staticmethod
    def get_header_text(a_rng, b_rng, affix='@@'):
        """Provide header for any ranges."""
        a_rng = format_range_unified(*a_rng)
        b_rng = format_range_unified(*b_rng)
        return '{0} -{1} +{2} {0}'.format(affix, a_rng, b_rng)

    def create_diff(self):
        """Generator of diff text for this hunk, without formatting."""
        # make sure each line ends with '\n' to prevent
        # behaviour like http://bugs.python.org/issue2142
        def check_line(l):
            if not l.endswith('\n'):
                return l + '\n'
            return l

        for tag, i1, i2, j1, j2 in self.group:
            # equal/delete/insert add additional space after the sign as it's
            # what difflib.ndiff does do too.
            if tag == 'equal':
                for line in self.a[i1:i2]:
                    yield '  ' + check_line(line)
            if tag in ('delete'):
                for line in self.a[i1:i2]:
                    yield '- ' + check_line(line)
            if tag in ('insert'):
                for line in self.b[j1:j2]:
                    yield '+ ' + check_line(line)
            if tag in ('replace'):
                for line in difflib.ndiff(self.a[i1:i2], self.b[j1:j2]):
                    yield check_line(line)

    def format_diff(self):
        """Color diff lines."""
        diff = iter(self.diff)

        l1, l2 = '', next(diff)
        for line in diff:
            l1, l2 = l2, line
            # do not show lines starting with '?'.
            if l1.startswith('?'):
                continue
            if l2.startswith('?'):
                yield self.color_line(l1, l2)
            else:
                yield self.color_line(l1)

        # handle last line
        if not l2.startswith('?'):
            yield self.color_line(l2)

    def color_line(self, line, line_ref=None):
        """Color line characters.

        If line_ref is None, the whole line is colored.
        If line_ref[i] is not blank, line[i] is colored.
        Color depends if line starts with +/-.

        line: string
        line_ref: string.

        """
        color = line[0]

        if line_ref is None:
            if color in self.colors:
                colored_line = color_format('{color}{0}{default}',
                                            line, color=self.colors[color])
                return colored_line
            else:<commit_msg>diff: do not highlight entire line when only text is removed<commit_after># -*- coding: utf-8  -*-
"""Diff module."""
#
# (C) Pywikibot team, 2014-2015
#
# Distributed under the terms of the MIT license.
#
from __future__ import absolute_import, unicode_literals

__version__ = '$Id$'


import difflib
import math
import sys

from collections import Sequence
if sys.version_info[0] > 2:
    from itertools import zip_longest
else:
    from itertools import izip_longest as zip_longest

try:
    from bs4 import BeautifulSoup
except ImportError as bserror:
    BeautifulSoup = False

import pywikibot
from pywikibot.tools import chars

from pywikibot.backports import format_range_unified  # introduced in 2.7.2
from pywikibot.tools import deprecated_args
from pywikibot.tools.formatter import color_format


class Hunk(object):

    """One change hunk between a and b.

    Note: parts of this code are taken from by difflib.get_grouped_opcodes().

    """

    APPR = 1
    NOT_APPR = -1
    PENDING = 0

    def __init__(self, a, b, grouped_opcode):
        """
        Constructor.

        @param a: sequence of lines
        @param b: sequence of lines
        @param grouped_opcode: list of 5-tuples describing how to turn a into b.
            it has the same format as returned by difflib.get_opcodes().

        """
        self.a = a
        self.b = b
        self.group = grouped_opcode
        self.header = u''
        self.colors = {
            '+': 'lightgreen',
            '-': 'lightred',
        }

        self.diff = list(self.create_diff())
        self.diff_plain_text = u''.join(self.diff)
        self.diff_text = u''.join(self.format_diff())

        first, last = self.group[0], self.group[-1]
        self.a_rng = (first[1], last[2])
        self.b_rng = (first[3], last[4])

        self.header = self.get_header()
        self.diff_plain_text = u'%s\n%s' % (self.header, self.diff_plain_text)
        self.diff_text = u'%s' % self.diff_text

        self.reviewed = self.PENDING

    def get_header(self):
        """Provide header of unified diff."""
        return self.get_header_text(self.a_rng, self.b_rng) + '\n'

    @staticmethod
    def get_header_text(a_rng, b_rng, affix='@@'):
        """Provide header for any ranges."""
        a_rng = format_range_unified(*a_rng)
        b_rng = format_range_unified(*b_rng)
        return '{0} -{1} +{2} {0}'.format(affix, a_rng, b_rng)

    def create_diff(self):
        """Generator of diff text for this hunk, without formatting."""
        # make sure each line ends with '\n' to prevent
        # behaviour like http://bugs.python.org/issue2142
        def check_line(l):
            if not l.endswith('\n'):
                return l + '\n'
            return l

        for tag, i1, i2, j1, j2 in self.group:
            # equal/delete/insert add additional space after the sign as it's
            # what difflib.ndiff does do too.
            if tag == 'equal':
                for line in self.a[i1:i2]:
                    yield '  ' + check_line(line)
            if tag in ('delete'):
                for line in self.a[i1:i2]:
                    yield '- ' + check_line(line)
            if tag in ('insert'):
                for line in self.b[j1:j2]:
                    yield '+ ' + check_line(line)
            if tag in ('replace'):
                for line in difflib.ndiff(self.a[i1:i2], self.b[j1:j2]):
                    yield check_line(line)

    def format_diff(self):
        """Color diff lines."""
        diff = iter(self.diff)

        fmt, line1, line2 = '', '', next(diff)
        for line in diff:
            fmt, line1, line2 = line1, line2, line
            # do not show lines starting with '?'.
            if line1.startswith('?'):
                continue
            if line2.startswith('?'):
                yield self.color_line(line1, line2)
                continue
            if line1.startswith('-'):
                # Color whole line to be removed.
                yield self.color_line(line1)
            elif line1.startswith('+'):
                # Reuse last available fmt as diff line, if possible,
                # or color whole line to be added.
                fmt = fmt if fmt.startswith('?') else ''
                fmt = fmt[:min(len(fmt), len(line1))]
                fmt = fmt if fmt else None
                yield self.color_line(line1, fmt)

        # handle last line
        # If line line2 is removed, color the whole line.
        # If line line2 is added, check if line1 is a '?-type' line, to prevent
        # the entire line line2 to be colored (see T130572).
        # The case where line2 start with '?' has been covered already.
        if line2.startswith('-'):
            # Color whole line to be removed.
            yield self.color_line(line2)
        elif line2.startswith('+'):
            # Reuse last available line1 as diff line, if possible,
            # or color whole line to be added.
            fmt = line1 if line1.startswith('?') else ''
            fmt = fmt[:min(len(fmt), len(line2))]
            fmt = fmt if fmt else None
            yield self.color_line(line2, fmt)

    def color_line(self, line, line_ref=None):
        """Color line characters.

        If line_ref is None, the whole line is colored.
        If line_ref[i] is not blank, line[i] is colored.
        Color depends if line starts with +/-.

        line: string
        line_ref: string.

        """
        color = line[0]

        if line_ref is None:
            if color in self.colors:
                colored_line = color_format('{color}{0}{default}',
                                            line, color=self.colors[color])
                return colored_line
            else:
{'fraction_non_alphanumeric': 0.10782405605293889, 'fraction_numerical': 0.014013234721681589, 'mean_word_length': 1.8738814317673378, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1308625', 'n_tokens_mistral': 3247, 'n_tokens_neox': 3105, 'n_words': 939}