<commit_before>from __future__ import unicode_literals

import re


CHUNK_RANGE_RE = re.compile(
    r'^@@ -(?P<orig_start>\d+)(,(?P<orig_len>\d+))? '
    r'\+(?P<new_start>\d+)(,(?P<new_len>\d+))? @@',
    re.M)


def filter_interdiff_opcodes(opcodes, filediff_data, interfilediff_data):
    """Filters the opcodes for an interdiff to remove unnecessary lines.

    An interdiff may contain lines of code that have changed as the result of
    updates to the tree between the time that the first and second diff were
    created. This leads to some annoyances when reviewing.

    This function will filter the opcodes to remove as much of this as
    possible. It will only output non-"equal" opcodes if it falls into the
    ranges of lines dictated in the uploaded diff files.
    """
    def _find_range_info(diff):
        lines = diff.splitlines()
        process_changes = False
        ranges = []

        chunk_start = None
        chunk_len = 0
        lines_of_context = 0

        # Look through the chunks of the diff, trying to find the amount
        # of context shown at the beginning of each chunk. Though this
        # will usually be 3 lines, it may be fewer or more, depending
        # on file length and diff generation settings.
        for line in lines:
            if process_changes:
                if line.startswith(('-', '+')):
                    # We've found the first change in the chunk. We now
                    # know how many lines of context we have.
                    #
                    # We reduce the indexes by 1 because the chunk ranges
                    # in diffs start at 1, and we want a 0-based index.
                    start = chunk_start - 1 + lines_of_context
                    ranges.append((start, start + chunk_len))
                    process_changes = False
                    continue
                else:
                    lines_of_context += 1

            # This was not a change within a chunk, or we weren't processing,
            # so check to see if this is a chunk header instead.
            m = CHUNK_RANGE_RE.match(line)

            if m:
                # It is a chunk header. Reset the state for the next range,
                # and pull the line number and length from the header.
                chunk_start = int(m.group('new_start'))
                chunk_len = int(m.group('new_len') or '1')
                process_changes = True<commit_msg>Fix one more unicode issue in diffs.<commit_after>from __future__ import unicode_literals

import re


CHUNK_RANGE_RE = re.compile(
    r'^@@ -(?P<orig_start>\d+)(,(?P<orig_len>\d+))? '
    r'\+(?P<new_start>\d+)(,(?P<new_len>\d+))? @@',
    re.M)


def filter_interdiff_opcodes(opcodes, filediff_data, interfilediff_data):
    """Filters the opcodes for an interdiff to remove unnecessary lines.

    An interdiff may contain lines of code that have changed as the result of
    updates to the tree between the time that the first and second diff were
    created. This leads to some annoyances when reviewing.

    This function will filter the opcodes to remove as much of this as
    possible. It will only output non-"equal" opcodes if it falls into the
    ranges of lines dictated in the uploaded diff files.
    """
    def _find_range_info(diff):
        lines = diff.splitlines()
        process_changes = False
        ranges = []

        chunk_start = None
        chunk_len = 0
        lines_of_context = 0

        # Look through the chunks of the diff, trying to find the amount
        # of context shown at the beginning of each chunk. Though this
        # will usually be 3 lines, it may be fewer or more, depending
        # on file length and diff generation settings.
        for line in lines:
            if process_changes:
                if line.startswith((b'-', b'+')):
                    # We've found the first change in the chunk. We now
                    # know how many lines of context we have.
                    #
                    # We reduce the indexes by 1 because the chunk ranges
                    # in diffs start at 1, and we want a 0-based index.
                    start = chunk_start - 1 + lines_of_context
                    ranges.append((start, start + chunk_len))
                    process_changes = False
                    continue
                else:
                    lines_of_context += 1

            # This was not a change within a chunk, or we weren't processing,
            # so check to see if this is a chunk header instead.
            m = CHUNK_RANGE_RE.match(line)

            if m:
                # It is a chunk header. Reset the state for the next range,
                # and pull the line number and length from the header.
                chunk_start = int(m.group('new_start'))
                chunk_len = int(m.group('new_len') or '1')
                process_changes = True
{'fraction_non_alphanumeric': 0.07100712105798576, 'fraction_numerical': 0.003662258392675483, 'mean_word_length': 1.8714953271028036, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7540847', 'n_tokens_mistral': 1342, 'n_tokens_neox': 1262, 'n_words': 542}