<commit_before>class _AssociativeArray(object):

    def __init__(self, data={}):
        self._data = data.copy()

    def __getitem__(self, key):
        return self._data.get(key)

    def __setitem__(self, key, value):
        self._data[key] = value


class Diff(object):

    def diff(self, old_string, new_string):
        # Allow subclasses to massage the input prior to running
        old_string = self.preprocess(old_string)
        new_string = self.preprocess(new_string)

        old_string = self.tokenize(old_string)
        new_string = self.tokenize(new_string)

        new_len = len(new_string)
        old_len = len(old_string)
        max_edit_length = new_len + old_len
        best_path = _AssociativeArray({0: {'new_pos': -1, 'components': []}})

        # Seed edit_length = 0, i.e. the content starts with the same values
        old_pos = self._extract_common(best_path[0], new_string, old_string, 0)
        if best_path[0]['new_pos'] + 1 >= new_len and old_pos + 1 >= old_len:
            # Identity per the equality and tokenizer
            return [{'value': self.join(new_string), 'count': new_len}]

        for edit_length in range(1, max_edit_length + 1):
            # Check all permutations of a given edit length for acceptance
            for diagonal_path in range(-1 * edit_length, edit_length + 1, 2):
                add_path = best_path[diagonal_path - 1]
                remove_path = best_path[diagonal_path + 1]
                old_pos = (remove_path['new_pos'] if remove_path else 0) - diagonal_path
                if add_path:
                    # No one else is going to attempt to use this value, clear it
                    best_path[diagonal_path - 1] = None

                can_add = add_path and add_path['new_pos'] + 1 < new_len
                can_remove = remove_path and 0 <= old_pos < old_len
                if not (can_add or can_remove):
                    # If this path is a terminal then prune
                    best_path[diagonal_path] = None
                    continue

                # Select the diagonal that we want to branch from. We select the prior
                # path whose position in the new string is the farthest from the origin<commit_msg>Replacing _AssociativeArray with a defaultdict<commit_after>from collections import defaultdict


class Diff(object):

    def diff(self, old_string, new_string):
        # Allow subclasses to massage the input prior to running
        old_string = self.preprocess(old_string)
        new_string = self.preprocess(new_string)

        old_string = self.tokenize(old_string)
        new_string = self.tokenize(new_string)

        new_len = len(new_string)
        old_len = len(old_string)
        max_edit_length = new_len + old_len
        best_path = defaultdict(lambda: None)
        best_path[0] = {'new_pos': -1, 'components': []}

        # Seed edit_length = 0, i.e. the content starts with the same values
        old_pos = self._extract_common(best_path[0], new_string, old_string, 0)
        if best_path[0]['new_pos'] + 1 >= new_len and old_pos + 1 >= old_len:
            # Identity per the equality and tokenizer
            return [{'value': self.join(new_string), 'count': new_len}]

        for edit_length in range(1, max_edit_length + 1):
            # Check all permutations of a given edit length for acceptance
            for diagonal_path in range(-1 * edit_length, edit_length + 1, 2):
                add_path = best_path[diagonal_path - 1]
                remove_path = best_path[diagonal_path + 1]
                old_pos = (remove_path['new_pos'] if remove_path else 0) - diagonal_path
                if add_path:
                    # No one else is going to attempt to use this value, clear it
                    best_path[diagonal_path - 1] = None

                can_add = add_path and add_path['new_pos'] + 1 < new_len
                can_remove = remove_path and 0 <= old_pos < old_len
                if not (can_add or can_remove):
                    # If this path is a terminal then prune
                    best_path[diagonal_path] = None
                    continue

                # Select the diagonal that we want to branch from. We select the prior
                # path whose position in the new string is the farthest from the origin
{'fraction_non_alphanumeric': 0.07889865802869042, 'fraction_numerical': 0.008792225821378992, 'mean_word_length': 2.1554744525547447, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4461527', 'n_tokens_mistral': 1283, 'n_tokens_neox': 1192, 'n_words': 385}