<commit_before>import types
from collections import deque


class DiffApplier(object):
    """ Diffs between two versions of a regulation are represented in our
    particular JSON format. This class applies that diff to the older version
    of the regulation, generating HTML that clearly shows the changes between
    old and new. """

    INSERT = u'insert'
    DELETE = u'delete'

    def __init__(self, diff_json):
        self.diff = diff_json

    def deconstruct_text(self, original):
        self.oq = [deque([c]) for c in original]

    def insert_text(self, pos, new_text):
        if pos == len(self.oq):
            self.oq[pos-1].extend(['<ins>', new_text, '</ins>'])
        else:
            self.oq[pos].extend(['<ins>', new_text, '</ins>'])

    def delete_text(self, start, end):
        self.oq[start].appendleft('<del>')
        self.oq[end-1].append('</del>')

    def get_text(self):
        return ''.join([''.join(d) for d in self.oq])

    def apply_diff(self, original, label):
        if label in self.diff:
            if 'text' in self.diff[label]:
                text_diffs = self.diff[label]['text']
                self.deconstruct_text(original)

                for d in text_diffs:
                    if d[0] == self.INSERT:
                        _, pos, new_text = d
                        self.insert_text(pos, new_text + ' ')
                    if d[0] == self.DELETE:
                        _, s, e = d
                        self.delete_text(s, e)<commit_msg>Add diffs for deleted paragraphs<commit_after>import types
from collections import deque


class DiffApplier(object):
    """ Diffs between two versions of a regulation are represented in our
    particular JSON format. This class applies that diff to the older version
    of the regulation, generating HTML that clearly shows the changes between
    old and new. """

    INSERT = u'insert'
    DELETE = u'delete'
    DELETED_OP = 'deleted'

    def __init__(self, diff_json):
        self.diff = diff_json

    def deconstruct_text(self, original):
        self.oq = [deque([c]) for c in original]

    def insert_text(self, pos, new_text):
        if pos == len(self.oq):
            self.oq[pos-1].extend(['<ins>', new_text, '</ins>'])
        else:
            self.oq[pos].extend(['<ins>', new_text, '</ins>'])

    def delete_text(self, start, end):
        self.oq[start].appendleft('<del>')
        self.oq[end-1].append('</del>')

    def get_text(self):
        return ''.join([''.join(d) for d in self.oq])

    def delete_all(self, text):
        """ Mark all the text passed in as deleted. """
        return '<del>' + text + '</del>'

    def apply_diff(self, original, label):
        if label in self.diff:
            if self.diff[label]['op'] == self.DELETED_OP:
                return self.delete_all(original)
            if 'text' in self.diff[label]:
                text_diffs = self.diff[label]['text']
                self.deconstruct_text(original)

                for d in text_diffs:
                    if d[0] == self.INSERT:
                        _, pos, new_text = d
                        self.insert_text(pos, new_text + ' ')
                    if d[0] == self.DELETE:
                        _, s, e = d
                        self.delete_text(s, e)
{'fraction_non_alphanumeric': 0.1251136019388064, 'fraction_numerical': 0.0024235080278703423, 'mean_word_length': 1.9561324977618622, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3135072', 'n_tokens_mistral': 999, 'n_tokens_neox': 959, 'n_words': 277}