<commit_before>import difflib
from regparser.tree import struct

INSERT = 'insert'
DELETE = 'delete'
REPLACE = 'replace'
EQUAL = 'equal'


def hash_nodes(reg_tree):
    """ Create a hash map to the nodes of a regulation tree.  """
    tree_hash = {}

    def per_node(node):
        tree_hash[node.label_id()] = node

    struct.walk(reg_tree, per_node)
    return tree_hash


def convert_insert(ins_op, new_text):
    """ The insert operation returned by difflib assumes we have access to both
    texts. We re-write the op, so that we don't make the same assumption. """
    return (INSERT, ins_op[1], ' '.join(new_text[ins_op[3]:ins_op[4]]))


def convert_opcode(op, new_text):
    """ We want to express changes as inserts and deletes only. """
    code = op[0]
    if code == INSERT:
        return convert_insert(op, new_text)
    elif code == DELETE:
        #Deletes have an extra set of co-ordinates which
        #we don't need.
        return (DELETE, op[1], op[2])
    elif code == REPLACE:
        del_op = (DELETE, op[1], op[2])
        add_op = convert_insert(
            (INSERT, op[1], op[1], op[3], op[4]), new_text)
        return [del_op, add_op]


def get_opcodes(old_text, new_text):
    """ Get the operation codes that convert old_text into
    new_text. """

    old_word_list =  old_text.split(' ')
    new_word_list =  new_text.split(' ')

    seqm = difflib.SequenceMatcher(
        lambda x: x in " \t\n",
        old_word_list,<commit_msg>PEP8 changes<commit_after>import difflib
from regparser.tree import struct

INSERT = 'insert'
DELETE = 'delete'
REPLACE = 'replace'
EQUAL = 'equal'


def hash_nodes(reg_tree):
    """ Create a hash map to the nodes of a regulation tree.  """
    tree_hash = {}

    def per_node(node):
        tree_hash[node.label_id()] = node

    struct.walk(reg_tree, per_node)
    return tree_hash


def convert_insert(ins_op, new_text):
    """ The insert operation returned by difflib assumes we have access to both
    texts. We re-write the op, so that we don't make the same assumption. """
    return (INSERT, ins_op[1], ' '.join(new_text[ins_op[3]:ins_op[4]]))


def convert_opcode(op, new_text):
    """ We want to express changes as inserts and deletes only. """
    code = op[0]
    if code == INSERT:
        return convert_insert(op, new_text)
    elif code == DELETE:
        #Deletes have an extra set of co-ordinates which
        #we don't need.
        return (DELETE, op[1], op[2])
    elif code == REPLACE:
        del_op = (DELETE, op[1], op[2])
        add_op = convert_insert(
            (INSERT, op[1], op[1], op[3], op[4]), new_text)
        return [del_op, add_op]


def get_opcodes(old_text, new_text):
    """ Get the operation codes that convert old_text into
    new_text. """

    old_word_list = old_text.split(' ')
    new_word_list = new_text.split(' ')

    seqm = difflib.SequenceMatcher(
        lambda x: x in " \t\n",
        old_word_list,
{'fraction_non_alphanumeric': 0.1205311542390194, 'fraction_numerical': 0.008512087163772558, 'mean_word_length': 2.9383378016085793, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4197066', 'n_tokens_mistral': 1006, 'n_tokens_neox': 983, 'n_words': 298}