<commit_before># -*- coding: utf-8 -*-

import codecs
import re
import difflib
import sys
import os

import duralex.alinea_parser as parser
import diff

from duralex.AbstractVisitor import AbstractVisitor

import duralex.tree as tree

class AddDiffVisitor(AbstractVisitor):
    REGEXP = {
        tree.TYPE_HEADER1_REFERENCE     : re.compile(r'([IVXCLDM]+\. - (?:(?:.|\n)(?![IVXCLDM]+\. - ))*)', re.UNICODE),
        tree.TYPE_HEADER2_REFERENCE     : re.compile(r'(\d+\. (?:(?:.|\n)(?!\d+\. ))*)', re.UNICODE),
        tree.TYPE_HEADER3_REFERENCE     : re.compile(r'([a-z]+\) (?:(?:.|\n)(?![a-z]+\) ))*)', re.UNICODE),
        tree.TYPE_ALINEA_REFERENCE      : re.compile(r'^(.+)$', re.UNICODE | re.MULTILINE),
        tree.TYPE_SENTENCE_REFERENCE    : re.compile(r'([A-ZÀÀÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏ].*?\.)', re.UNICODE),
        tree.TYPE_WORD_REFERENCE        : re.compile(r'(\b\w.*?\b)', re.UNICODE)
    }

    def __init__(self):
        self.filename = ''
        self.content = {}
        self.begin = 0
        self.end = -1
        super(AddDiffVisitor, self).__init__()

    def visit_alinea_reference_node(self, node, post):
        if post:
            return

        match = re.finditer(AddDiffVisitor.REGEXP['alinea-reference'], self.content[self.filename][self.begin:self.end])
        match = list(match)[node['order'] - 1 if node['order'] > 0 else node['order']]
        self.begin = match.start()
        self.end = match.start() + len(match.group(1))

    def visit_sentence_reference_node(self, node, post):
        if post:
            return

        match = re.finditer(AddDiffVisitor.REGEXP['sentence-reference'], self.content[self.filename][self.begin:self.end])
        match = list(match)[node['order'] - 1 if node['order'] > 0 else node['order']]
        self.begin = match.start()
        self.end = match.start() + len(match.group(1))

    def visit_header1_reference_node(self, node, post):
        if post:
            return

        match = re.finditer(AddDiffVisitor.REGEXP['header1-reference'], self.content[self.filename][self.begin:self.end])
        match = list(match)[node['order'] - 1 if node['order'] > 0 else node['order']]
        self.begin = match.start()
        self.end = match.start() + len(match.group(1))

    def visit_header2_reference_node(self, node, post):
        if post:
            return

        match = re.finditer(AddDiffVisitor.REGEXP['header2-reference'], self.content[self.filename][self.begin:self.end])
        match = list(match)[node['order'] - 1 if node['order'] > 0 else node['order']]
        self.begin = match.start()
        self.end = match.start() + len(match.group(1))

    def visit_header3_reference_node(self, node, post):
        if post:
            return

        match = re.finditer(AddDiffVisitor.REGEXP['header3-reference'], self.content[self.filename][self.begin:self.end])
        match = list(match)[node['order'] - 1 if node['order'] > 0 else node['order']]
        self.begin = match.start()
        self.end = match.start() + len(match.group(1))

    def visit_words_reference_node(self, node, post):
        if post:
            return
<commit_msg>Use tree.TYPE_* constant instead of string literals.<commit_after># -*- coding: utf-8 -*-

import codecs
import re
import difflib
import sys
import os

import duralex.alinea_parser as parser
import diff

from duralex.AbstractVisitor import AbstractVisitor

import duralex.tree as tree

class AddDiffVisitor(AbstractVisitor):
    REGEXP = {
        tree.TYPE_HEADER1_REFERENCE     : re.compile(r'([IVXCLDM]+\. - (?:(?:.|\n)(?![IVXCLDM]+\. - ))*)', re.UNICODE),
        tree.TYPE_HEADER2_REFERENCE     : re.compile(r'(\d+\. (?:(?:.|\n)(?!\d+\. ))*)', re.UNICODE),
        tree.TYPE_HEADER3_REFERENCE     : re.compile(r'([a-z]+\) (?:(?:.|\n)(?![a-z]+\) ))*)', re.UNICODE),
        tree.TYPE_ALINEA_REFERENCE      : re.compile(r'^(.+)$', re.UNICODE | re.MULTILINE),
        tree.TYPE_SENTENCE_REFERENCE    : re.compile(r'([A-ZÀÀÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏ].*?\.)', re.UNICODE),
        tree.TYPE_WORD_REFERENCE        : re.compile(r'(\b\w.*?\b)', re.UNICODE)
    }

    def __init__(self):
        self.filename = ''
        self.content = {}
        self.begin = 0
        self.end = -1
        super(AddDiffVisitor, self).__init__()

    def visit_alinea_reference_node(self, node, post):
        if post:
            return

        match = re.finditer(AddDiffVisitor.REGEXP[tree.TYPE_ALINEA_REFERENCE], self.content[self.filename][self.begin:self.end])
        match = list(match)[node['order'] - 1 if node['order'] > 0 else node['order']]
        self.begin = match.start()
        self.end = match.start() + len(match.group(1))

    def visit_sentence_reference_node(self, node, post):
        if post:
            return

        match = re.finditer(AddDiffVisitor.REGEXP[tree.TYPE_SENTENCE_REFERENCE], self.content[self.filename][self.begin:self.end])
        match = list(match)[node['order'] - 1 if node['order'] > 0 else node['order']]
        self.begin = match.start()
        self.end = match.start() + len(match.group(1))

    def visit_header1_reference_node(self, node, post):
        if post:
            return

        match = re.finditer(AddDiffVisitor.REGEXP[tree.TYPE_HEADER1_REFERENCE], self.content[self.filename][self.begin:self.end])
        match = list(match)[node['order'] - 1 if node['order'] > 0 else node['order']]
        self.begin = match.start()
        self.end = match.start() + len(match.group(1))

    def visit_header2_reference_node(self, node, post):
        if post:
            return

        match = re.finditer(AddDiffVisitor.REGEXP[tree.TYPE_HEADER2_REFERENCE], self.content[self.filename][self.begin:self.end])
        match = list(match)[node['order'] - 1 if node['order'] > 0 else node['order']]
        self.begin = match.start()
        self.end = match.start() + len(match.group(1))

    def visit_header3_reference_node(self, node, post):
        if post:
            return

        match = re.finditer(AddDiffVisitor.REGEXP[tree.TYPE_HEADER3_REFERENCE], self.content[self.filename][self.begin:self.end])
        match = list(match)[node['order'] - 1 if node['order'] > 0 else node['order']]
        self.begin = match.start()
        self.end = match.start() + len(match.group(1))

    def visit_words_reference_node(self, node, post):
        if post:
            return

{'fraction_non_alphanumeric': 0.16417440579253895, 'fraction_numerical': 0.008499921297025028, 'mean_word_length': 2.8579234972677594, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '482205', 'n_tokens_mistral': 2189, 'n_tokens_neox': 2145, 'n_words': 374}