<commit_before>import sys
import os
import logging

import utilities

from ufal.udpipe import Model,Pipeline,ProcessingError


class UDpipeparser():

    def __init__(self, modeldir):

        self.modeldir = os.path.abspath(modeldir)
        self.model = Model.load(self.modeldir)
        self.error = ProcessingError()

        if not self.model:
            sys.stderr.write("Udpipe language model loading failed:"+self.modeldir)
            sys.exit(1)

        self.pipeline = Pipeline(self.model,"tokenize",Pipeline.DEFAULT,Pipeline.DEFAULT,"conllu")


    def udpipe_parse_sent(self, text):
        logger = logging.getLogger('petr_log')

        processed = self.pipeline.process(text,self.error)

        if self.error.occurred():
            raise ValueError(self.error.message)

        #print(processed)
        parsed = []
        for line in processed.split("\n"):
            if line.startswith("#"):
                continue
            parsed.append(line)

        #print(("\n").join(parsed))
        #input(" ")
        return ("\n").join(parsed)


    def udpipe_parse_events(self, events):
        logger = logging.getLogger('petr_log')

        #modeldir = os.path.abspath(PETRglobals.udpipe_modsel_dir)
    
        #self.model = Model.load(modeldir)
        #self.pipeline = Pipeline(model,"tokenize",Pipeline.DEFAULT,Pipeline.DEFAULT,"conllu")

<commit_msg>remove accents of noun phrases in preprocessing<commit_after>import sys
import os
import logging

import utilities
import unidecode

from ufal.udpipe import Model,Pipeline,ProcessingError


class UDpipeparser():

    def __init__(self, modeldir):

        self.modeldir = os.path.abspath(modeldir)
        self.model = Model.load(self.modeldir)
        self.error = ProcessingError()

        if not self.model:
            sys.stderr.write("Udpipe language model loading failed:"+self.modeldir)
            sys.exit(1)

        self.pipeline = Pipeline(self.model,"tokenize",Pipeline.DEFAULT,Pipeline.DEFAULT,"conllu")


    def udpipe_parse_sent(self, text):
        logger = logging.getLogger('petr_log')

        processed = self.pipeline.process(text,self.error)

        if self.error.occurred():
            raise ValueError(self.error.message)

        #print(processed)
        parsed = []
        for line in processed.split("\n"):
            if line.startswith("#"):
                continue
            if "VERB\t" not in line:
                line = unidecode.unidecode(line)
            parsed.append(line)

        #print(("\n").join(parsed))
        #input(" ")
        return ("\n").join(parsed)


    def udpipe_parse_events(self, events):
        logger = logging.getLogger('petr_log')

        #modeldir = os.path.abspath(PETRglobals.udpipe_modsel_dir)
    
        #self.model = Model.load(modeldir)
        #self.pipeline = Pipeline(model,"tokenize",Pipeline.DEFAULT,Pipeline.DEFAULT,"conllu")


{'fraction_non_alphanumeric': 0.1193515005174198, 'fraction_numerical': 0.0006898930665746809, 'mean_word_length': 2.5935563816604708, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2248739', 'n_tokens_mistral': 883, 'n_tokens_neox': 859, 'n_words': 155}