<commit_before># License: MIT
'''
:author: <NAME> (<EMAIL>)
:organization: ETS
'''

import ctypes as c
import os

class DepParser(object):
    """The ZPar English Dependency Parser"""

    def __init__(self, modelpath, libptr):
        super(DepParser, self).__init__()

        # get the library method that loads the parser models
        self._load_depparser = libptr.load_depparser
        self._load_depparser.restype = c.c_int
        self._load_depparser.argtypes = [c.c_char_p]

        # get the library methods that parse sentences and files
        self._dep_parse_sentence = libptr.dep_parse_sentence
        self._dep_parse_sentence.restype = c.c_char_p
        self._dep_parse_sentence.argtypes = [c.c_char_p]

        self._parse_file = libptr.dep_parse_file
        self._parse_file.restype = None
        self._parse_file.argtypes = [c.c_char_p, c.c_char_p]

        if self._load_depparser(modelpath.encode('utf-8')):
            raise OSError('Cannot find dependency parser model at {}\n'.format(modelpath))

    def parse_sentence(self, sentence):
        zpar_compatible_sentence = sentence.strip() + "\n "
        zpar_compatible_sentence = zpar_compatible_sentence.encode('utf-8')
        parsed_sent = self._dep_parse_sentence(zpar_compatible_sentence)
        return parsed_sent.decode('utf-8')

    def parse_file(self, inputfile, outputfile):
        if os.path.exists(inputfile):
            self._parse_file(inputfile.encode('utf-8'), outputfile.encode('utf-8'))
        else:
            raise OSError('File {} does not exist.'.format(inputfile))

    def cleanup(self):
        self._load_depparser = None
        self._dep_parse_sentence = None
        self._parse_file = None
<commit_msg>Added support for pre-tokenization and modified method names<commit_after># License: MIT
'''
:author: <NAME> (<EMAIL>)
:organization: ETS
'''

import ctypes as c
import os

class DepParser(object):
    """The ZPar English Dependency Parser"""

    def __init__(self, modelpath, libptr):
        super(DepParser, self).__init__()

        # get the library method that loads the parser models
        self._load_depparser = libptr.load_depparser
        self._load_depparser.restype = c.c_int
        self._load_depparser.argtypes = [c.c_char_p]

        # get the library methods that parse sentences and files
        self._dep_parse_sentence = libptr.dep_parse_sentence
        self._dep_parse_sentence.restype = c.c_char_p
        self._dep_parse_sentence.argtypes = [c.c_char_p, c.c_bool]

        self._parse_file = libptr.dep_parse_file
        self._parse_file.restype = None
        self._parse_file.argtypes = [c.c_char_p, c.c_char_p, c.c_bool]

        if self._load_depparser(modelpath.encode('utf-8')):
            raise OSError('Cannot find dependency parser model at {}\n'.format(modelpath))

    def dep_parse_sentence(self, sentence, tokenize=True):
        zpar_compatible_sentence = sentence.strip() + "\n "
        zpar_compatible_sentence = zpar_compatible_sentence.encode('utf-8')
        parsed_sent = self._dep_parse_sentence(zpar_compatible_sentence, tokenize)
        return parsed_sent.decode('utf-8')

    def dep_parse_file(self, inputfile, outputfile, tokenize=True):
        if os.path.exists(inputfile):
            self._parse_file(inputfile.encode('utf-8'), outputfile.encode('utf-8'), tokenize)
        else:
            raise OSError('File {} does not exist.'.format(inputfile))

    def cleanup(self):
        self._load_depparser = None
        self._dep_parse_sentence = None
        self._parse_file = None

{'fraction_non_alphanumeric': 0.10288758059994393, 'fraction_numerical': 0.0028034763106251754, 'mean_word_length': 3.1536670547147847, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4508210', 'n_tokens_mistral': 1163, 'n_tokens_neox': 1104, 'n_words': 229}