natural language toolkit interface to the repp tokenizer c 20012015 nltk project s rebecca dridan and stephan oepen contributors liling tan url https www nltk org for license information see license txt a class for word tokenization using the repp parser described in rebecca dridan and stephan oepen 2012 tokenization returning to a long solved problem a survey contrastive experiment recommendations and toolkit in acl http anthology aclweb orgpp12p122 pdfpage406 sents tokenization is widely regarded as a solved problem due to the high accuracy that rulebased tokenizers achieve but rulebased tokenizers are hard to maintain and their rules language specific we evaluated our method on three languages and obtained error rates of 0 27 english 0 35 dutch and 0 76 italian for our best models tokenizer repptokenizer homealvasrepp doctest skip for sent in sents doctest skip tokenizer tokenizesent doctest skip u tokenization u is u widely u regarded u as u a u solved u problem u due u to u the u high u accuracy u that u rulebased u tokenizers u achieve u u but u rulebased u tokenizers u are u hard u to u maintain u and u their u rules u language u specific u u we u evaluated u our u method u on u three u languages u and u obtained u error u rates u of u 0 27 u u u english u u u 0 35 u u u dutch u u and u 0 76 u u u italian u u for u our u best u models u for sent in tokenizer tokenizesentssents doctest skip printsent doctest skip u tokenization u is u widely u regarded u as u a u solved u problem u due u to u the u high u accuracy u that u rulebased u tokenizers u achieve u u but u rulebased u tokenizers u are u hard u to u maintain u and u their u rules u language u specific u u we u evaluated u our u method u on u three u languages u and u obtained u error u rates u of u 0 27 u u u english u u u 0 35 u u u dutch u u and u 0 76 u u u italian u u for u our u best u models u for sent in tokenizer tokenizesentssents keeptokenpositionstrue doctest skip printsent doctest skip u tokenization 0 12 u is 13 15 u widely 16 22 u regarded 23 31 u as 32 34 u a 35 36 u solved 37 43 u problem 44 51 u due 52 55 u to 56 58 u the 59 62 u high 63 67 u accuracy 68 76 u that 77 81 u rulebased 82 91 u tokenizers 92 102 u achieve 103 110 u 110 111 u but 0 3 u rulebased 4 14 u tokenizers 15 25 u are 26 29 u hard 30 34 u to 35 37 u maintain 38 46 u and 47 50 u their 51 56 u rules 57 62 u language 63 71 u specific 72 80 u 80 81 u we 0 2 u evaluated 3 12 u our 13 16 u method 17 23 u on 24 26 u three 27 32 u languages 33 42 u and 43 46 u obtained 47 55 u error 56 61 u rates 62 67 u of 68 70 u 0 27 71 75 u 75 76 u 77 78 u english 78 85 u 85 86 u 86 87 u 0 35 88 92 u 92 93 u 94 95 u dutch 95 100 u 100 101 u and 102 105 u 0 76 106 110 u 110 111 u 112 113 u italian 113 120 u 120 121 u for 122 125 u our 126 129 u best 130 134 u models 135 141 u 141 142 set a directory to store the temporary files set an encoding for the input strings use repp to tokenize a single sentence param sentence a single sentence string type sentence str return a tuple of tokens rtype tuplestr tokenize multiple sentences using repp param sentences a list of sentence strings type sentences liststr return a list of tuples of tokens rtype itertuplestr write sentences to temporary input file generate command to run repp decode the stdout and strips the ending newline removes token position information this module generates the repp command to be used at the terminal param inputfilename path to the input file type inputfilename str this module parses the trituple format that repp outputs using the format triple option and returns an generator with tuple of string tokens param reppoutput type reppoutput type return an iterable of the tokenized sentences as tuples of strings rtype itertuple a module to find repp tokenizer binary and its repp set config file checks for the repp binary and ergrepp set config file natural language toolkit interface to the repp tokenizer c 2001 2015 nltk project s rebecca dridan and stephan oepen contributors liling tan url https www nltk org for license information see license txt a class for word tokenization using the repp parser described in rebecca dridan and stephan oepen 2012 tokenization returning to a long solved problem a survey contrastive experiment recommendations and toolkit in acl http anthology aclweb org p p12 p12 2 pdf page 406 sents tokenization is widely regarded as a solved problem due to the high accuracy that rulebased tokenizers achieve but rule based tokenizers are hard to maintain and their rules language specific we evaluated our method on three languages and obtained error rates of 0 27 english 0 35 dutch and 0 76 italian for our best models tokenizer repptokenizer home alvas repp doctest skip for sent in sents doctest skip tokenizer tokenize sent doctest skip u tokenization u is u widely u regarded u as u a u solved u problem u due u to u the u high u accuracy u that u rulebased u tokenizers u achieve u u but u rule based u tokenizers u are u hard u to u maintain u and u their u rules u language u specific u u we u evaluated u our u method u on u three u languages u and u obtained u error u rates u of u 0 27 u u u english u u u 0 35 u u u dutch u u and u 0 76 u u u italian u u for u our u best u models u for sent in tokenizer tokenize_sents sents doctest skip print sent doctest skip u tokenization u is u widely u regarded u as u a u solved u problem u due u to u the u high u accuracy u that u rulebased u tokenizers u achieve u u but u rule based u tokenizers u are u hard u to u maintain u and u their u rules u language u specific u u we u evaluated u our u method u on u three u languages u and u obtained u error u rates u of u 0 27 u u u english u u u 0 35 u u u dutch u u and u 0 76 u u u italian u u for u our u best u models u for sent in tokenizer tokenize_sents sents keep_token_positions true doctest skip print sent doctest skip u tokenization 0 12 u is 13 15 u widely 16 22 u regarded 23 31 u as 32 34 u a 35 36 u solved 37 43 u problem 44 51 u due 52 55 u to 56 58 u the 59 62 u high 63 67 u accuracy 68 76 u that 77 81 u rulebased 82 91 u tokenizers 92 102 u achieve 103 110 u 110 111 u but 0 3 u rule based 4 14 u tokenizers 15 25 u are 26 29 u hard 30 34 u to 35 37 u maintain 38 46 u and 47 50 u their 51 56 u rules 57 62 u language 63 71 u specific 72 80 u 80 81 u we 0 2 u evaluated 3 12 u our 13 16 u method 17 23 u on 24 26 u three 27 32 u languages 33 42 u and 43 46 u obtained 47 55 u error 56 61 u rates 62 67 u of 68 70 u 0 27 71 75 u 75 76 u 77 78 u english 78 85 u 85 86 u 86 87 u 0 35 88 92 u 92 93 u 94 95 u dutch 95 100 u 100 101 u and 102 105 u 0 76 106 110 u 110 111 u 112 113 u italian 113 120 u 120 121 u for 122 125 u our 126 129 u best 130 134 u models 135 141 u 141 142 set a directory to store the temporary files set an encoding for the input strings use repp to tokenize a single sentence param sentence a single sentence string type sentence str return a tuple of tokens rtype tuple str tokenize multiple sentences using repp param sentences a list of sentence strings type sentences list str return a list of tuples of tokens rtype iter tuple str write sentences to temporary input file generate command to run repp decode the stdout and strips the ending newline removes token position information this module generates the repp command to be used at the terminal param inputfilename path to the input file type inputfilename str this module parses the tri tuple format that repp outputs using the format triple option and returns an generator with tuple of string tokens param repp_output type repp_output type return an iterable of the tokenized sentences as tuples of strings rtype iter tuple a module to find repp tokenizer binary and its repp set config file if a full path is given try to find path to repp directory in environment variables checks for the repp binary and erg repp set config file
import os
import re
import subprocess
import sys
import tempfile

from nltk.data import ZipFilePathPointer
from nltk.internals import find_dir
from nltk.tokenize.api import TokenizerI


class ReppTokenizer(TokenizerI):
    

    def __init__(self, repp_dir, encoding="utf8"):
        self.repp_dir = self.find_repptokenizer(repp_dir)
        
        self.working_dir = tempfile.gettempdir()
        
        self.encoding = encoding

    def tokenize(self, sentence):
        
        return next(self.tokenize_sents([sentence]))

    def tokenize_sents(self, sentences, keep_token_positions=False):
        
        with tempfile.NamedTemporaryFile(
            prefix="repp_input.", dir=self.working_dir, mode="w", delete=False
        ) as input_file:
            
            for sent in sentences:
                input_file.write(str(sent) + "\n")
            input_file.close()
            
            cmd = self.generate_repp_command(input_file.name)
            
            repp_output = self._execute(cmd).decode(self.encoding).strip()
            for tokenized_sent in self.parse_repp_outputs(repp_output):
                if not keep_token_positions:
                    
                    tokenized_sent, starts, ends = zip(*tokenized_sent)
                yield tokenized_sent

    def generate_repp_command(self, inputfilename):
        
        cmd = [self.repp_dir + "/src/repp"]
        cmd += ["-c", self.repp_dir + "/erg/repp.set"]
        cmd += ["--format", "triple"]
        cmd += [inputfilename]
        return cmd

    @staticmethod
    def _execute(cmd):
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        stdout, stderr = p.communicate()
        return stdout

    @staticmethod
    def parse_repp_outputs(repp_output):
        
        line_regex = re.compile(r"^\((\d+), (\d+), (.+)\)$", re.MULTILINE)
        for section in repp_output.split("\n\n"):
            words_with_positions = [
                (token, int(start), int(end))
                for start, end, token in line_regex.findall(section)
            ]
            words = tuple(t[2] for t in words_with_positions)
            yield words_with_positions

    def find_repptokenizer(self, repp_dirname):
        
        if os.path.exists(repp_dirname):  
            _repp_dir = repp_dirname
        else:  
            _repp_dir = find_dir(repp_dirname, env_vars=("REPP_TOKENIZER",))
        
        assert os.path.exists(_repp_dir + "/src/repp")
        assert os.path.exists(_repp_dir + "/erg/repp.set")
        return _repp_dir