natural language toolkit dependency corpus reader c 20012023 nltk project kepa sarasola kepa sarasolaehu es iker manterola returntothehangarhotmail com url https www nltk org for license information see license txt read the next sentence strip off the docstart marker if present extract word and tag from any of the formats discard tags if they weren t requested return the result natural language toolkit dependency corpus reader c 2001 2023 nltk project kepa sarasola kepa sarasola ehu es iker manterola returntothehangar hotmail com url https www nltk org for license information see license txt dokumentu hasiera definitzen da read the next sentence strip off the docstart marker if present extract word and tag from any of the formats discard tags if they weren t requested return the result
from nltk.corpus.reader.api import *
from nltk.corpus.reader.util import *
from nltk.parse import DependencyGraph
from nltk.tokenize import *


class DependencyCorpusReader(SyntaxCorpusReader):
    def __init__(
        self,
        root,
        fileids,
        encoding="utf8",
        word_tokenizer=TabTokenizer(),
        sent_tokenizer=RegexpTokenizer("\n", gaps=True),
        para_block_reader=read_blankline_block,
    ):
        SyntaxCorpusReader.__init__(self, root, fileids, encoding)

    

    def words(self, fileids=None):
        return concat(
            [
                DependencyCorpusView(fileid, False, False, False, encoding=enc)
                for fileid, enc in self.abspaths(fileids, include_encoding=True)
            ]
        )

    def tagged_words(self, fileids=None):
        return concat(
            [
                DependencyCorpusView(fileid, True, False, False, encoding=enc)
                for fileid, enc in self.abspaths(fileids, include_encoding=True)
            ]
        )

    def sents(self, fileids=None):
        return concat(
            [
                DependencyCorpusView(fileid, False, True, False, encoding=enc)
                for fileid, enc in self.abspaths(fileids, include_encoding=True)
            ]
        )

    def tagged_sents(self, fileids=None):
        return concat(
            [
                DependencyCorpusView(fileid, True, True, False, encoding=enc)
                for fileid, enc in self.abspaths(fileids, include_encoding=True)
            ]
        )

    def parsed_sents(self, fileids=None):
        sents = concat(
            [
                DependencyCorpusView(fileid, False, True, True, encoding=enc)
                for fileid, enc in self.abspaths(fileids, include_encoding=True)
            ]
        )
        return [DependencyGraph(sent) for sent in sents]


class DependencyCorpusView(StreamBackedCorpusView):
    _DOCSTART = "-DOCSTART- -DOCSTART- O\n"  

    def __init__(
        self,
        corpus_file,
        tagged,
        group_by_sent,
        dependencies,
        chunk_types=None,
        encoding="utf8",
    ):
        self._tagged = tagged
        self._dependencies = dependencies
        self._group_by_sent = group_by_sent
        self._chunk_types = chunk_types
        StreamBackedCorpusView.__init__(self, corpus_file, encoding=encoding)

    def read_block(self, stream):
        
        sent = read_blankline_block(stream)[0].strip()
        
        if sent.startswith(self._DOCSTART):
            sent = sent[len(self._DOCSTART) :].lstrip()

        
        if not self._dependencies:
            lines = [line.split("\t") for line in sent.split("\n")]
            if len(lines[0]) == 3 or len(lines[0]) == 4:
                sent = [(line[0], line[1]) for line in lines]
            elif len(lines[0]) == 10:
                sent = [(line[1], line[4]) for line in lines]
            else:
                raise ValueError("Unexpected number of fields in dependency tree file")

            
            if not self._tagged:
                sent = [word for (word, tag) in sent]

        
        if self._group_by_sent:
            return [sent]
        else:
            return list(sent)