natural language toolkit pros and cons corpus reader c 20012023 nltk project pierpaolo pantone 24alsecondogmail com url https www nltk org for license information see license txt corpusreader for the pros and cons dataset pros and cons dataset information contact bing liu liubcs uic edu https www cs uic eduliub distributed with permission related papers murthy ganapathibhotla and bing liu mining opinions in comparative sentences proceedings of the 22nd international conference on computational linguistics coling2008 manchester 1822 august 2008 bing liu minqing hu and junsheng cheng opinion observer analyzing and comparing opinions on the web proceedings of the 14th international world wide web conference www2005 may 1014 2005 in chiba japan reader for the pros and cons sentence dataset from nltk corpus import proscons proscons sentscategories cons doctest normalizewhitespace east batteries on off switch too easy to maneuver eats no gulps batteries proscons words integratedpros txt easy to use economical param root the root directory for the corpus param fileids a list or regexp specifying the fileids in the corpus param wordtokenizer a tokenizer for breaking sentences or paragraphs into words default whitespacetokenizer param encoding the encoding that should be used to read the corpus param kwargs additional parameters passed to categorizedcorpusreader return all sentences in the corpus or in the specified filescategories param fileids a list or regexp specifying the ids of the files whose sentences have to be returned param categories a list specifying the categories whose sentences have to be returned return the given files as a list of sentences each sentence is tokenized using the specified wordtokenizer rtype listliststr return all words and punctuation symbols in the corpus or in the specified filescategories param fileids a list or regexp specifying the ids of the files whose words have to be returned param categories a list specifying the categories whose words have to be returned return the given files as a list of words and punctuation symbols rtype liststr natural language toolkit pros and cons corpus reader c 2001 2023 nltk project pierpaolo pantone 24alsecondo gmail com url https www nltk org for license information see license txt corpusreader for the pros and cons dataset pros and cons dataset information contact bing liu liub cs uic edu https www cs uic edu liub distributed with permission related papers murthy ganapathibhotla and bing liu mining opinions in comparative sentences proceedings of the 22nd international conference on computational linguistics coling 2008 manchester 18 22 august 2008 bing liu minqing hu and junsheng cheng opinion observer analyzing and comparing opinions on the web proceedings of the 14th international world wide web conference www 2005 may 10 14 2005 in chiba japan reader for the pros and cons sentence dataset from nltk corpus import pros_cons pros_cons sents categories cons doctest normalize_whitespace east batteries on off switch too easy to maneuver eats no gulps batteries pros_cons words integratedpros txt easy to use economical param root the root directory for the corpus param fileids a list or regexp specifying the fileids in the corpus param word_tokenizer a tokenizer for breaking sentences or paragraphs into words default whitespacetokenizer param encoding the encoding that should be used to read the corpus param kwargs additional parameters passed to categorizedcorpusreader return all sentences in the corpus or in the specified files categories param fileids a list or regexp specifying the ids of the files whose sentences have to be returned param categories a list specifying the categories whose sentences have to be returned return the given file s as a list of sentences each sentence is tokenized using the specified word_tokenizer rtype list list str return all words and punctuation symbols in the corpus or in the specified files categories param fileids a list or regexp specifying the ids of the files whose words have to be returned param categories a list specifying the categories whose words have to be returned return the given file s as a list of words and punctuation symbols rtype list str read 20 lines at a time
import re

from nltk.corpus.reader.api import *
from nltk.tokenize import *


class ProsConsCorpusReader(CategorizedCorpusReader, CorpusReader):
    

    CorpusView = StreamBackedCorpusView

    def __init__(
        self,
        root,
        fileids,
        word_tokenizer=WordPunctTokenizer(),
        encoding="utf8",
        **kwargs
    ):
        

        CorpusReader.__init__(self, root, fileids, encoding)
        CategorizedCorpusReader.__init__(self, kwargs)
        self._word_tokenizer = word_tokenizer

    def sents(self, fileids=None, categories=None):
        
        fileids = self._resolve(fileids, categories)
        if fileids is None:
            fileids = self._fileids
        elif isinstance(fileids, str):
            fileids = [fileids]
        return concat(
            [
                self.CorpusView(path, self._read_sent_block, encoding=enc)
                for (path, enc, fileid) in self.abspaths(fileids, True, True)
            ]
        )

    def words(self, fileids=None, categories=None):
        
        fileids = self._resolve(fileids, categories)
        if fileids is None:
            fileids = self._fileids
        elif isinstance(fileids, str):
            fileids = [fileids]
        return concat(
            [
                self.CorpusView(path, self._read_word_block, encoding=enc)
                for (path, enc, fileid) in self.abspaths(fileids, True, True)
            ]
        )

    def _read_sent_block(self, stream):
        sents = []
        for i in range(20):  
            line = stream.readline()
            if not line:
                continue
            sent = re.match(r"^(?!\n)\s*<(Pros|Cons)>(.*)</(?:Pros|Cons)>", line)
            if sent:
                sents.append(self._word_tokenizer.tokenize(sent.group(2).strip()))
        return sents

    def _read_word_block(self, stream):
        words = []
        for sent in self._read_sent_block(stream):
            words.extend(sent)
        return words