natural language toolkit comparative sentence corpus reader c 20012023 nltk project pierpaolo pantone 24alsecondogmail com url https www nltk org for license information see license txt corpusreader for the comparative sentence dataset comparative sentence dataset information annotated by nitin jindal and bing liu 2006 department of computer sicence university of illinois at chicago contact nitin jindal njindalcs uic edu bing liu liubcs uic edu https www cs uic eduliub distributed with permission related papers nitin jindal and bing liu identifying comparative sentences in text documents proceedings of the acm sigir international conference on information retrieval sigir06 2006 nitin jindal and bing liu mining comprative sentences and relations proceedings of twenty first national conference on artificial intelligence aaai2006 2006 murthy ganapathibhotla and bing liu mining opinions in comparative sentences proceedings of the 22nd international conference on computational linguistics coling2008 manchester 1822 august 2008 regular expressions for dataset components a comparison represents a comparative sentence and its constituents param text a string optionally tokenized containing a comparison param comptype an integer defining the type of comparison expressed values can be 1 nonequal gradable 2 equative 3 superlative 4 nongradable param entity1 the first entity considered in the comparison relation param entity2 the second entity considered in the comparison relation param feature the feature considered in the comparison relation param keyword the word or phrase which is used for that comparative relation reader for the comparative sentence dataset by jindal and liu 2006 from nltk corpus import comparativesentences comparison comparativesentences comparisons0 comparison text doctest normalizewhitespace its fastforward and rewind work much more smoothly and consistently than those of other models i ve had comparison entity2 models comparison feature comparison keyword rewind more lencomparativesentences comparisons 853 param root the root directory for this corpus param fileids a list or regexp specifying the fileids in this corpus param wordtokenizer tokenizer for breaking sentences or paragraphs into words default whitespacetokenizer param senttokenizer tokenizer for breaking paragraphs into sentences param encoding the encoding that should be used to read the corpus return all comparisons in the corpus param fileids a list or regexp specifying the ids of the files whose comparisons have to be returned return the given files as a list of comparison objects rtype listcomparison return a set of all keywords used in the corpus param fileids a list or regexp specifying the ids of the files whose keywords have to be returned return the set of keywords and comparative phrases used in the corpus rtype setstr return the list of words and constituents considered as clues of a comparison from listofkeywords txt return all sentences in the corpus param fileids a list or regexp specifying the ids of the files whose sentences have to be returned return all sentences of the corpus as lists of tokens or as plain strings if no word tokenizer is specified rtype listliststr or liststr return all words and punctuation symbols in the corpus param fileids a list or regexp specifying the ids of the files whose words have to be returned return the given files as a list of words and punctuation symbols rtype liststr advance to the next line it contains the comparative sentence skip the next line it contains closing comparison tags if gradable comparisons are found create comparison instances and populate their fields each comparison tag has its own relations on a separate line if nongradable comparisons are found create a simple comparison instance for each one comptype in this case should always be 4 flatten the list of comparisons before returning them return concatcomparisonbundle natural language toolkit comparative sentence corpus reader c 2001 2023 nltk project pierpaolo pantone 24alsecondo gmail com url https www nltk org for license information see license txt corpusreader for the comparative sentence dataset comparative sentence dataset information annotated by nitin jindal and bing liu 2006 department of computer sicence university of illinois at chicago contact nitin jindal njindal cs uic edu bing liu liub cs uic edu https www cs uic edu liub distributed with permission related papers nitin jindal and bing liu identifying comparative sentences in text documents proceedings of the acm sigir international conference on information retrieval sigir 06 2006 nitin jindal and bing liu mining comprative sentences and relations proceedings of twenty first national conference on artificial intelligence aaai 2006 2006 murthy ganapathibhotla and bing liu mining opinions in comparative sentences proceedings of the 22nd international conference on computational linguistics coling 2008 manchester 18 22 august 2008 regular expressions for dataset components a comparison represents a comparative sentence and its constituents param text a string optionally tokenized containing a comparison param comp_type an integer defining the type of comparison expressed values can be 1 non equal gradable 2 equative 3 superlative 4 non gradable param entity_1 the first entity considered in the comparison relation param entity_2 the second entity considered in the comparison relation param feature the feature considered in the comparison relation param keyword the word or phrase which is used for that comparative relation reader for the comparative sentence dataset by jindal and liu 2006 from nltk corpus import comparative_sentences comparison comparative_sentences comparisons 0 comparison text doctest normalize_whitespace its fast forward and rewind work much more smoothly and consistently than those of other models i ve had comparison entity_2 models comparison feature comparison keyword rewind more len comparative_sentences comparisons 853 param root the root directory for this corpus param fileids a list or regexp specifying the fileids in this corpus param word_tokenizer tokenizer for breaking sentences or paragraphs into words default whitespacetokenizer param sent_tokenizer tokenizer for breaking paragraphs into sentences param encoding the encoding that should be used to read the corpus return all comparisons in the corpus param fileids a list or regexp specifying the ids of the files whose comparisons have to be returned return the given file s as a list of comparison objects rtype list comparison return a set of all keywords used in the corpus param fileids a list or regexp specifying the ids of the files whose keywords have to be returned return the set of keywords and comparative phrases used in the corpus rtype set str return the list of words and constituents considered as clues of a comparison from listofkeywords txt return all sentences in the corpus param fileids a list or regexp specifying the ids of the files whose sentences have to be returned return all sentences of the corpus as lists of tokens or as plain strings if no word tokenizer is specified rtype list list str or list str return all words and punctuation symbols in the corpus param fileids a list or regexp specifying the ids of the files whose words have to be returned return the given file s as a list of words and punctuation symbols rtype list str end of file advance to the next line it contains the comparative sentence skip the next line it contains closing comparison tags if gradable comparisons are found create comparison instances and populate their fields each comparison tag has its own relations on a separate line if non gradable comparisons are found create a simple comparison instance for each one comp_type in this case should always be 4 flatten the list of comparisons before returning them return concat comparison_bundle
import re

from nltk.corpus.reader.api import *
from nltk.tokenize import *


STARS = re.compile(r"^\*+$")
COMPARISON = re.compile(r"<cs-[1234]>")
CLOSE_COMPARISON = re.compile(r"</cs-[1234]>")
GRAD_COMPARISON = re.compile(r"<cs-[123]>")
NON_GRAD_COMPARISON = re.compile(r"<cs-4>")
ENTITIES_FEATS = re.compile(r"(\d)_((?:[\.\w\s/-](?!\d_))+)")
KEYWORD = re.compile(r"\(([^\(]*)\)$")


class Comparison:
    

    def __init__(
        self,
        text=None,
        comp_type=None,
        entity_1=None,
        entity_2=None,
        feature=None,
        keyword=None,
    ):
        
        self.text = text
        self.comp_type = comp_type
        self.entity_1 = entity_1
        self.entity_2 = entity_2
        self.feature = feature
        self.keyword = keyword

    def __repr__(self):
        return (
            'Comparison(text="{}", comp_type={}, entity_1="{}", entity_2="{}", '
            'feature="{}", keyword="{}")'
        ).format(
            self.text,
            self.comp_type,
            self.entity_1,
            self.entity_2,
            self.feature,
            self.keyword,
        )


class ComparativeSentencesCorpusReader(CorpusReader):
    

    CorpusView = StreamBackedCorpusView

    def __init__(
        self,
        root,
        fileids,
        word_tokenizer=WhitespaceTokenizer(),
        sent_tokenizer=None,
        encoding="utf8",
    ):
        

        CorpusReader.__init__(self, root, fileids, encoding)
        self._word_tokenizer = word_tokenizer
        self._sent_tokenizer = sent_tokenizer
        self._readme = "README.txt"

    def comparisons(self, fileids=None):
        
        if fileids is None:
            fileids = self._fileids
        elif isinstance(fileids, str):
            fileids = [fileids]
        return concat(
            [
                self.CorpusView(path, self._read_comparison_block, encoding=enc)
                for (path, enc, fileid) in self.abspaths(fileids, True, True)
            ]
        )

    def keywords(self, fileids=None):
        
        all_keywords = concat(
            [
                self.CorpusView(path, self._read_keyword_block, encoding=enc)
                for (path, enc, fileid) in self.abspaths(fileids, True, True)
            ]
        )

        keywords_set = {keyword.lower() for keyword in all_keywords if keyword}
        return keywords_set

    def keywords_readme(self):
        
        keywords = []
        with self.open("listOfkeywords.txt") as fp:
            raw_text = fp.read()
        for line in raw_text.split("\n"):
            if not line or line.startswith("//"):
                continue
            keywords.append(line.strip())
        return keywords

    def sents(self, fileids=None):
        
        return concat(
            [
                self.CorpusView(path, self._read_sent_block, encoding=enc)
                for (path, enc, fileid) in self.abspaths(fileids, True, True)
            ]
        )

    def words(self, fileids=None):
        
        return concat(
            [
                self.CorpusView(path, self._read_word_block, encoding=enc)
                for (path, enc, fileid) in self.abspaths(fileids, True, True)
            ]
        )

    def _read_comparison_block(self, stream):
        while True:
            line = stream.readline()
            if not line:
                return []  
            comparison_tags = re.findall(COMPARISON, line)
            if comparison_tags:
                grad_comparisons = re.findall(GRAD_COMPARISON, line)
                non_grad_comparisons = re.findall(NON_GRAD_COMPARISON, line)
                
                comparison_text = stream.readline().strip()
                if self._word_tokenizer:
                    comparison_text = self._word_tokenizer.tokenize(comparison_text)
                
                stream.readline()
                
                
                comparison_bundle = []
                if grad_comparisons:
                    
                    for comp in grad_comparisons:
                        comp_type = int(re.match(r"<cs-(\d)>", comp).group(1))
                        comparison = Comparison(
                            text=comparison_text, comp_type=comp_type
                        )
                        line = stream.readline()
                        entities_feats = ENTITIES_FEATS.findall(line)
                        if entities_feats:
                            for (code, entity_feat) in entities_feats:
                                if code == "1":
                                    comparison.entity_1 = entity_feat.strip()
                                elif code == "2":
                                    comparison.entity_2 = entity_feat.strip()
                                elif code == "3":
                                    comparison.feature = entity_feat.strip()
                        keyword = KEYWORD.findall(line)
                        if keyword:
                            comparison.keyword = keyword[0]
                        comparison_bundle.append(comparison)
                
                
                if non_grad_comparisons:
                    for comp in non_grad_comparisons:
                        
                        comp_type = int(re.match(r"<cs-(\d)>", comp).group(1))
                        comparison = Comparison(
                            text=comparison_text, comp_type=comp_type
                        )
                        comparison_bundle.append(comparison)
                
                
                return comparison_bundle

    def _read_keyword_block(self, stream):
        keywords = []
        for comparison in self._read_comparison_block(stream):
            keywords.append(comparison.keyword)
        return keywords

    def _read_sent_block(self, stream):
        while True:
            line = stream.readline()
            if re.match(STARS, line):
                while True:
                    line = stream.readline()
                    if re.match(STARS, line):
                        break
                continue
            if (
                not re.findall(COMPARISON, line)
                and not ENTITIES_FEATS.findall(line)
                and not re.findall(CLOSE_COMPARISON, line)
            ):
                if self._sent_tokenizer:
                    return [
                        self._word_tokenizer.tokenize(sent)
                        for sent in self._sent_tokenizer.tokenize(line)
                    ]
                else:
                    return [self._word_tokenizer.tokenize(line)]

    def _read_word_block(self, stream):
        words = []
        for sent in self._read_sent_block(stream):
            words.extend(sent)
        return words