<commit_before># Natural Language Toolkit: Product Reviews Corpus Reader
#
# Copyright (C) 2001-2015 NLTK Project
# Author: <NAME> <<EMAIL>>
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT

"""
CorpusReader for reviews corpora (syntax based on Customer Review Corpus).

- Customer Review Corpus information -
Annotated by: <NAME> and <NAME>, 2004.
    Department of Computer Sicence
    University of Illinois at Chicago

Contact: <NAME>, <EMAIL>
        http://www.cs.uic.edu/~liub

Distributed with permission.

The "product_reviews_1" and "product_reviews_2" datasets respectively contain
annotated customer reviews of 5 and 9 products from amazon.com.

Related papers:

- <NAME> and <NAME>. "Mining and summarizing customer reviews".
    Proceedings of the ACM SIGKDD International Conference on Knowledge
    Discovery & Data Mining (KDD-04), 2004.

- <NAME> and <NAME>. "Mining Opinion Features in Customer Reviews".
    Proceedings of Nineteeth National Conference on Artificial Intelligence
    (AAAI-2004), 2004.

- <NAME>, <NAME> and <NAME>. "A Holistic Lexicon-Based Appraoch to
    Opinion Mining." Proceedings of First ACM International Conference on Web
    Search and Data Mining (WSDM-2008), Feb 11-12, 2008, Stanford University,
    Stanford, California, USA.

Symbols used in the annotated reviews:

    [t] : the title of the review: Each [t] tag starts a review.
    xxxx[+|-n]: xxxx is a product feature.
    [+n]: Positive opinion, n is the opinion strength: 3 strongest, and 1 weakest.
          Note that the strength is quite subjective.
          You may want ignore it, but only considering + and -
    [-n]: Negative opinion
    ##  : start of each sentence. Each line is a sentence.
    [u] : feature not appeared in the sentence.
    [p] : feature not appeared in the sentence. Pronoun resolution is needed.
    [s] : suggestion or recommendation.
    [cc]: comparison with a competing product from a different brand.
    [cs]: comparison with a competing product from the same brand.

Note: Some of the files (e.g. "ipod.txt", "Canon PowerShot SD500.txt") do not
    provide separation between different reviews. This is due to the fact that
    the dataset was specifically designed for aspect/feature-based sentiment
    analysis, for which sentence-level annotation is sufficient. For document-
    level classification and analysis, this peculiarity should be taken into
    consideration.
"""
import re

from __future__ import division

from nltk.corpus.reader.api import *
from nltk.tokenize import *

TITLE = re.compile(r'^\[t\](.*)$') # [t] Title
FEATURES = re.compile(r'((?:(?:\w+\s)+)?\w+)\[((?:\+|\-)\d)\]') # find 'feature' in feature[+3]
NOTES = re.compile(r'\[(?!t)(p|u|s|cc|cs)\]') # find 'p' in camera[+2][p]
SENT = re.compile(r'##(.*)$') # find tokenized sentence


@compat.python_2_unicode_compatible
class Review(object):
    """
    A Review is the main block of a ReviewsCorpusReader.
    """
    def __init__(self, title=None, review_lines=None):
        """
        :param title: the title of the review.
        :param review_lines: the list of the ReviewLines that belong to the Review.
        """
        self.title = title
        if review_lines is None:
            self.review_lines = []
        else:
            self.review_lines = review_lines

    def add_line(self, review_line):
        """
        Add a line (ReviewLine) to the review.

        :param review_line: a ReviewLine instance that belongs to the Review.
        """
        assert isinstance(review_line, ReviewLine)
        self.review_lines.append(review_line)

    def features(self):
        """
        Return a list of features in the review. Each feature is a tuple made of
        the specific item feature and the opinion strength about that feature.

        :return: all features of the review as a list of tuples (feat, score).
        :rtype: list(tuple)
        """
        features = []
        for review_line in self.review_lines:
            features.extend(review_line.features)
        return features

    def sents(self):
        """
        Return all tokenized sentences in the review.

        :return: all sentences of the review as lists of tokens.
        :rtype: list(list(str))
        """
        return [review_line.sent for review_line in self.review_lines]

    def __repr__(self):
        return 'Review(title=\"{}\", review_lines={})'.format(self.title, self.review_lines)


@compat.python_2_unicode_compatible
class ReviewLine(object):
    """
    A ReviewLine represents a sentence of the review, together with (optional)
    annotations of its features and notes about the reviewed item.
    """
    def __init__(self, sent, features=None, notes=None):
        self.sent = sent
        if features is None:
            self.features = []
        else:
            self.features = features

        if notes is None:
            self.notes = []
        else:
            self.notes = notes

    def __repr__(self):
        return ('ReviewLine(features={}, notes={}, sent={})'.format(
            self.features, self.notes, self.sent))


class ReviewsCorpusReader(CorpusReader):
    """
    Reader for the Customer Review Data dataset by <NAME> (2004).
    Note: we are not applying any sentence tokenization at the moment, just word
    tokenization.

        >>> from nltk.corpus import product_reviews_1
        >>> camera_reviews = product_reviews_1.reviews('Canon_G3.txt')
        >>> review = camera_reviews[0]
        >>> review.sents()[0]
        ['i', 'recently', 'purchased', 'the', 'canon', 'powershot', 'g3', 'and', 'am',
        'extremely', 'satisfied', 'with', 'the', 'purchase', '.']
        >>> review.features()
        [('canon powershot g3', '+3'), ('use', '+2'), ('picture', '+2'),
        ('picture quality', '+1'), ('picture quality', '+1'), ('camera', '+2'),
        ('use', '+2'), ('feature', '+1'), ('picture quality', '+3'), ('use', '+1'),
        ('option', '+1')]

    We can also reach the same information directly from the stream:

        >>> product_reviews_1.features('Canon_G3.txt')
        [('canon powershot g3', '+3'), ('use', '+2'), ...]

    We can compute stats for specific product features:

        >>> n_reviews = len([(feat,score) for (feat,score) in product_reviews_1.features('Canon_G3.txt') if feat=='picture'])
        >>> tot = sum([int(score) for (feat,score) in product_reviews_1.features('Canon_G3.txt') if feat=='picture'])
        >>> # We use float for backward compatibility with division in Python2.7
        >>> mean = tot / n_reviews<commit_msg>Move division import into the doctest in corpus/reader/reviews.py<commit_after># Natural Language Toolkit: Product Reviews Corpus Reader
#
# Copyright (C) 2001-2015 NLTK Project
# Author: <NAME> <<EMAIL>>
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT

"""
CorpusReader for reviews corpora (syntax based on Customer Review Corpus).

- Customer Review Corpus information -
Annotated by: <NAME> and <NAME>, 2004.
    Department of Computer Sicence
    University of Illinois at Chicago

Contact: <NAME>, <EMAIL>
        http://www.cs.uic.edu/~liub

Distributed with permission.

The "product_reviews_1" and "product_reviews_2" datasets respectively contain
annotated customer reviews of 5 and 9 products from amazon.com.

Related papers:

- Minqing Hu and <NAME>. "Mining and summarizing customer reviews".
    Proceedings of the ACM SIGKDD International Conference on Knowledge
    Discovery & Data Mining (KDD-04), 2004.

- <NAME> and <NAME>. "Mining Opinion Features in Customer Reviews".
    Proceedings of Nineteeth National Conference on Artificial Intelligence
    (AAAI-2004), 2004.

- <NAME>, <NAME> and <NAME>. "A Holistic Lexicon-Based Appraoch to
    Opinion Mining." Proceedings of First ACM International Conference on Web
    Search and Data Mining (WSDM-2008), Feb 11-12, 2008, Stanford University,
    Stanford, California, USA.

Symbols used in the annotated reviews:

    [t] : the title of the review: Each [t] tag starts a review.
    xxxx[+|-n]: xxxx is a product feature.
    [+n]: Positive opinion, n is the opinion strength: 3 strongest, and 1 weakest.
          Note that the strength is quite subjective.
          You may want ignore it, but only considering + and -
    [-n]: Negative opinion
    ##  : start of each sentence. Each line is a sentence.
    [u] : feature not appeared in the sentence.
    [p] : feature not appeared in the sentence. Pronoun resolution is needed.
    [s] : suggestion or recommendation.
    [cc]: comparison with a competing product from a different brand.
    [cs]: comparison with a competing product from the same brand.

Note: Some of the files (e.g. "ipod.txt", "Canon PowerShot SD500.txt") do not
    provide separation between different reviews. This is due to the fact that
    the dataset was specifically designed for aspect/feature-based sentiment
    analysis, for which sentence-level annotation is sufficient. For document-
    level classification and analysis, this peculiarity should be taken into
    consideration.
"""
import re

from nltk.corpus.reader.api import *
from nltk.tokenize import *

TITLE = re.compile(r'^\[t\](.*)$') # [t] Title
FEATURES = re.compile(r'((?:(?:\w+\s)+)?\w+)\[((?:\+|\-)\d)\]') # find 'feature' in feature[+3]
NOTES = re.compile(r'\[(?!t)(p|u|s|cc|cs)\]') # find 'p' in camera[+2][p]
SENT = re.compile(r'##(.*)$') # find tokenized sentence


@compat.python_2_unicode_compatible
class Review(object):
    """
    A Review is the main block of a ReviewsCorpusReader.
    """
    def __init__(self, title=None, review_lines=None):
        """
        :param title: the title of the review.
        :param review_lines: the list of the ReviewLines that belong to the Review.
        """
        self.title = title
        if review_lines is None:
            self.review_lines = []
        else:
            self.review_lines = review_lines

    def add_line(self, review_line):
        """
        Add a line (ReviewLine) to the review.

        :param review_line: a ReviewLine instance that belongs to the Review.
        """
        assert isinstance(review_line, ReviewLine)
        self.review_lines.append(review_line)

    def features(self):
        """
        Return a list of features in the review. Each feature is a tuple made of
        the specific item feature and the opinion strength about that feature.

        :return: all features of the review as a list of tuples (feat, score).
        :rtype: list(tuple)
        """
        features = []
        for review_line in self.review_lines:
            features.extend(review_line.features)
        return features

    def sents(self):
        """
        Return all tokenized sentences in the review.

        :return: all sentences of the review as lists of tokens.
        :rtype: list(list(str))
        """
        return [review_line.sent for review_line in self.review_lines]

    def __repr__(self):
        return 'Review(title=\"{}\", review_lines={})'.format(self.title, self.review_lines)


@compat.python_2_unicode_compatible
class ReviewLine(object):
    """
    A ReviewLine represents a sentence of the review, together with (optional)
    annotations of its features and notes about the reviewed item.
    """
    def __init__(self, sent, features=None, notes=None):
        self.sent = sent
        if features is None:
            self.features = []
        else:
            self.features = features

        if notes is None:
            self.notes = []
        else:
            self.notes = notes

    def __repr__(self):
        return ('ReviewLine(features={}, notes={}, sent={})'.format(
            self.features, self.notes, self.sent))


class ReviewsCorpusReader(CorpusReader):
    """
    Reader for the Customer Review Data dataset by <NAME> (2004).
    Note: we are not applying any sentence tokenization at the moment, just word
    tokenization.

        >>> from nltk.corpus import product_reviews_1
        >>> camera_reviews = product_reviews_1.reviews('Canon_G3.txt')
        >>> review = camera_reviews[0]
        >>> review.sents()[0]
        ['i', 'recently', 'purchased', 'the', 'canon', 'powershot', 'g3', 'and', 'am',
        'extremely', 'satisfied', 'with', 'the', 'purchase', '.']
        >>> review.features()
        [('canon powershot g3', '+3'), ('use', '+2'), ('picture', '+2'),
        ('picture quality', '+1'), ('picture quality', '+1'), ('camera', '+2'),
        ('use', '+2'), ('feature', '+1'), ('picture quality', '+3'), ('use', '+1'),
        ('option', '+1')]

    We can also reach the same information directly from the stream:

        >>> product_reviews_1.features('Canon_G3.txt')
        [('canon powershot g3', '+3'), ('use', '+2'), ...]

    We can compute stats for specific product features:

        >>> from __future__ import division
        >>> n_reviews = len([(feat,score) for (feat,score) in product_reviews_1.features('Canon_G3.txt') if feat=='picture'])
        >>> tot = sum([int(score) for (feat,score) in product_reviews_1.features('Canon_G3.txt') if feat=='picture'])
        >>> # We use float for backward compatibility with division in Python2.7
        >>> mean = tot / n_reviews
{'fraction_non_alphanumeric': 0.12233200090504563, 'fraction_numerical': 0.012670638811373407, 'mean_word_length': 2.9891696750902526, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 97, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '97802', 'n_tokens_mistral': 3966, 'n_tokens_neox': 3751, 'n_words': 1375}