<jupyter_start><jupyter_text>Dynamic Aspect Extraction for `camera` Reviews Part <NAME> (<EMAIL>)As a follow-up demonstration, this ipynb is focused on extracting aspects from datasets called `AmazonReviews`, which has much more reviews on cameras.   Set up<jupyter_code>import json
import nltk
import string

from srs.utilities import Product, AspectPattern<jupyter_output><empty_output><jupyter_text>s1: load raw data from `AmazonReviews` datasets<jupyter_code>product_name = 'B00000JFIF'
reviewJsonFile = product_name + '.json'
product = Product(name=product_name)
product.loadReviewsFromJsonFile('../data/trainingFiles/AmazonReviews/cameras/' + reviewJsonFile)<jupyter_output><empty_output><jupyter_text>s2: define aspect patterns<jupyter_code>aspectPatterns = []
# define an aspect pattern1
pattern_name = 'adj_nn'
pattern_structure ="""
adj_nn:{<JJ><NN.?>}
"""
aspectTagIndices = [1]
aspectPattern = AspectPattern(name='adj_nn', structure=pattern_structure, aspectTagIndices=aspectTagIndices)
aspectPatterns.append(aspectPattern)
# define an aspect pattern2
pattern_name = 'nn_nn'
pattern_structure ="""
nn_nn:{<NN.?><NN.?>}
"""
aspectTagIndices = [0,1]
aspectPattern = AspectPattern(name='nn_nn', structure=pattern_structure, aspectTagIndices=aspectTagIndices)
aspectPatterns.append(aspectPattern)<jupyter_output><empty_output><jupyter_text>s3: match sentence to pattern to extract aspects<jupyter_code># pos tagging
for review in product.reviews:
    for sentence in review.sentences:
        sentence.pos_tag()
        sentence.matchDaynamicAspectPatterns(aspectPatterns)<jupyter_output><empty_output><jupyter_text>s4: statistic analysis on aspects extracted across all reviews<jupyter_code>word_dict = {}
for review in product.reviews:
    for sentence in review.sentences:
        for aspect in sentence.dynamic_aspects:
            if aspect in word_dict:
                word_dict[aspect] += 1
            else:
                word_dict[aspect] = 1
word_sorted = sorted(word_dict.items(), key=lambda tup:-tup[1])
word_sorted[:15]<jupyter_output><empty_output><jupyter_text>s5: save most frequent dynamic aspects<jupyter_code>import json
word_output = open('../data/word_list/{0}_wordlist.txt'.format(product_name), 'w')
json.dump(word_sorted[:15], word_output)
word_output.close()<jupyter_output><empty_output><jupyter_text>s6: stemming analysis<jupyter_code>from nltk.stem import SnowballStemmer
stemmer = SnowballStemmer('english')
# collect word with same stem
stemmedWord_dict = {}
for word in word_dict:
    stemmedWord = stemmer.stem(word)
    if stemmedWord in stemmedWord_dict:
        stemmedWord_dict[stemmedWord] += word_dict[word]
    else:
        stemmedWord_dict[stemmedWord] = word_dict[word]
# frequency ranking
stemmedWord_sorted = sorted(stemmedWord_dict.items(), key=lambda tup:-tup[1])
stemmedWord_sorted[:15]
# save most frequent stemmed words
stemmedWord_output = open('../data/word_list/{0}_stemmedwordlist.txt'.format(product_name), 'w')
json.dump(stemmedWord_sorted[:15], stemmedWord_output)
stemmedWord_output.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10751295336787564, 'fraction_numerical': 0.009715025906735751, 'mean_word_length': 6.66501240694789, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '200766', 'license': 'MIT', 'n_tokens_mistral': 1050, 'n_tokens_neox': 1013, 'n_words': 200, 'path': 'ipynbs/Dynamic_Aspect_Extraction_Part_B.ipynb', 'repo_name': 'MachineLearningStudyGroup/Smart_Review_Summarization', 'size': '3100'}