<jupyter_start><jupyter_text>Calculate Basic Statistics of Detailed Reviews<jupyter_code>import gzip
import json
import re
import os
import sys
import numpy as np
import pandas as pd
pd.options.display.float_format = '{:,}'.format<jupyter_output><empty_output><jupyter_text>**Specify your directory here:**<jupyter_code>DIR = './'
DIR_GENRE = './genre/'<jupyter_output><empty_output><jupyter_text>**Calculate statistics of book reviews (without spoiler tags)**<jupyter_code>def count_reviews(file_name):
    print('counting file:', file_name)
    n_review = 0
    book_set, user_set = set(), set()
    print('current line: ', end='')
    with gzip.open(file_name) as fin:
        for l in fin:
            d = json.loads(l)
            if n_review % 1000000 == 0:
                print(n_review, end=',')
            n_review += 1
            book_set.add(d['book_id'])
            user_set.add(d['user_id'])
    print('complete')
    print('done!')
    return n_review, len(book_set), len(user_set)
n_review, n_book, n_user = count_reviews(os.path.join(DIR, 'goodreads_reviews_dedup.json.gz'))
df_stats_review = pd.DataFrame([n_review, n_book, n_user], dtype=float,
                               columns=['count'], index=['# review', '# book', '# user'])
display(df_stats_review)<jupyter_output>counting file: ./goodreads_reviews_dedup.json.gz
current line: 0,1000000,2000000,3000000,4000000,5000000,6000000,7000000,8000000,9000000,10000000,11000000,12000000,13000000,14000000,15000000,complete
done!<jupyter_text>**Calculate statistics of the review subset with spoiler tags**<jupyter_code>def count_spoilers(file_name):
    print('counting file:', file_name)
    n_review, n_sentence, n_spoiler_review, n_spoiler_sentence = 0, 0, 0, 0
    book_set, user_set = set(), set()
    print('current line: ', end='')
    with gzip.open(file_name) as fin:
        for l in fin:
            d = json.loads(l)
            if n_review % 1000000 == 0:
                print(n_review, end=',')
            n_review += 1
            for _t, _ in d['review_sentences']:
                n_sentence += 1
                n_spoiler_sentence += _t
            n_spoiler_review += int(d['has_spoiler'])
            book_set.add(d['book_id'])
            user_set.add(d['user_id'])
    print('complete')
    print('done!')
    return n_review, n_sentence, n_spoiler_review, n_spoiler_sentence, len(book_set), len(user_set)
res = count_spoilers(os.path.join(DIR, 'goodreads_reviews_spoiler.json.gz'))
df_stats_spoiler = pd.DataFrame(res, columns=['count'], dtype=float, 
                               index=['# review', '# sentence', '# spoiler review', '# spoiler sentence',
                                      '# book', '# user'])
display(df_stats_spoiler)<jupyter_output>counting file: ./goodreads_reviews_spoiler.json.gz
current line: 0,1000000,complete
done!<jupyter_text>**Calculate basic statistics of review files for each genre:**<jupyter_code>genre_list = ['children', 'comics_graphic', 'fantasy_paranormal', 'history_biography',
              'mystery_thriller_crime', 'poetry', 'romance', 'young_adult']

def count_all_genres(genre_list):
    res = []
    for g in genre_list:
        n_review, n_book, n_user = count_reviews(
            os.path.join(DIR_GENRE, 'goodreads_reviews_'+g+'.json.gz'))
        res.append([n_review, n_book, n_user])
    df_stats_by_genre = pd.DataFrame(res, dtype = float, 
                                     columns = ['# review', '# book', '# user'],
                                     index = genre_list)
    return df_stats_by_genre
df_stats_by_genre = count_all_genres(genre_list)
display(df_stats_by_genre)<jupyter_output>counting file: ./genre/goodreads_reviews_children.json.gz
current line: 0,complete
done!
counting file: ./genre/goodreads_reviews_comics_graphic.json.gz
current line: 0,complete
done!
counting file: ./genre/goodreads_reviews_fantasy_paranormal.json.gz
current line: 0,1000000,2000000,3000000,complete
done!
counting file: ./genre/goodreads_reviews_history_biography.json.gz
current line: 0,1000000,2000000,complete
done!
counting file: ./genre/goodreads_reviews_mystery_thriller_crime.json.gz
current line: 0,1000000,complete
done!
counting file: ./genre/goodreads_reviews_poetry.json.gz
current line: 0,complete
done!
counting file: ./genre/goodreads_reviews_romance.json.gz
current line: 0,1000000,2000000,3000000,complete
done!
counting file: ./genre/goodreads_reviews_young_adult.json.gz
current line: 0,1000000,2000000,complete
done!
{'fraction_non_alphanumeric': 0.12458286985539488, 'fraction_numerical': 0.05094549499443826, 'mean_word_length': 3.79317697228145, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '170626', 'license': 'Apache-2.0', 'n_tokens_mistral': 1746, 'n_tokens_neox': 1525, 'n_words': 289, 'path': 'reviews.ipynb', 'repo_name': 'xiw483/cse158-assignment2', 'size': '4495'}