<jupyter_start><jupyter_text>Exploring Review Data- let's look at some typical reviews, how many files there are, what the distributions of review lengths and hours played are, etc.<jupyter_code>import os
import sys
from json import loads
from collections import Counter

import numpy as np
import pandas as pd
# So, let's take a look at how many lines are in our reviews files
# Note: I just started processing GTAV after ending all the other processes
os.chdir('..')
! wc -l data/*.jsonlines<jupyter_output>7151 data/Arma_3.jsonlines
     7073 data/Counter_Strike_Global_Offensive.jsonlines
     6040 data/Counter_Strike.jsonlines
     9720 data/Dota_2.jsonlines
     1522 data/Football_Manager_2015.jsonlines
     7151 data/Garrys_Mod.jsonlines
    13349 data/Grand_Theft_Auto_V.jsonlines
        3 data/sample.jsonlines
     7467 data/Sid_Meiers_Civilization_5.jsonlines
     5676 data/Team_Fortress_2.jsonlines
     7165 data/The_Elder_Scrolls_V.jsonlines
     7123 data/Warframe.jsonlines
    79440 total<jupyter_text>Football Manager 2015 Stats<jupyter_code># So, let's first get all of the reviews for the game with the smallest amount of
# review data, i.e., Football Manager 2015 (not including GTAV)
reviews = [loads(review)['review'] for review in open('data/Football_Manager_2015.jsonlines')]
# First of all, how many reviews are there?
print('number of reviews: {}'.format(len(reviews)))
# Here's a couple reviews from the beginning of the file
reviews[:3]<jupyter_output><empty_output><jupyter_text>Football Manager 2015: Review Length Distribution<jupyter_code># Let's measure the lengths of each review using a "list comprehension"
lengths = [len(review) for review in reviews]
# Let's print out the first 10 lengths, just to see what we're working with
lengths[:10]
# Compute the average length value
avg_len = sum(lengths)/len(lengths)
print('average length: {}'.format(avg_len))
min_len = min(lengths)
print('minimum review length = {}'.format(min_len))
max_len = max(lengths)
print('maximum review length = {}'.format(max_len))
_lengths = np.array(lengths)
std = _lengths.std()
_lengths.size
bins = std/10.0
_rounded_lengths = np.array([np.ceil(l/bins)*bins for l in lengths])
_rounded_lengths
# Let's make bins of size 300
rounded_lengths = [np.ceil(l/60)*60 for l in lengths]
print('original lengths (first 10):          {}\nrounded lengths (to the nearest 300): {}'
      .format(lengths[:10], rounded_lengths[:10]))
# Now, let's make a frequency distribution with the collections.Counter module
rounded_length_fdist = Counter(_rounded_lengths)
rounded_length_fdist
# It is obvious from looking at the freq. dist. above that length drops off a cliff
# after about 1000 characters
# In fact, for length up to 900, almost 1,050 reviews are accounted for, which means
# that only a little over 100 reviews are thinly-distributed over the remaining area
# above 1,000 characters
# Usually, when you use Pandas, you're using a dataframe, but a dataframe, as I understand
# it, it just made up on a set of "Series"
# Let's make a Series from our rounded lengths and then call its value_counts() method to
# get exactly what collections.Counter was doing (but we'll be able to use it to make a
# nice plot)
rounded_length_series = pd.Series(_rounded_lengths)
rounded_length_series.value_counts()
# The table above is nice, but let's do better and try to get a histogram
# Don't worry about all this importing stuff, it's just from something I read in a blog
# post
# Actually, go and check out the blog post here:
# http://nbviewer.ipython.org/github/mwaskom/seaborn/blob/master/examples/plotting_distributions.ipynb
# It's in, you guessed it, an IPython notebook! Shows what you can do with
# matplotlib.
%matplotlib inline
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
sns.set_palette("deep", desat=.6)
sns.set_context(rc={"figure.figsize": (8, 4)})
fig = plt.figure()
ax = fig.add_subplot(111)
ax.hist(pd.Series(lengths), label='Game')
ax.set_xlabel('Review length (in characters)')
ax.set_ylabel('Total reviews')
ax.set_title('Histogram')
fig.savefig('sample_fig')
# That is one nice-looking histogram!
# From it, we can really see just how few reviews there are past 1,000
# For this item, we could probably set the cap at 1,200, let's say
# What do you think?
# If 90% of the reviews have length <= 1200 of the reviews, but not 900,
# let's set MAXLEN to 1200
len([r for r in reviews if len(r) <= 1200])/len(reviews) >= 0.9<jupyter_output><empty_output><jupyter_text>Football Manager 2015: Hours Distribution<jupyter_code># Let's do a similar kind of thing for the hours values
hours  = [loads(review)['total_game_hours'] for review in open('data/Football_Manager_2015.jsonlines')]
hours[:10]
print("min: {}\nmax: {}".format(min(hours), max(hours)))
rounded_hours = [np.ceil(h/200)*200 for h in hours]
rounded_hours[:10]
# Let's use pandas again instead of collections.Counter
rounded_hours_series = pd.Series(rounded_hours)
rounded_hours_series.value_counts()
# Hmm, well, would you look at that! The distribution looks exactly the same
# as that for length!
# Let's plot it!
plt.hist(rounded_hours_series)
# If 90% of the reviews have hours values <= 800 of the reviews, but not 600,
# let's set MAXHOURS to 800
len([r for r in hours if float(r) <= 400])/len(hours) >= 0.9<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09598506069094305, 'fraction_numerical': 0.03436041083099907, 'mean_word_length': 5.163406214039125, 'pattern_counts': {'":': 1, '<': 20, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '248890', 'license': 'MIT', 'n_tokens_mistral': 1878, 'n_tokens_neox': 1655, 'n_words': 625, 'path': 'jupyter_notebooks/Exploring_Review_Data.ipynb', 'repo_name': 'mulhod/reviewer_experience_prediction', 'size': '5355'}