<jupyter_start><jupyter_text>Specify inputs only in the cell below.<jupyter_code>export_file_path = 'data/goodreads_library_export.csv'<jupyter_output><empty_output><jupyter_text>Do not change anything except for in the cell above unless you're changing functionality.<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
plt.rcParams['figure.figsize'] = [15, 3]
df = pd.read_csv(
    export_file_path,
    usecols=[
        'My Rating',
        'Average Rating',
        'Number of Pages',
        'Read Count'
    ]
)

df.rename(columns={'Average Rating': 'Community Rating'}, inplace=True)

df['Number of Pages'] = df['Number of Pages'].fillna(0)
df['Number of Pages'] = df['Number of Pages'].astype('int64')

df = df[df['Read Count'] > 0]
df.drop(axis=1, columns=['Read Count'], inplace=True)

print('Dataset loaded and prepared.')
print(f'Total pages read: {df["Number of Pages"].sum()}')
comm_ratings_np = df[["Community Rating"]].to_numpy().squeeze()
my_ratings_np = df[["My Rating"]].to_numpy().squeeze()

print(f'Overall my average ratings:        {round(my_ratings_np.mean(), 2)}')
print(f'Overall community average ratings: {round(comm_ratings_np.mean(), 2)}')
print(f'Correlation:                       {round(np.corrcoef(comm_ratings_np, my_ratings_np)[0][1], 2)}')
ticks = [i/2 for i in range(2, 11)]

fig, ax = plt.subplots()
ax.scatter(comm_ratings_np, my_ratings_np, color='brown')

# Credit: https://stackoverflow.com/questions/22239691/code-for-best-fit-straight-line-of-a-scatter-plot-in-python
ax.plot(np.unique(comm_ratings_np), np.poly1d(np.polyfit(comm_ratings_np, my_ratings_np, 1))(np.unique(comm_ratings_np)))

ax.set_xticks(ticks=ticks)
ax.set_yticks(ticks=ticks)
ax.title.set_text('My Ratings vs. Community Ratings')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15448504983388706, 'fraction_numerical': 0.014950166112956811, 'mean_word_length': 5.5234657039711195, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '360241', 'license': 'Apache-2.0', 'n_tokens_mistral': 687, 'n_tokens_neox': 657, 'n_words': 131, 'path': 'run_me.ipynb', 'repo_name': 'kapoor1992/goodreads-info', 'size': '1806'}