<jupyter_start><jupyter_text>Display Sample Records<jupyter_code>import gzip
import json
import re
import os
import sys
import numpy as np
import pandas as pd<jupyter_output><empty_output><jupyter_text>**Specify your directory here:**<jupyter_code>DIR = './'<jupyter_output><empty_output><jupyter_text>**This function shows how to load datasets**<jupyter_code>def load_data(file_name, head = 500):
    count = 0
    data = []
    with gzip.open(file_name) as fin:
        for l in fin:
            d = json.loads(l)
            count += 1
            data.append(d)
            
            # break if reaches the 100th line
            if (head is not None) and (count > head):
                break
    return data<jupyter_output><empty_output><jupyter_text>**Load and display sample records of books/authors/works/series**<jupyter_code>books = load_data(os.path.join(DIR, 'goodreads_books.json.gz'))
authors = load_data(os.path.join(DIR, 'goodreads_book_authors.json.gz'))
works = load_data(os.path.join(DIR, 'goodreads_book_works.json.gz'))
series = load_data(os.path.join(DIR, 'goodreads_book_series.json.gz'))
print(' == sample record (books) ==')
display(np.random.choice(books))
print(' == sample record (authors) ==')
display(np.random.choice(authors))
print(' == sample record (works) ==')
display(np.random.choice(works))
print(' == sample record (series) ==')
display(np.random.choice(series))<jupyter_output>== sample record (books) ==<jupyter_text>**Load and display sample records of user-book interactions (shelves)**<jupyter_code>interactions = load_data(os.path.join(DIR, 'goodreads_interactions_dedup.json.gz'))
np.random.choice(interactions)<jupyter_output><empty_output><jupyter_text>**Load and display sample records of book reviews**<jupyter_code>reviews = load_data(os.path.join(DIR, 'goodreads_reviews_dedup.json.gz'))
np.random.choice(reviews)<jupyter_output><empty_output><jupyter_text>**Load and display sample records of book reviews (with spoiler tags)**<jupyter_code>spoilers = load_data(os.path.join(DIR, 'goodreads_reviews_spoiler.json.gz'))
np.random.choice(spoilers)
spoilers = load_data(os.path.join(DIR, 'goodreads_reviews_spoiler_raw.json.gz'))
np.random.choice([s for s in spoilers if 'view spoiler' in s['review_text']])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.137117903930131, 'fraction_numerical': 0.0034934497816593887, 'mean_word_length': 5.564469914040115, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '120217', 'license': 'Apache-2.0', 'n_tokens_mistral': 805, 'n_tokens_neox': 780, 'n_words': 154, 'path': 'samples.ipynb', 'repo_name': 'MengtingWan/goodreads', 'size': '2290'}