<jupyter_start><jupyter_text>Setup<jupyter_code>cd ..
import sys
sys.path.append('../src')
import pandas as pd
from utils.data import *<jupyter_output><empty_output><jupyter_text>Clean data<jupyter_code># combine book txt files to a json (warning: may have MemoryError)
clean_books(
    "data/raw/text",                          # folder directory where the TXT books are stored
    "data/interim/text.json",                 # output file directory (.json)
    n_tokens = 8000                           # extract first n tokens with .split() tokenization
)
# extract required metadata (e.g., title, subjects) from the raw data
extract_metadata(
    "data/raw/metadata",                      # folder directory where the metadata RDF files are stored
    "data/interim/metadata.json"              # output file directory (.json)
)
# remove metadata not in the downloaded books and remove suffix in subjects (words after --) 
clean_metadata(
    "data/interim/metadata.json",             # input file directory for metadata (.json)
    "data/raw/text",                          # folder directory where the text files are stored
    "data/interim/metadata.json"              # output file directory (.json)
)<jupyter_output><empty_output><jupyter_text>Combine text and metadata<jupyter_code># import book texts and metadata
text = pd.read_json("data/interim/text.json", typ="series")
text = pd.DataFrame({"id": text.index, "text": text})
metadata = pd.read_json("data/interim/metadata.json")
# merge text and metadata
dataset = pd.merge(metadata, text, how="left", left_on='id', right_on='id', sort=True)
dataset.head()
# save dataset as json
dataset.to_json("data/interim/dataset.json")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10989643268124281, 'fraction_numerical': 0.0023014959723820483, 'mean_word_length': 3.091764705882353, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '108833', 'license': 'MIT', 'n_tokens_mistral': 516, 'n_tokens_neox': 507, 'n_words': 146, 'path': 'nbs/wrangling.ipynb', 'repo_name': 'tonychu-yw/lc-classification', 'size': '1738'}