<jupyter_start><jupyter_text>Demo for `mastersign.datascience.files`<jupyter_code>import os
import numpy as np
import pandas as pd<jupyter_output><empty_output><jupyter_text>Import the `mastersign.datascience.files` package.<jupyter_code>from mastersign.datascience import files as io<jupyter_output><empty_output><jupyter_text>Create demo data.<jupyter_code>data = pd.DataFrame(
    {
        "t": np.arange(0.0, 100.0, 0.1),
        "group": np.random.choice(['A', 'B', 'C'], 1000, p=[0.5, 0.2, 0.3]),
        "value_a": np.random.normal(size=1000) * 256,
        "value_b": np.random.gamma(2, size=1000),
        "value_c": np.random.laplace(size=1000),
    })
data = data.astype({'t': 'float32', 'group': 'category', 'value_a': 'uint8'})
data.info()
data.head()<jupyter_output><empty_output><jupyter_text>`write_parquet()` Write a DataFrame into a compressed Parquet file.<jupyter_code>io.write_parquet(data, 'files-demo-1.parq', compress=True)
print(f"File size: {os.stat('files-demo-1.parq').st_size / 1024.0 :.1f} KB")<jupyter_output>File size: 20.5 KB<jupyter_text>`read_parquet()` Read the data from a Parquet file.<jupyter_code>df = io.read_parquet('files-demo-1.parq')
df.info()
df.head()<jupyter_output><empty_output><jupyter_text>Read only a subset of columns and specify an index column.<jupyter_code>df = io.read_parquet('files-demo-1.parq',
                     columns=['t', 'group', 'value_a', 'value_c'],
                     index='t')
df.info()
df.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1700996677740864, 'fraction_numerical': 0.03322259136212625, 'mean_word_length': 5.0, 'pattern_counts': {'":': 5, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '235856', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 616, 'n_tokens_neox': 575, 'n_words': 99, 'path': 'demo-files.ipynb', 'repo_name': 'mastersign/mastersign-datascience', 'size': '1505'}