<jupyter_start><jupyter_text>ArrowLoaf<jupyter_code>from pathlib import Path
from arrowloaf import ArrowLoaf
FOLDER = Path('~/data/Loaf').expanduser()
CHUNKS = FOLDER/'Chunks'
INPATH = FOLDER/'names.parquet'
OUTPATH = FOLDER/'clone.parquet'
print(ArrowLoaf.__doc__)<jupyter_output>Query, read, and save Parquet tables using pandas and pyarrow.

    Inputs
        data    ArrowLoaf, DataFrame, RecordBatch, Series, Table
                OR any valid DataFrame input.
        index   bool: Keep index column? (Ignored for non-pandas input.)

    Magic
        self[key]   pyarrow.Column: Selected column.
        iter(self)  iterator: Column names.
        len(self)   int: Count rows in table.

    Example
        path = '/path/to/data.parquet'
        data = ArrowLoaf.read(path,columns=['spam','eggs'])
        data = data.query('spam > 0 and eggs > 42',chunksize=100_000)
        data.frame().to_csv('path/to/newdata.csv')<jupyter_text>build an ArrowLoaf<jupyter_code># from nothing
ArrowLoaf().frame()
# from any valid DataFrame input
data = dict()
data['name'] = ['Arthur','Lancelot','Knight']
data['quest'] = ['grail','grail','ni!']
data = ArrowLoaf(data)
data.frame()
# from DataFrame with index
ixdata = data.frame().set_index('name')
ixdata = ArrowLoaf(ixdata,index=True)
ixdata.frame()<jupyter_output><empty_output><jupyter_text>inspect data<jupyter_code>print(data)
data.head(2)
data.columns
data.shape<jupyter_output><empty_output><jupyter_text>read Parquet file<jupyter_code>%time bigdata = ArrowLoaf.read(INPATH)
print(bigdata)
bigdata.head()<jupyter_output><empty_output><jupyter_text>save Parquet file<jupyter_code>%time bigdata.save(OUTPATH)
assert bigdata == ArrowLoaf.read(OUTPATH)<jupyter_output>CPU times: user 3.41 s, sys: 500 ms, total: 3.91 s
Wall time: 6.14 s<jupyter_text>concatenate Parquet files<jupyter_code>paths = map("{}.parquet".format,range(3))
paths = [ CHUNKS/x for x in paths ]

%time catloaf = ArrowLoaf.cat(paths)
print(catloaf)<jupyter_output>CPU times: user 640 ms, sys: 100 ms, total: 740 ms
Wall time: 746 ms
ArrowLoaf
3,000,000 x 6
nconst: string
primaryName: string
birthYear: double
deathYear: double
primaryProfession: string
knownForTitles: string<jupyter_text>generate DataFrames<jupyter_code>help(ArrowLoaf.chunks)
%%time
msg = "{} with {:,} rows".format
for df in bigdata.chunks(1_000_000):
    print(msg(type(df),len(df)))<jupyter_output><class 'pandas.core.frame.DataFrame'> with 1,000,000 rows
<class 'pandas.core.frame.DataFrame'> with 1,000,000 rows
<class 'pandas.core.frame.DataFrame'> with 1,000,000 rows
<class 'pandas.core.frame.DataFrame'> with 1,000,000 rows
<class 'pandas.core.frame.DataFrame'> with 1,000,000 rows
<class 'pandas.core.frame.DataFrame'> with 1,000,000 rows
<class 'pandas.core.frame.DataFrame'> with 1,000,000 rows
<class 'pandas.core.frame.DataFrame'> with 1,000,000 rows
<class 'pandas.core.frame.DataFrame'> with 808,095 rows
CPU times: user 2.81 s, sys: 680 ms, total: 3.49 s
Wall time: 3.45 s<jupyter_text>filter columns<jupyter_code>help(ArrowLoaf.select)
cols = ['primaryName','birthYear','primaryProfession']
%time bigdata.select(cols).head()<jupyter_output>CPU times: user 10 ms, sys: 0 ns, total: 10 ms
Wall time: 1.77 ms<jupyter_text>filter rows<jupyter_code>help(ArrowLoaf.loaf)
def classic(df):
    return df.query('birthYear < 1920')

%time bigdata.loaf(classic).head()
def baldwins(df):
    hasname = df['primaryName'].str.contains
    return df.loc[hasname('Baldwin')]

%time len(bigdata.loaf(baldwins))<jupyter_output>CPU times: user 7.33 s, sys: 580 ms, total: 7.91 s
Wall time: 7.9 s
{'fraction_non_alphanumeric': 0.13699776785714285, 'fraction_numerical': 0.041015625, 'mean_word_length': 5.5181818181818185, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '539067', 'license': 'Apache-2.0', 'n_tokens_mistral': 1452, 'n_tokens_neox': 1326, 'n_words': 293, 'path': 'arrowloaf.ipynb', 'repo_name': 'samkennerly/arrowloaf', 'size': '3584'}