<jupyter_start><jupyter_text>Dask AnalysisDask + Analysis.<jupyter_code>import glob
files = glob.glob("*.parquet")
for file in files:
    ...
    #dd.read_parquet(file)
file
import dask
import dask.dataframe as dd
import os
dd.read_parquet(file)
lazy_results = list()
for file in files:
    task = dask.delayed(dd.read_parquet)(os.path.abspath(file))
    task = task.repartition(npartitions=10)
    lazy_results.append(task)
from dask.cache import Cache
cache = Cache(2e9)  # Leverage two gigabytes of memory
cache.register()    # Turn cache on globally
from dask.distributed import Client, progress
client = Client()
client
df_ = dask.delayed(dd.concat)(lazy_results).repartition(npartitions=8)
df = dask.compute(df_)
df2 = dask.persist(df)
df[0]
d = df[0]
result = d.groupby("can_id_hex").agg({"timestamp": ["count"]})
r = result.compute()
r.sort_values(("timestamp", "count"))
h421 = d[d.can_id_hex=="421"]
h421_2 = h421.groupby("can_data_hex").agg({"timestamp": ["count"]})
h421_2.compute()
h421_3 = h421.groupby("B00").agg({"timestamp": ["count"]})
h421.groupby(['B00','B01','B02']).agg({"timestamp": ["count"]}).compute()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17286084701815038, 'fraction_numerical': 0.037165082108902334, 'mean_word_length': 6.148148148148148, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '381395', 'license': 'MIT', 'n_tokens_mistral': 459, 'n_tokens_neox': 436, 'n_words': 83, 'path': 'Dask_Analysis.ipynb', 'repo_name': 'AutomotiveDevOps/ctf_tesla_logs', 'size': '1157'}