<jupyter_start><jupyter_text>Read from Parquet<jupyter_code># select only the columns that you plan to use
%timeit
df = dd.read_parquet('/Volumes/Mosadoluwa 2TB/Neurovista/Pat1Train.parquet',engine='pyarrow')<jupyter_output><empty_output><jupyter_text>Data Description<jupyter_code>df.head()
df.dtypes
print(f'There are {int(df.size.compute() / df.shape[1])} rows and {df.shape[1]} columns')

df['class'].value_counts().compute()<jupyter_output><empty_output><jupyter_text>Preictal<jupyter_code>pat1_preictal_description = df[df['class'] == 1].iloc[:, 3:-1].describe().compute()
pat1_preictal_description.head()

mins = pat1_preictal_description.iloc[3]
maxes = pat1_preictal_description.iloc[7]
means = pat1_preictal_description.iloc[1]
std = pat1_preictal_description.iloc[2]
# create stacked errorbars:
numOfCh = pat1_preictal_description.shape[1]
plt.errorbar(np.arange(numOfCh), means, std, fmt='ok', lw=3)
plt.errorbar(np.arange(numOfCh), means, [means - mins, maxes - means],
             fmt='.k', ecolor='gray', lw=1)
plt.xlim(-1, numOfCh)
plt.show()<jupyter_output><empty_output><jupyter_text>Interictal<jupyter_code>pat1_interictal_description = df[df['class'] == 0].iloc[:, 3:-1].describe().compute()

pat1_interictal_description
mins = pat1_interictal_description.iloc[3]
maxes = pat1_interictal_description.iloc[7]
means = pat1_interictal_description.iloc[1]
std = pat1_interictal_description.iloc[2]
# create stacked errorbars:
numOfCh = pat1_interictal_description.shape[1]
plt.errorbar(np.arange(numOfCh), means, std, fmt='ok', lw=3)
plt.errorbar(np.arange(numOfCh), means, [means - mins, maxes - means],
             fmt='.k', ecolor='gray', lw=1)
plt.xlim(-1, numOfCh)
plt.show()<jupyter_output><empty_output><jupyter_text>Analyze data by 1 hour blocks = every 6th segmentsPreictal training files after segment 150 or 25 hours can be viewed as extra files.<jupyter_code>pdf_interictal_block1 = df[df['block_id'] == 1].compute()
pdf_interictal_block1['segment'].value_counts()<jupyter_output><empty_output><jupyter_text>Plot the first 10 minutes of block 1<jupyter_code>pdf_seg1_block1 = pdf_interictal_block1.loc[(
    pdf_interictal_block1['segment'] == 1)].iloc[:, 3:-1]
pdf_seg1_block1.shape
pdf_seg1_block1.plot(legend=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14486391571553994, 'fraction_numerical': 0.028094820017559263, 'mean_word_length': 7.9372549019607845, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '36586', 'license': 'MIT', 'n_tokens_mistral': 926, 'n_tokens_neox': 885, 'n_words': 133, 'path': 'Train_PreProcess_Dask.ipynb', 'repo_name': 'mobatusi/NeuromatchAcademy_DeepLearning_Coursework', 'size': '2278'}