<jupyter_start><jupyter_text>File saving<jupyter_code>for df in dfs.keys():
    dfs[df].to_parquet(prcsd_path+df+".gzip", engine="fastparquet", compression="gzip", index=False)
# from datetime import datetime
# START_TIME = datetime.now()

# # one of the following lines is uncommented before execution
# s = [np.float64(1), np.float32(1), np.float16(1), 1.0]

# for j, k in enumerate(s):
#     for i in range(10000000):
#         s[j] = (s[j] + 8) * s[j] % 2399232

#     print(s[j])
#     print('Runtime:', datetime.now() - START_TIME)
    

# 1340409.0
# Runtime: 0:00:11.860372
# 1340409.0
# Runtime: 0:00:23.970025
# 1340409.0
# Runtime: 0:00:35.694635
# 1340409.0
# Runtime: 0:00:39.258874
# def umap_trans(n_neighbors=15, min_dist=0.1, n_components=2, plot=False):
#     reducer = umap.UMAP(n_neighbors=n_neighbors, min_dist=min_dist, n_components=n_components, metric="euclidean")
#     embedding = reducer.fit_transform(train_data[num_vnames])
#     print("Embedding shape", embedding.shape)
    
#     if plot:
#         sns.set(style='white', context='notebook', rc={'figure.figsize':(14,10)})
#         plt.scatter(embedding[:, 0], embedding[:, 1], c=[sns.color_palette()[x] for x in train_data["class"].astype(int)])
#         plt.gca().set_aspect('equal', 'datalim')
#         plt.title('UMAP projection of the dataset', fontsize=24)
#     return reducer
# umap_trans(5, 0.1, 2, plot=True)
# max_feat = 100
# umap_reducer = dict()
# for vname_type in num_bd_vnames.keys():
#     if len(num_bd_vnames[vname_type]) > max_feat//len(num_bd_vnames.keys()):
#         print(vname_type, end=" ")
#         umap_reducer[vname_type] = umap_trans(5, 0.1, int(max_feat//len(num_bd_vnames.keys())))
# def apply_umap(data, num_vnames=num_bd_vnames, return_reducer=False):
#     y = data["class"].copy()
#     X = pd.DataFrame()
#     for vname_type in num_bd_vnames.keys():
#         if vname_type in umap_reducer.keys():
#             umap_reducer[vname_type] = umap_trans(5, 0.1, int(max_feat//len(num_bd_vnames.keys())))
#             X = pd.concat(
#                 [X, pd.DataFrame(umap_reducer[vname_type].transform(data[num_bd_vnames[vname_type]]), index=data.index).add_prefix(vname_type)], axis=1)
#         else:
#             X = pd.concat([X, data[num_bd_vnames[vname_type]]], axis=1)
#     if return_reducer:
#         return X, y, umap_reducer
#     else:
#         return X, y
# X_train, y_train, umap_reducer = apply_umap(train_data, return_reducer=True)
# X_test, y_test = apply_umap(test_data)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.162799842705466, 'fraction_numerical': 0.05151395988989382, 'mean_word_length': 4.017751479289941, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '330839', 'license': 'MIT', 'n_tokens_mistral': 1088, 'n_tokens_neox': 1004, 'n_words': 168, 'path': 'notebooks/outdated/assemble_reduce_data.ipynb', 'repo_name': 'cibic89/kaggle_ccs', 'size': '2543'}