<jupyter_start><jupyter_text>UMAP dimensionality reduction<jupyter_code>reducer_cosine = umap.UMAP(metric='cosine')
sc = StandardScaler()
reducer_pipeline = Pipeline(steps=[('scale', sc), 
                                   ('reduce', reducer_cosine)])
cosine_embeddings = pd.DataFrame(reducer_pipeline.fit_transform(df_model))
umap.plot.points(reducer_cosine, labels = df_input_type)
df_model_1 = df_model.loc[(df['WSG_ID']==15)
                     & (df['WZD_ID']==10027)
                     & (df['ST_ID']==20371)
                     & (df['AT_ID']==10082), :]

df_NIO_labels_1 = df['NIO_GROUP'][(df['WSG_ID']==15)
                            & (df['WZD_ID']==10027)
                            & (df['ST_ID']==20371)
                            & (df['AT_ID']==10082)]

reducer_cosine_1 = umap.UMAP(metric='cosine')
sc_1 = StandardScaler()
reducer_pipeline_1 = Pipeline(steps=[('scale', sc_1), 
                                   ('reduce', reducer_cosine_1)])
cosine_embeddings_1 = pd.DataFrame(reducer_pipeline_1.fit_transform(df_model_1))
umap.plot.points(reducer_cosine_1, labels=df_NIO_labels_1)
df_model_1.columns<jupyter_output><empty_output><jupyter_text>1) data -> clustering  = labelsdata -> umap = visualizationdata + cluster labels = > classification 2)data -> dim. reduction -> clustering algo = labels<jupyter_code>reducer_default_1 = umap.UMAP()
sc_default_1 = StandardScaler()
reducer_default_pipeline_1 = Pipeline(steps=[('scale', sc_default_1), 
                                   ('reduce', reducer_default_1)])
reducer_default_pipeline_1.fit(df_model_1)
umap.plot.points(reducer_default_1, labels=df_NIO_labels_1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13892215568862276, 'fraction_numerical': 0.03473053892215569, 'mean_word_length': 3.456, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '269236', 'license': 'FTL', 'n_tokens_mistral': 618, 'n_tokens_neox': 587, 'n_words': 65, 'path': 'notebooks/MS-03-EDA-MA_NR_78635.ipynb', 'repo_name': 'malwinasanjose/anomaly_detection', 'size': '1670'}