<jupyter_start><jupyter_text>todo - add cm- add xgboost to custom.json- add cleaning- add scaling- add PCA , t-SNE- add other datasets- - iris, moon, blobs<jupyter_code>%matplotlib inline

import matplotlib

import matplotlib.pyplot as plt
from sklearn.datasets import load_breast_cancer
from sklearn.datasets import load_iris
from sklearn.datasets import make_blobs
from sklearn.datasets import make_moons

iris = load_iris()

data_X, data_y = iris.data,iris.target
plt.scatter(data_X[:, 0], data_X[:, 1], s=50);
%load_ext autoreload
%autoreload 2
packages = !conda list
packages
!pwd<jupyter_output>/docker/photon_experiments<jupyter_text>Output registry<jupyter_code>from __future__ import print_function
import sys, os

old__file__ = !pwd
__file__ = !cd ../photon ;pwd
#__file__ = !pwd
__file__ = __file__[0]
__file__
sys.path.append(__file__)
print(sys.path)
os.chdir(old__file__[0])
!pwd
old__file__[0]

import seaborn as sns; sns.set()  # for plot styling
import numpy as np
import pandas as pd
from sklearn.model_selection import KFold


from photonai.base import Hyperpipe, PipelineElement, Preprocessing, OutputSettings
from photonai.optimization import FloatRange, Categorical, IntegerRange
from photonai.base.photon_elements import PhotonRegistry
#from photonai.base.registry.registry import PhotonRegistry
#import pixiedust

def results_to_df(results):
    ll = []
    for obj in results:
        ll.append([obj.operation,
        obj.value,
        obj.metric_name])
    _results=pd.DataFrame(ll).pivot(index=2, columns=0, values=1)
    _results.columns=['Mean','STD']
    
    return(_results)
__file__ = "exp1.log"
base_folder = os.path.dirname(os.path.abspath(''))
custom_elements_folder = os.path.join(base_folder, 'custom_elements')
custom_elements_folder
registry = PhotonRegistry(custom_elements_folder=custom_elements_folder)
registry.activate()
registry.PHOTON_REGISTRIES,PhotonRegistry.PHOTON_REGISTRIES
registry.activate()
registry.list_available_elements()
# take off last name<jupyter_output>PhotonCore
ARDRegression                       sklearn.linear_model.ARDRegression                                          Estimator
AdaBoostClassifier                  sklearn.ensemble.AdaBoostClassifier                                         Estimator
AdaBoostRegressor                   sklearn.ensemble.AdaBoostRegressor                                          Estimator
BaggingClassifier                   sklearn.ensemble.BaggingClassifier                                          Estimator
BaggingRegressor                    sklearn.ensemble.BaggingRegressor                                           Estimator
BayesianGaussianMixture             sklearn.mixture.BayesianGaussianMixture                                     Estimator
BayesianRidge                       sklearn.linear_model.BayesianRidge                                          Estimator
BernoulliNB                         sklearn.naive_bayes.BernoulliNB                                             Estimator
BernoulliRBM[...]<jupyter_text>KMedoids iris<jupyter_code>registry.info("KMedoids")
#import pixiedust
#%%pixie_debugger

"""
Example script for KMedoids hopt
"""
X, y = data_X, data_y
# DESIGN YOUR PIPELINE
settings = OutputSettings(project_folder='./tmp/')

my_pipe = Hyperpipe('batching',
                    optimizer='sk_opt',
#                    optimizer_params={'n_configurations': 25},
                    metrics=['ARI', 'MI', 'HCV', 'FM'],
                    best_config_metric='ARI',
                    outer_cv=KFold(n_splits=5),
                    inner_cv=KFold(n_splits=10),
                    verbosity=0,
                    output_settings=settings)


my_pipe += PipelineElement('KMedoids', hyperparameters={
                                                    'n_clusters': IntegerRange(2, 8),
                                                    },random_state=777)

# NOW TRAIN YOUR PIPELINE
my_pipe.fit(X, y)

debug = True


lab= my_pipe.predict(X)
colors = ['red','green','blue','purple']

fig = plt.figure(figsize=(8,8))

plt.scatter(X[:, 0], X[:, 1], s=50, c=lab
            ,cmap=matplotlib.colors.ListedColormap(colors) );

pd.DataFrame(my_pipe.best_config.items(),columns=['n_clusters', 'k'])
train=results_to_df(my_pipe.results.metrics_train)
train
test = results_to_df(my_pipe.results.metrics_test)
test
test-train<jupyter_output><empty_output><jupyter_text>Show kmeans iris<jupyter_code>registry.info("KMeans")
#import pixiedust
#%%pixie_debugger

"""
Example script for kmeans hopt
"""

X, y = data_X, data_y
# DESIGN YOUR PIPELINE
settings = OutputSettings(project_folder='./tmp/')

my_pipe = Hyperpipe('batching',
                    optimizer='sk_opt',
#                    optimizer_params={'n_configurations': 25},
                    metrics=['ARI', 'MI', 'HCV', 'FM'],
                    best_config_metric='ARI',
                    outer_cv=KFold(n_splits=5),
                    inner_cv=KFold(n_splits=10),
                    verbosity=0,
                    output_settings=settings)


my_pipe += PipelineElement('KMeans', hyperparameters={
                                                    'n_clusters': IntegerRange(2, 8),
                                                    },random_state=777)

# NOW TRAIN YOUR PIPELINE
my_pipe.fit(X, y)

debug = True



lab= my_pipe.predict(X)
colors = ['red','green','blue','purple']

fig = plt.figure(figsize=(8,8))

plt.scatter(X[:, 0], X[:, 1], s=50, c=lab
            ,cmap=matplotlib.colors.ListedColormap(colors) );
pd.DataFrame(my_pipe.best_config.items(),columns=['n_clusters', 'k'])
train=results_to_df(my_pipe.results.metrics_train)
train
test = results_to_df(my_pipe.results.metrics_test)
test
test-train<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1023855127981891, 'fraction_numerical': 0.008009751001218875, 'mean_word_length': 2.614852108244179, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 23, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '38443', 'license': 'MIT', 'n_tokens_mistral': 1843, 'n_tokens_neox': 1754, 'n_words': 323, 'path': 'Cluster/kmeans-kmedoids/kmeans-kmeroids-iris.ipynb', 'repo_name': 'bcottman/photon_experiments', 'size': '5743'}