<commit_before>'''
Created on Jan 16, 2017

@author: <NAME>

    Purpose:
        Code for performing robust density clustering 
        
'''

import numpy as np
from matplotlib import pyplot as plt
import seaborn as sns
from sklearn.neighbors import KernelDensity,NearestNeighbors
from sklearn import datasets
import plotting
import time
from sklearn.model_selection import train_test_split
from numpy.random import random
import sys,os
from special_datasets import gaussian_mixture
import pickle

def main():
    """
    Example on a gaussian mixture with n=15 centers in 2 dimension with 100000 data points
    """

    n_true_center=10
    #X,y=datasets.make_blobs(10000,2,n_true_center,random_state=0)

    X,y=gaussian_mixture(n_sample=10000,n_center=n_true_center,sigma_range=[0.25,0.5,1.25],pop_range=[0.1,0.02,0.1,0.1,0.3,0.1,0.08,0.02,0.08,0.1],
                          random_state=10)

    dcluster=DCluster(bandwidth='auto',NH_size=40)
    cluster_label,idx_centers,rho,delta,kde_tree=dcluster.fit(X)    
    plotting.summary(idx_centers,cluster_label,rho,n_true_center,X,y)



##########################
##########################
##########################


class DCluster:
    """ Fast two dimensional density clustering via kernel density modelling
    
    
       
    Parameters
    ----------

    NH_size : int, optional (default: 40)
        Neighborhood size. This is related to the perplexity (in t-SNE)
        and is an effective scale that defines the number of neighbors of each data point. 
        Larger datasets usually require a larger perplexity/NH_size. Consider selecting a value
        between 20 and 100.
    
    random_state: int, optional (default: 0)
        Random number for seeding random number generator
    
    verbose: int, optional (default: 1)
        Set to 0 if you don't want to see print to screen.
    
    bandwidth: str, optional (default: 'auto')
        If you want the bandwidth to be set automatically or want to set it yourself.
        Valid options = {'auto' | 'manual'}
        
    bandwidth_value: float, required if bandwidth='manual'
    
    """
    
    def __init__(self,NH_size=40,test_size=0.1,
                 random_state=0,verbose=1,
                 bandwidth='auto',bandwidth_value=None):
        self.test_size=test_size
        self.random_state=random_state
        self.verbose=verbose
        self.bandwidth_value=None
        self.NH_size=NH_size
        if bandwidth is not 'auto': # Need to implement something else here
            assert bandwidth is 'manual'
            self.bandwidth_value=bandwidth_value
    
    def fit(self,X):
        if self.verbose==0:
            blockPrint()
            
        n_sample=X.shape[0]
        print("--> Starting clustering with n=%i samples..."%n_sample)
        start=time.time()
    
        if self.bandwidth_value is None:
            print("--> Finding optimal bandwidth ...")
            X_train, X_test, y_train,y_test = train_test_split(X,range(X.shape[0]),test_size=self.test_size, random_state=self.random_state)
            bandwidthCV=find_optimal_bandwidth(X,X_train,X_test)
        else:
            bandwidthCV=self.bandwidth_value
        
        print("--> Using bandwidth = %.3f"%bandwidthCV)
    
        print("--> Computing density ...")
        rho,kde=compute_density(X,bandwidth=bandwidthCV)
    
        print("--> Finding centers ...")
        delta,nn_delta,idx_centers,density_graph=compute_delta(X,rho,kde.tree_,cutoff=self.NH_size)
        
        print("--> Assigning labels ...")
        cluster_label=assign_cluster(idx_centers,nn_delta,density_graph)
        
        print("--> Done in %.3f s"%(time.time()-start))
        
        print("--> Found %i centers ! ..."%idx_centers.shape[0])
        
        enablePrint()
        
        return cluster_label,idx_centers,rho,delta,kde.tree_
    
 
def bandwidth_estimate(X):
    """
    Purpose:
        Gives a rough estimate of the optimal bandwidth (based on the notion 
        of some effective neigborhood)
    """
    nbrs = NearestNeighbors(n_neighbors=40,algorithm='kd_tree').fit(X)
    nn_dist,_ = nbrs.kneighbors(X)

    return np.median(nn_dist[:,-1]),np.mean(nn_dist[:,1]),
  
def log_likelihood_test_set(bandwidth,X_train,X_test):
    """
    Purpose:
        Fit the kde model on the training set given some bandwidth and evaluates the log-likelihood of the test set
    """
    kde=KernelDensity(bandwidth=bandwidth, algorithm='kd_tree', atol=0.0005, rtol=0.0005,leaf_size=40)
    kde.fit(X_train)<commit_msg>Bandwidth option corrected<commit_after>'''
Created on Jan 16, 2017

@author: <NAME>

    Purpose:
        Code for performing robust density clustering 
        
'''

import numpy as np
from matplotlib import pyplot as plt
import seaborn as sns
from sklearn.neighbors import KernelDensity,NearestNeighbors
from sklearn import datasets
import plotting
import time
from sklearn.model_selection import train_test_split
from numpy.random import random
import sys,os
from special_datasets import gaussian_mixture
import pickle

def main():
    """
    Example on a gaussian mixture with n=15 centers in 2 dimension with 100000 data points
    """

    n_true_center=10
    #X,y=datasets.make_blobs(10000,2,n_true_center,random_state=0)

    X,y=gaussian_mixture(n_sample=10000,n_center=n_true_center,sigma_range=[0.25,0.5,1.25],
                            pop_range=[0.1,0.02,0.1,0.1,0.3,0.1,0.08,0.02,0.08,0.1],
                            random_state=10)

    dcluster=DCluster(NH_size=40)
    cluster_label,idx_centers,rho,delta,kde_tree=dcluster.fit(X)    
    plotting.summary(idx_centers,cluster_label,rho,n_true_center,X,y)



##########################
##########################
##########################


class DCluster:
    """ Fast two dimensional density clustering via kernel density modelling
      
    Parameters
    ----------

    NH_size : int, optional (default: 40)
        Neighborhood size. This is related to the perplexity (in t-SNE)
        and is an effective scale that defines the number of neighbors of each data point. 
        Larger datasets usually require a larger perplexity/NH_size. Consider selecting a value
        between 20 and 100.
    
    random_state: int, optional (default: 0)
        Random number for seeding random number generator
    
    verbose: int, optional (default: 1)
        Set to 0 if you don't want to see print to screen.
    
    bandwidth: str, optional (default: 'auto')
        If you want the bandwidth to be set automatically or want to set it yourself.
        Valid options = {'auto' | 'manual'}
        
    bandwidth_value: float, required if bandwidth='manual'
    
    """
    
    def __init__(self,NH_size = 40,test_size = 0.1,
                 random_state = 0,verbose = 1,
                 bandwidth = None):

        self.test_size = test_size
        self.random_state = random_state
        self.verbose = verbose
        self.NH_size = NH_size
        self.bandwidth = bandwidth
    
    def fit(self,X):
        if self.verbose==0:
            blockPrint()
            
        n_sample = X.shape[0]
        print("--> Starting clustering with n=%i samples..." % n_sample)
        start = time.time()
    
        if self.bandwidth:
            bandwidthCV = self.bandwidth
        else:
            print("--> Finding optimal bandwidth ...")
            X_train, X_test, y_train, y_test = train_test_split(X, range(X.shape[0]), test_size = self.test_size, random_state = self.random_state)
            bandwidthCV = find_optimal_bandwidth(X, X_train, X_test)
        
        print("--> Using bandwidth = %.3f" % bandwidthCV)
    
        print("--> Computing density ...")
        rho, kde = compute_density(X, bandwidth = bandwidthCV)
    
        print("--> Finding centers ...")
        delta,nn_delta,idx_centers,density_graph=compute_delta(X,rho,kde.tree_,cutoff=self.NH_size)
        
        print("--> Assigning labels ...")
        cluster_label=assign_cluster(idx_centers,nn_delta,density_graph)
        
        print("--> Done in %.3f s" % (time.time()-start))
        
        print("--> Found %i centers ! ..." % idx_centers.shape[0])
        
        enablePrint()
        
        return cluster_label, idx_centers, rho, delta, kde.tree_
    
 
def bandwidth_estimate(X):
    """
    Purpose:
        Gives a rough estimate of the optimal bandwidth (based on the notion 
        of some effective neigborhood)
    """
    nbrs = NearestNeighbors(n_neighbors=40,algorithm='kd_tree').fit(X)
    nn_dist,_ = nbrs.kneighbors(X)

    return np.median(nn_dist[:,-1]),np.mean(nn_dist[:,1]),
  
def log_likelihood_test_set(bandwidth,X_train,X_test):
    """
    Purpose:
        Fit the kde model on the training set given some bandwidth and evaluates the log-likelihood of the test set
    """
    kde=KernelDensity(bandwidth=bandwidth, algorithm='kd_tree', atol=0.0005, rtol=0.0005,leaf_size=40)
    kde.fit(X_train)
{'fraction_non_alphanumeric': 0.11685994647636039, 'fraction_numerical': 0.02297056199821588, 'mean_word_length': 2.726215205650187, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2179107', 'n_tokens_mistral': 2965, 'n_tokens_neox': 2727, 'n_words': 678}