<commit_before>#!/usr/bin/env python2.7

import numpy as np
from mini_batch import MiniBatch
from sklearn.metrics import euclidean_distances
from collections import namedtuple

import logging

KMeansResult = namedtuple("KMeansResult", ("labels", "centroids"))

FORMAT = '[%(asctime)s] %(levelname)s - %(funcName)s:%(lineno)d: %(message)s'
logging.basicConfig(format=FORMAT, level=logging.FATAL)

class XMeans(object):
    def __init__(self, kmin, kmax=None, init='kmeans++', sample_percent = 0.20, minibatch_args=None, verbose = False):
        self.kmin = kmin
        self.kmax = kmax
        self.init = init
        self.sample_percent = sample_percent

        self.verbose = verbose
        if self.verbose:
            logging.getLogger().setLevel(logging.INFO)

        self.data_ = None
        self._minibatch_args = minibatch_args or {}
        self.cluster_centers_ = []

    def fit(self, data):
        self.data_ = data
        k = self.kmin
        cluster_centers = self.init
        while self.kmax is None or k <= self.kmax:
            logging.info("Fitting with k=%d", k)
            self._model = self._fit(k, data, cluster_centers)
            
            centroid_distances = euclidean_distances(self._model.centroids, self._model.centroids)
            centroid_distances += np.diag([np.Infinity] * k)
            centroids_range = centroid_distances.min(axis=-1)

            cluster_centers = []
            for i, centroid in enumerate(self._model.centroids):
                logging.info("\tSplitting cluster %d / %d (k=%d)", 
                    i+1, 
                    len(self._model.centroids), 
                    len(cluster_centers),
                )
                direction = np.random.random(centroid.shape)
                vector = direction * (centroids_range[i] / np.sqrt(direction.dot(direction)))

                new_point1 = centroid + vector
                new_point2 = centroid - vector

                logging.info("\t\tRunning secondary kmeans")
                points = data[self._model.labels == i]
                test_model = self._fit(2, points, np.asarray([new_point1, new_point2]))

                cluster1 = points[test_model.labels == 0]
                cluster2 = points[test_model.labels == 1]

                bic_parent = XMeans.bic([points], [centroid,])
                bic_child = XMeans.bic([cluster1, cluster2], test_model.centroids)
                logging.info("\t\tbic_parent = %f, bic_child = %f", bic_parent, bic_child)
                if bic_child > bic_parent:
                    logging.info("\tUsing children")
                    cluster_centers.append(test_model.centroids[0])
                    cluster_centers.append(test_model.centroids[1])
                else:
                    logging.info("\tUsing parent")
                    cluster_centers.append(centroid)
            if k == len(cluster_centers):
                logging.info("Done")
                break<commit_msg>bugfixes<commit_after>#!/usr/bin/env python2.7

import numpy as np
from mini_batch import MiniBatch
from sklearn.metrics import euclidean_distances
from collections import namedtuple

import logging

KMeansResult = namedtuple("KMeansResult", ("labels", "centroids"))

FORMAT = '[%(asctime)s] %(levelname)s - %(funcName)s:%(lineno)d: %(message)s'
logging.basicConfig(format=FORMAT, level=logging.FATAL)

class XMeans(object):
    def __init__(self, kmin, kmax=None, init='kmeans++', sample_percent = 0.20, minibatch_args=None, verbose = False):
        self.kmin = kmin
        self.kmax = kmax
        self.init = init
        self.sample_percent = sample_percent

        self.verbose = verbose
        if self.verbose:
            logging.getLogger().setLevel(logging.INFO)

        self._minibatch_args = minibatch_args or {}
        self.data_ = None
        self.cluster_centers_ = []

    def fit(self, data):
        data = self.data_ = np.asarray(data)
        k = self.kmin
        cluster_centers = self.init
        while self.kmax is None or k <= self.kmax:
            logging.info("Fitting with k=%d", k)
            self._model = self._fit(k, data, cluster_centers)
            
            centroid_distances = euclidean_distances(self._model.centroids, self._model.centroids)
            centroid_distances += np.diag([np.Infinity] * k)
            centroids_range = centroid_distances.min(axis=-1)

            cluster_centers = []
            for i, centroid in enumerate(self._model.centroids):
                logging.info("\tSplitting cluster %d / %d (k=%d)", 
                    i+1, 
                    len(self._model.centroids), 
                    len(cluster_centers),
                )
                direction = np.random.random(centroid.shape)
                vector = direction * (centroids_range[i] / np.sqrt(direction.dot(direction)))

                new_point1 = centroid + vector
                new_point2 = centroid - vector

                logging.info("\t\tRunning secondary kmeans")
                model_index = (self._model.labels == i)
                if not np.any(model_index):
                    logging.info("Disregarding cluster since it has no citizens")
                    continue
                points = data[model_index]
                test_model = self._fit(2, points, np.asarray([new_point1, new_point2]))

                cluster1 = points[test_model.labels == 0]
                cluster2 = points[test_model.labels == 1]

                bic_parent = XMeans.bic([points], [centroid,])
                bic_child = XMeans.bic([cluster1, cluster2], test_model.centroids)
                logging.info("\t\tbic_parent = %f, bic_child = %f", bic_parent, bic_child)
                if bic_child > bic_parent:
                    logging.info("\tUsing children")
                    cluster_centers.append(test_model.centroids[0])
                    cluster_centers.append(test_model.centroids[1])
                else:
                    logging.info("\tUsing parent")
                    cluster_centers.append(centroid)
            if k == len(cluster_centers):
                logging.info("Done")
                break
{'fraction_non_alphanumeric': 0.10784472957609226, 'fraction_numerical': 0.0064966704563911, 'mean_word_length': 1.9806389157792836, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1633359', 'n_tokens_mistral': 1885, 'n_tokens_neox': 1758, 'n_words': 345}