<commit_before>"""
Author: <NAME>
Author: <NAME>
Date:   2014-04-28
"""

import sys
import logging
from time import time

from sklearn import metrics

from sklearn.cluster import MiniBatchKMeans # Fast large scale KMeans
from sklearn.cluster import Ward            # Ward's method

from vectorize_data import vectorize_data



# Perform KMeans {{{
def do_kmeans( data, labels ):
    """
    Do KMeans: perform KMeans clustering on an input corpus. Input is
    expected to be a dictionary of categories to tf-idf vectors.
    """

    # Construct a KMeans clustering machine
    km = MiniBatchKMeans(
        n_clusters=3,       # expected number of clusters
        init="k-means++",   # initialization method (smart)
        n_init=5,           # number of random retries
        #init_size=1000,
        batch_size=1000
    )

    logging.info("Beginning KMeans clustering.")

    t0 = time()
    km.fit(data)
    t1 = time()

    # Perform metrics
    runtime       = (t1 - t0)
    homogeneity   = metrics.homogeneity_score(labels, km.labels_)
    completeness  = metrics.completeness_score(labels, km.labels_)
    v_measure     = metrics.v_measure_score(labels, km.labels_)
    adjusted_rand = metrics.adjusted_rand_score(labels, km.labels_)

    # Output
    logging.info("done in       %fs"         % runtime)
    logging.info("Homogeneity:  %0.3f"       % homogeneity)
    logging.info("Completeness: %0.3f"       % completeness)
    logging.info("V-measure:    %0.3f"       % v_measure)
    logging.info("Adjusted Rand-Index: %.3f" % adjusted_rand)
# }}}



# TODO: Perform Affinity Propagation {{{
def do_affinity_propagation( data, labels ):
    """
    Do Affinity Propagation: perform Affinity Propagation clustering on an
    input corpus.  Input is expected to be a dictionary of categories to
    tf-idf vectors.
    """

    logging.info("Beginning Affinity Propagation clustering.")

    return
# }}}



# TODO: Perform Mean Shift {{{
def do_mean_shift( data, labels ):
    """
    Do Mean Shift: perform Mean Shift clustering on an input corpus.  Input
    is expected to be a dictionary of categories to tf-idf vectors.
    """

    logging.info("Beginning Mean Shift clustering.")

    return
# }}}



# TODO: Perform Spectral Clustering {{{
def do_spectral( data, labels ):
    """
    Do Spectral: perform Spectral clustering on an input corpus.  Input is
    expected to be a dictionary of categories to tf-idf vectors.
    """

    logging.info("Beginning Spectral clustering.")

    return
# }}}



# TODO: Perform Ward's Hierarchical Clustering {{{
def do_wards( data, labels ):
    """
    Do Ward's Hierarchical: perform Ward's Hierarchical clustering on an
    input corpus.  Input is expected to be a dictionary of categories to
    tf-idf vectors.
    """

    # Construct a Ward's clustering machine
    ward = Ward(
        n_clusters=2,
        connectivity=None,
        n_components=None,
        compute_full_tree='auto'
    )

    logging.info("Beginning Ward's Hierarhical clustering.")

    data = data.toarray()
    t0 = time()
    ward.fit(data)
    t1 = time()

    # Perform metrics
    runtime       = (t1 - t0)
    homogeneity   = metrics.homogeneity_score(labels, ward.labels_)
    completeness  = metrics.completeness_score(labels, ward.labels_)
    v_measure     = metrics.v_measure_score(labels, ward.labels_)
    adjusted_rand = metrics.adjusted_rand_score(labels, ward.labels_)

    # Output
    logging.info("done in       %fs"         % (t1 - t0))
    logging.info("Homogeneity:  %0.3f"       % homogeneity)
    logging.info("Completeness: %0.3f"       % completeness)
    logging.info("V-measure:    %0.3f"       % v_measure)
    logging.info("Adjusted Rand-Index: %.3f" % adjusted_rand)
# }}}



# TODO: Perform DBSCAN {{{
def do_dbscan( data, labels ):<commit_msg>Corrected number of expected clusters for ward's method<commit_after>"""
Author: <NAME>
Author: <NAME>
Date:   2014-04-28
"""

import sys
import logging
from time import time

from sklearn import metrics

from sklearn.cluster import MiniBatchKMeans # Fast large scale KMeans
from sklearn.cluster import Ward            # Ward's method

from vectorize_data import vectorize_data



# Perform KMeans {{{
def do_kmeans( data, labels ):
    """
    Do KMeans: perform KMeans clustering on an input corpus. Input is
    expected to be a dictionary of categories to tf-idf vectors.
    """

    # Construct a KMeans clustering machine
    km = MiniBatchKMeans(
        n_clusters=3,       # expected number of clusters
        init="k-means++",   # initialization method (smart)
        n_init=5,           # number of random retries
        #init_size=1000,
        batch_size=1000
    )

    logging.info("Beginning KMeans clustering.")

    t0 = time()
    km.fit(data)
    t1 = time()

    # Perform metrics
    runtime       = (t1 - t0)
    homogeneity   = metrics.homogeneity_score(labels, km.labels_)
    completeness  = metrics.completeness_score(labels, km.labels_)
    v_measure     = metrics.v_measure_score(labels, km.labels_)
    adjusted_rand = metrics.adjusted_rand_score(labels, km.labels_)

    # Output
    logging.info("done in       %fs"         % runtime)
    logging.info("Homogeneity:  %0.3f"       % homogeneity)
    logging.info("Completeness: %0.3f"       % completeness)
    logging.info("V-measure:    %0.3f"       % v_measure)
    logging.info("Adjusted Rand-Index: %.3f" % adjusted_rand)
# }}}



# TODO: Perform Affinity Propagation {{{
def do_affinity_propagation( data, labels ):
    """
    Do Affinity Propagation: perform Affinity Propagation clustering on an
    input corpus.  Input is expected to be a dictionary of categories to
    tf-idf vectors.
    """

    logging.info("Beginning Affinity Propagation clustering.")

    return
# }}}



# TODO: Perform Mean Shift {{{
def do_mean_shift( data, labels ):
    """
    Do Mean Shift: perform Mean Shift clustering on an input corpus.  Input
    is expected to be a dictionary of categories to tf-idf vectors.
    """

    logging.info("Beginning Mean Shift clustering.")

    return
# }}}



# TODO: Perform Spectral Clustering {{{
def do_spectral( data, labels ):
    """
    Do Spectral: perform Spectral clustering on an input corpus.  Input is
    expected to be a dictionary of categories to tf-idf vectors.
    """

    logging.info("Beginning Spectral clustering.")

    return
# }}}



# TODO: Perform Ward's Hierarchical Clustering {{{
def do_wards( data, labels ):
    """
    Do Ward's Hierarchical: perform Ward's Hierarchical clustering on an
    input corpus.  Input is expected to be a dictionary of categories to
    tf-idf vectors.
    """

    # Construct a Ward's clustering machine
    ward = Ward(
        n_clusters=3,               # expected number of clusters
        connectivity=None,          # no connectivity matrix
    )

    logging.info("Beginning Ward's Hierarhical clustering.")

    data = data.toarray()
    t0 = time()
    ward.fit(data)
    t1 = time()

    # Perform metrics
    runtime       = (t1 - t0)
    homogeneity   = metrics.homogeneity_score(labels, ward.labels_)
    completeness  = metrics.completeness_score(labels, ward.labels_)
    v_measure     = metrics.v_measure_score(labels, ward.labels_)
    adjusted_rand = metrics.adjusted_rand_score(labels, ward.labels_)

    # Output
    logging.info("done in       %fs"         % (t1 - t0))
    logging.info("Homogeneity:  %0.3f"       % homogeneity)
    logging.info("Completeness: %0.3f"       % completeness)
    logging.info("V-measure:    %0.3f"       % v_measure)
    logging.info("Adjusted Rand-Index: %.3f" % adjusted_rand)
# }}}



# TODO: Perform DBSCAN {{{
def do_dbscan( data, labels ):
{'fraction_non_alphanumeric': 0.10176531671858775, 'fraction_numerical': 0.011163032191069575, 'mean_word_length': 2.8181367690782952, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '754895', 'n_tokens_mistral': 2504, 'n_tokens_neox': 2316, 'n_words': 676}