<commit_before>#!/usr/bin/python
#
# \file Eiger.py
# \author <NAME> <<EMAIL>>
# \date July 6, 2012
#
# \brief Command line interface into Eiger modeling framework
#
# \changes Added more plot functionality; <NAME>, SNL 5/2013
#

import argparse
import matplotlib.pyplot as plt
import sys
import numpy as np
import math
import tempfile
import shutil

from eiger import database, ClusterAnalysis, PCA, LinearRegression

def run(args):
    """
    Interface into Eiger model generation/polling/serialization/printing/etc.
    """
    
    print "Training model(s)..."
    training_DC = database.DataCollection(args['training_datacollection'], 
                                          db=args['db'], 
                                          user=args['user'], 
                                          passwd=args['<PASSWORD>'],
                                          host=args['host'])
    metric_ids = training_DC.metricIndexByType('deterministic', 
                                               'nondeterministic')
    metric_names = [training_DC.metrics[id][0] for id in metric_ids]
    training_profile = training_DC.profile[:,metric_ids]
    for idx,metric in enumerate(training_DC.metrics):
        if(metric[0] == args['performance_metric']):
            performance_metric_id = idx
    training_performance = training_DC.profile[:,performance_metric_id]

    #pca
    training_pca = PCA.PCA(training_profile)
    rotation_matrix = training_pca.reduced(args['pca_variance'], 
                                           args['pca_components'])[0]
    rotated_training_profile = np.dot(training_profile, rotation_matrix)

    #kmeans
    kmeans = ClusterAnalysis.KMeans(rotated_training_profile, k=args['clusters'])
    clusters = kmeans.kmeans()

    # reserve a vector for each model created per cluster
    models = [0] * len(clusters)

    with tempfile.TemporaryFile() as modelfile:
        for i,cluster in enumerate(clusters):
            cluster_profile = rotated_training_profile[cluster,:]
            cluster_performance = training_performance[cluster,:]
            regression = LinearRegression.LinearRegression(cluster_profile,
                                                           cluster_performance)
            pool = regression.powerLadder(cluster_profile.shape)
            (models[i], r_squared, attempts) = regression.select(pool, 
                                                    threshold=args['threshold'])
            
            # dump model to file
            modelfile.write('Model %s\n' % i)
            modelfile.write("[%s,%s](" % rotation_matrix.shape)
            for row in range(rotation_matrix.shape[0]):
                modelfile.write("(")
                for col in range(rotation_matrix.shape[1]):
                    modelfile.write("%s," % rotation_matrix[row,col])
                modelfile.write("),")
            modelfile.write(")\n")
            models[i].toFile(modelfile)
            for name in metric_names:
                modelfile.write("%s\n" % name)

            print "Finished modeling cluster %s: r squared = %s" % (i,r_squared)
       
        # if we want to save the model file, copy it now
        if(args['output'] is not None):
            shutil.copy(modelfile, args['output'])

    print "Visualizing..."
    if(args['plot_scree']):
        PCA.PlotScree(training_pca.loadings, log=False, 
                          title="PCA Scree Plot")
    if(args['plot_pcs_per_metric']):
        PCA.PlotPCsPerMetric(rotation_matrix, metric_names, 
                             title="PCs Per Metric")
    if(args['plot_metrics_per_pc']):
        PCA.PlotMetricsPerPC(rotation_matrix, metric_names, 
                             title="Metrics Per PC")

    if(args['test_fit']):
        _runExperiment(kmeans, models, rotation_matrix, training_DC, args)
    if(args['experiment_datacollection']):
        experiment_DC = database.DataCollection(args['experiment_datacollection'], 
                                                db=args['db'], 
                                                user=args['user'], 
                                                passwd=args['passwd'],
                                                host=args['host'])
        _runExperiment(kmeans, models, rotation_matrix, experiment_DC, args)

def _runExperiment(kmeans, models, rotation_matrix, 
                   experiment_DC, args):
    expr_metric_ids = experiment_DC.metricIndexByType('deterministic', 
                                                      'nondeterministic')
    expr_metric_names = [experiment_DC.metrics[id][0] for id in metric_ids]
    if expr_metric_names != metric_names:
        print "Training datacollection and experiment datacollection \
               do not have matching metrics. Aborting..."
        return
    for idx,metric in enumerate(experiment_DC.metrics):
        if(metric[0] == args['performance_metric']):
            performance_metric_id = idx
    performance = experiment_DC.profile[:,performance_metric_id]
    profile = experiment_DC.profile[:,expr_metric_ids]
    rotated_profile = np.dot(profile, rotation_matrix)
    
    cluster_membership = kmeans.closestCluster(rotated_profile)
    clusters = [[i for i,cluster in enumerate(cluster_membership) 
                 if cluster == cluster_id] for cluster_id in range(kmeans.k)]
<commit_msg>reorder printing of terminal output<commit_after>#!/usr/bin/python
#
# \file Eiger.py
# \author <NAME> <<EMAIL>>
# \date July 6, 2012
#
# \brief Command line interface into Eiger modeling framework
#
# \changes Added more plot functionality; <NAME>, SNL 5/2013
#

import argparse
import matplotlib.pyplot as plt
import sys
import numpy as np
import math
import tempfile
import shutil

from eiger import database, ClusterAnalysis, PCA, LinearRegression

def run(args):
    """
    Interface into Eiger model generation/polling/serialization/printing/etc.
    """
    print "Loading training data..."
    training_DC = database.DataCollection(args['training_datacollection'], 
                                          db=args['db'], 
                                          user=args['user'], 
                                          passwd=args['passwd'],
                                          host=args['host'])
    metric_ids = training_DC.metricIndexByType('deterministic', 
                                               'nondeterministic')
    metric_names = [training_DC.metrics[id][0] for id in metric_ids]
    try:
        training_profile = training_DC.profile[:,metric_ids]
    except IndexError:
        print "Unable to make model for empty data collection. Aborting..."
        return
    for idx,metric in enumerate(training_DC.metrics):
        if(metric[0] == args['performance_metric']):
            performance_metric_id = idx
    training_performance = training_DC.profile[:,performance_metric_id]

    #pca
    training_pca = PCA.PCA(training_profile)
    rotation_matrix = training_pca.reduced(args['pca_variance'], 
                                           args['pca_components'])[0]
    rotated_training_profile = np.dot(training_profile, rotation_matrix)

    print "Visualizing PCA..."
    if(args['plot_scree']):
        PCA.PlotScree(training_pca.loadings, log=False, 
                          title="PCA Scree Plot")
    if(args['plot_pcs_per_metric']):
        PCA.PlotPCsPerMetric(rotation_matrix, metric_names, 
                             title="PCs Per Metric")
    if(args['plot_metrics_per_pc']):
        PCA.PlotMetricsPerPC(rotation_matrix, metric_names, 
                             title="Metrics Per PC")
    #kmeans
    kmeans = ClusterAnalysis.KMeans(rotated_training_profile, k=args['clusters'])
    clusters = kmeans.kmeans()

    # reserve a vector for each model created per cluster
    models = [0] * len(clusters)

    print "Modeling..."
    with tempfile.TemporaryFile() as modelfile:
        for i,cluster in enumerate(clusters):
            cluster_profile = rotated_training_profile[cluster,:]
            cluster_performance = training_performance[cluster,:]
            regression = LinearRegression.LinearRegression(cluster_profile,
                                                           cluster_performance)
            pool = regression.powerLadder(cluster_profile.shape)
            (models[i], r_squared, attempts) = regression.select(pool, 
                                                    threshold=args['threshold'])
            
            # dump model to file
            modelfile.write('Model %s\n' % i)
            modelfile.write("[%s,%s](" % rotation_matrix.shape)
            for row in range(rotation_matrix.shape[0]):
                modelfile.write("(")
                for col in range(rotation_matrix.shape[1]):
                    modelfile.write("%s," % rotation_matrix[row,col])
                modelfile.write("),")
            modelfile.write(")\n")
            models[i].toFile(modelfile)
            for name in metric_names:
                modelfile.write("%s\n" % name)

            print "Finished modeling cluster %s: r squared = %s" % (i,r_squared)
       
        # if we want to save the model file, copy it now
        if(args['output'] is not None):
            shutil.copy(modelfile, args['output'])


    if(args['test_fit']):
        print "Testing fit..."
        _runExperiment(kmeans, models, rotation_matrix, training_DC, args)
    if(args['experiment_datacollection']):
        print "Running experiment..."
        _runExperiment(kmeans, models, rotation_matrix, training_DC, args)
        experiment_DC = database.DataCollection(args['experiment_datacollection'], 
                                                db=args['db'], 
                                                user=args['user'], 
                                                passwd=args['<PASSWORD>'],
                                                host=args['host'])
        _runExperiment(kmeans, models, rotation_matrix, experiment_DC, args)
    print "Done!"

def _runExperiment(kmeans, models, rotation_matrix, 
                   experiment_DC, args):
    expr_metric_ids = experiment_DC.metricIndexByType('deterministic', 
                                                      'nondeterministic')
    expr_metric_names = [experiment_DC.metrics[id][0] for id in metric_ids]
    if expr_metric_names != metric_names:
        print "Training datacollection and experiment datacollection \
               do not have matching metrics. Aborting..."
        return
    for idx,metric in enumerate(experiment_DC.metrics):
        if(metric[0] == args['performance_metric']):
            performance_metric_id = idx
    performance = experiment_DC.profile[:,performance_metric_id]
    profile = experiment_DC.profile[:,expr_metric_ids]
    rotated_profile = np.dot(profile, rotation_matrix)
    
    cluster_membership = kmeans.closestCluster(rotated_profile)
    clusters = [[i for i,cluster in enumerate(cluster_membership) 
                 if cluster == cluster_id] for cluster_id in range(kmeans.k)]

{'fraction_non_alphanumeric': 0.09891500904159133, 'fraction_numerical': 0.0032549728752260397, 'mean_word_length': 2.0844952593418853, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 52, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7079272', 'n_tokens_mistral': 3025, 'n_tokens_neox': 2854, 'n_words': 617}