<commit_before>#!/usr/bin/env python

"""Functions related to machine learning"""

import logging
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
import os
import argparse
from itertools import izip
from collections import Counter
from time import time
import tempfile

import numpy as np
import sklearn
import matplotlib as plt
from sklearn.externals import joblib
from sklearn.metrics import confusion_matrix

import feature_extraction
import util



def default_classifier():
    dot = os.path.dirname(os.path.realpath(__file__))
    DEFAULT = os.path.join(dot, 'data/default_svm.p')
    return load_classifier(DEFAULT)

def confusion_str(cm, labels, hide_zeroes=False, hide_diagonal=False, hide_threshold=None):
    """pretty print for confusion matrices"""
    ret_str = ''
    columnwidth = max([len(x) for x in labels]+[5])  # 5 is value length
    empty_cell = ' ' * columnwidth
    # Print header
    ret_str += '    ' + empty_cell
    for label in labels:
        ret_str += '%{0}s'.format(columnwidth) % label
    ret_str += '\n'
    # Print rows
    for i, label1 in enumerate(labels):
        ret_str += '    %{0}s'.format(columnwidth) % label1
        for j in range(len(labels)):
            cell = "%{0}.1f".format(columnwidth) % cm[i, j]
            if hide_zeroes:
                cell = cell if float(cm[i, j]) != 0 else empty_cell
            if hide_diagonal:
                cell = cell if i != j else empty_cell
            if hide_threshold:
                cell = cell if cm[i, j] > hide_threshold else empty_cell
            ret_str += cell
        ret_str += '\n'
    return ret_str


def plot_confusion_matrix(cm, labels, title='Confusion matrix', cmap=plt.cm.Blues, save=False):
    plt.figure()
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(labels))
    plt.xticks(tick_marks, labels, rotation=45)
    plt.yticks(tick_marks, labels)
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.show()
    if save:
        plt.savefig(save)


def confusion(actual, predicted, plot=False, disp=False):
    confusion_labels = sorted(set(np.concatenate([actual, predicted])))
    confusion = confusion_matrix(actual, predicted, confusion_labels)
    if disp:
        print(confusion_str(confusion, confusion_labels))
    if plot:
        plot_confusion_matrix(confusion, confusion_labels)
    return confusion


def flatten(l):
    return [item for sublist in l for item in sublist]


def shape_labels(labels):
    Y = np.array(flatten(labels))
    return Y


def shape_features(tracks, feature_names):
    logging.debug('Shaping features/labels to sklearn format')
    features = []
    labels = []
    for track in tracks:
        feature = [track[f] for f in feature_names]
        feature = np.vstack(feature)
        features.append(feature)
        labels.append([track['label']] * feature.shape[1])

    X = np.concatenate([feature.T for feature in features])
    Y = np.array(flatten(labels))
    logging.debug('X: {}'.format(X.shape))
    logging.debug('Y: {}'.format(Y.shape))
    return X, Y


def train_tracks(tracks, feature_names):
    X, Y = shape_features(tracks, feature_names)
    clf = train(X, Y)
    return clf


def train(X, Y):
    #X = sklearn.preprocessing.normalize(X, norm='l1', axis=1, copy=True)
    logging.info('Training...')
    clf = classifier()
    clf.fit(X, Y)
    return clf


def predict_file(path, remove_silence=True):
    with tempfile.NamedTemporaryFile(suffix=os.path.splitext(path)[1]) as f:
        path = util.remove_silence(path, f.name, block=True)
        clf, feature_names = default_classifier()
        track = {
            'file_path': path,
            'label': 'unknown',
        }
        args = type('', (), {})
        args.n_fft = 4096
        args.average = 1
        args.normalize = True
        args.delta = True
        args.delta_delta = False
        args.save_features = False
        feature_extraction.set_features([track, ], args)

        prediction, predictions = predict_track(clf, track, feature_names)
    return prediction, predictions


def predict_track(clf, track, feature_names):
    predicted = predict_tracks(clf, [track, ], feature_names)
    track['sample_predictions'] = predicted
    prediction, predictions = util.most_common(predicted)
    track['prediction'] = prediction
    track['predictions'] = prediction
    return prediction, predictions


def predict_tracks(clf, tracks, feature_names):
    X, Y = shape_features(tracks, feature_names)
    return predict(X, clf)


def predict_features(features, clf):
    X = np.concatenate([feature.T for feature in features])
    return predict(X, clf)


def predict(X, clf):
    #X = sklearn.preprocessing.normalize(X, norm='l1', axis=1, copy=True)<commit_msg>Add logging to predict_file<commit_after>#!/usr/bin/env python

"""Functions related to machine learning"""

import logging
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
import os
import argparse
from itertools import izip
from collections import Counter
from time import time
import tempfile

import numpy as np
import sklearn
import matplotlib as plt
from sklearn.externals import joblib
from sklearn.metrics import confusion_matrix

import feature_extraction
import util



def default_classifier():
    dot = os.path.dirname(os.path.realpath(__file__))
    DEFAULT = os.path.join(dot, 'data/default_svm.p')
    return load_classifier(DEFAULT)

def confusion_str(cm, labels, hide_zeroes=False, hide_diagonal=False, hide_threshold=None):
    """pretty print for confusion matrices"""
    ret_str = ''
    columnwidth = max([len(x) for x in labels]+[5])  # 5 is value length
    empty_cell = ' ' * columnwidth
    # Print header
    ret_str += '    ' + empty_cell
    for label in labels:
        ret_str += '%{0}s'.format(columnwidth) % label
    ret_str += '\n'
    # Print rows
    for i, label1 in enumerate(labels):
        ret_str += '    %{0}s'.format(columnwidth) % label1
        for j in range(len(labels)):
            cell = "%{0}.1f".format(columnwidth) % cm[i, j]
            if hide_zeroes:
                cell = cell if float(cm[i, j]) != 0 else empty_cell
            if hide_diagonal:
                cell = cell if i != j else empty_cell
            if hide_threshold:
                cell = cell if cm[i, j] > hide_threshold else empty_cell
            ret_str += cell
        ret_str += '\n'
    return ret_str


def plot_confusion_matrix(cm, labels, title='Confusion matrix', cmap=plt.cm.Blues, save=False):
    plt.figure()
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(labels))
    plt.xticks(tick_marks, labels, rotation=45)
    plt.yticks(tick_marks, labels)
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.show()
    if save:
        plt.savefig(save)


def confusion(actual, predicted, plot=False, disp=False):
    confusion_labels = sorted(set(np.concatenate([actual, predicted])))
    confusion = confusion_matrix(actual, predicted, confusion_labels)
    if disp:
        print(confusion_str(confusion, confusion_labels))
    if plot:
        plot_confusion_matrix(confusion, confusion_labels)
    return confusion


def flatten(l):
    return [item for sublist in l for item in sublist]


def shape_labels(labels):
    Y = np.array(flatten(labels))
    return Y


def shape_features(tracks, feature_names):
    logging.debug('Shaping features/labels to sklearn format')
    features = []
    labels = []
    for track in tracks:
        feature = [track[f] for f in feature_names]
        feature = np.vstack(feature)
        features.append(feature)
        labels.append([track['label']] * feature.shape[1])

    X = np.concatenate([feature.T for feature in features])
    Y = np.array(flatten(labels))
    logging.debug('X: {}'.format(X.shape))
    logging.debug('Y: {}'.format(Y.shape))
    return X, Y


def train_tracks(tracks, feature_names):
    X, Y = shape_features(tracks, feature_names)
    clf = train(X, Y)
    return clf


def train(X, Y):
    #X = sklearn.preprocessing.normalize(X, norm='l1', axis=1, copy=True)
    logging.info('Training...')
    clf = classifier()
    clf.fit(X, Y)
    return clf


def predict_file(path, remove_silence=True):
    with tempfile.NamedTemporaryFile(suffix=os.path.splitext(path)[1]) as f:
        path = util.remove_silence(path, f.name, block=True)
        clf, feature_names = default_classifier()
        track = {
            'file_path': path,
            'label': 'unknown',
        }
        args = type('', (), {})
        args.n_fft = 4096
        args.average = 1
        args.normalize = True
        args.delta = True
        args.delta_delta = False
        args.save_features = False
        feature_extraction.set_features([track, ], args)

        prediction, predictions = predict_track(clf, track, feature_names)
    logging.info(predictions)
    return prediction, predictions


def predict_track(clf, track, feature_names):
    predicted = predict_tracks(clf, [track, ], feature_names)
    track['sample_predictions'] = predicted
    prediction, predictions = util.most_common(predicted)
    track['prediction'] = prediction
    track['predictions'] = prediction
    return prediction, predictions


def predict_tracks(clf, tracks, feature_names):
    X, Y = shape_features(tracks, feature_names)
    return predict(X, clf)


def predict_features(features, clf):
    X = np.concatenate([feature.T for feature in features])
    return predict(X, clf)


def predict(X, clf):
    #X = sklearn.preprocessing.normalize(X, norm='l1', axis=1, copy=True)
{'fraction_non_alphanumeric': 0.11758084322554237, 'fraction_numerical': 0.0045026606631191155, 'mean_word_length': 3.227076124567474, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7617057', 'n_tokens_mistral': 3129, 'n_tokens_neox': 3035, 'n_words': 765}