<commit_before>#!/usr/bin/python

import cv2
import numpy as np

from sklearn.linear_model import Perceptron
from sklearn.linear_model import SGDClassifier
from sklearn import svm
import sklearn.neural_network as nn

import naoth.features as feat
from naoth.util import *

def unroll_responses(responses, class_n):
        sample_n = len(responses)
        new_responses = np.zeros(sample_n*class_n, np.int32)
        resp_idx = np.int32( responses + np.arange(sample_n)*class_n )
        new_responses[resp_idx] = 1
        return new_responses

def makeTrainData(X, labels, unroll=False):
  
  testFeat = feat.histo(np.zeros((12,12),dtype=np.float32))
  n_feat = testFeat.shape[1]
  
  samples = np.zeros((0, n_feat), dtype=np.float32)
  for s in X:
    img = s.reshape((12,12))
    f = feat.histo(img)
    samples = np.vstack([samples, f])
  
  if unroll:
    responses = np.float32(unroll_responses(labels, 2).reshape(-1, 2))
  else:
    responses = np.int32(labels)
    
  return n_feat, samples, responses
  
def learn(X, labels):
  
  n_feat, samples, responses = makeTrainData(X, labels, False)
   
  
  estimator = cv2.ml.SVM_create()
  estimator.setType(cv2.ml.SVM_C_SVC)
  estimator.setKernel(cv2.ml.SVM_RBF)
  ##estimator.setC(XXX)
  ##estimator.setGamma(XXX)
  
  trainData = cv2.ml.TrainData_create(samples=samples, 
    layout=cv2.ml.ROW_SAMPLE , responses=responses)
  
  estimator.train(trainData)
  
  return estimator
  
def classify(X, labels, estimator):  
  n_feat, samples, responses = makeTrainData(X, labels, False)
  
  classified = np.zeros(samples.shape[0], dtype=np.int)
  
  # classify
  for i in range(0, samples.shape[0]):
    ret, result = estimator.predict(np.asmatrix(samples[i,:]))
    classified[i] = result[0,0];
    
  return classified

def show_errors_asfeat(X, goldstd_response, actual_response):  
  
  image = np.zeros(((patch_size[1]+1)*show_size[1], (patch_size[0]+1)*show_size[0]))
  
  errorIdx = list()<commit_msg>much faster<commit_after>#!/usr/bin/python

import cv2
import numpy as np

from sklearn.linear_model import Perceptron
from sklearn.linear_model import SGDClassifier
from sklearn import svm
import sklearn.neural_network as nn

import naoth.features as feat
from naoth.util import *

def unroll_responses(responses, class_n):
        sample_n = len(responses)
        new_responses = np.zeros(sample_n*class_n, np.int32)
        resp_idx = np.int32( responses + np.arange(sample_n)*class_n )
        new_responses[resp_idx] = 1
        return new_responses

def makeTrainData(X, labels, unroll=False):
  
  testFeat = feat.histo(np.zeros((12,12),dtype=np.float32))
  n_feat = testFeat.shape[1]
  
  samples = np.zeros((X.shape[0], n_feat), dtype=np.float32)
  for s in X:
    img = s.reshape((12,12))
    f = feat.histo(img)
    samples = np.vstack([samples, f])
  
  if unroll:
    responses = np.float32(unroll_responses(labels, 2).reshape(-1, 2))
  else:
    responses = np.int32(labels)
    
  return n_feat, samples, responses
  
def learn(X, labels):
  
  n_feat, samples, responses = makeTrainData(X, labels, False)
      
  estimator = cv2.ml.SVM_create()
  estimator.setType(cv2.ml.SVM_C_SVC)
  estimator.setKernel(cv2.ml.SVM_CHI2)
  #estimator.setC(XXX)
  #estimator.setGamma(XXX)
  
  trainData = cv2.ml.TrainData_create(samples=samples, 
    layout=cv2.ml.ROW_SAMPLE , responses=responses)
  
  estimator.train(trainData)
  
  return estimator
  
def classify(X, labels, estimator):  
  n_feat, samples, responses = makeTrainData(X, labels, False)
  
  classified = np.zeros(samples.shape[0], dtype=np.int)
  
  # classify
  for i in range(0, samples.shape[0]):
    ret, result = estimator.predict(np.asmatrix(samples[i,:]))
    classified[i] = result[0,0];
    
  return classified

def show_errors_asfeat(X, goldstd_response, actual_response):  
  
  image = np.zeros(((patch_size[1]+1)*show_size[1], (patch_size[0]+1)*show_size[0]))
  
  errorIdx = list()
{'fraction_non_alphanumeric': 0.12385554425228891, 'fraction_numerical': 0.02212614445574771, 'mean_word_length': 3.9471698113207547, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4922336', 'n_tokens_mistral': 1528, 'n_tokens_neox': 1484, 'n_words': 278}