<commit_before>"""
======================================================
Corss validation on the data using multiple processes.
======================================================

"""

from svmutil import *
import commands
import numpy
import random
import multiprocessing
import sys

class result():
    # internal class passed to by sub process
    def __init__(self, i, n):
        self.fp_ind = i # fingerprint index 
        self.pred_fp_ = numpy.zeros(n) # predicted fingerprints
        self.acc = 0 # cross validation accuracy
def internalCV_mp(kernel, labels, n_folds, select_c=False, n_p=8, prob=False):    
    """
    Internel cross validation using train data.

    Parameters:
    -----------
    kernel, numpy 2d array, n_train*n_train, training kernel

    labels, numpy 2d array, n_train*n_fingerprints, training labels

    n_folds, number of cross validations

    pred_f, string, file to store predicted fingerprints in the CV

    select_c, bool, whether doing C selection in CV

    n_p, int, number of processes to use

    prob, boolean, probability output if prob=True.

    Returns:
    --------

    pred_fp: numpy 2d array, cross validation predictions or probability of
             positive label if prob=True.

    Note:
    -----
    Wrtie the cross validation predition fingerprints in pred_f
    """
    
    (n_x, n_x) = kernel.shape
    (n_x, n_y) = labels.shape
    x = kernel
    if n_y < n_p:
        n_p = n_y

    #cv_accs = numpy.zeros(n_y)
    pred_fp = numpy.zeros((n_x, n_y))

    #tags = _label_folds(n_x, n_folds)
    result_queue = multiprocessing.Queue(n_y)

    task_dict = {}
    for i in range(n_y):
        task_dict[i%n_p] = []
    for i in range(n_y):
        task_dict[i%n_p].append(i)
<commit_msg>use tags<commit_after>"""
======================================================
Corss validation on the data using multiple processes.
======================================================

"""

from svmutil import *
import commands
import numpy
import random
import multiprocessing
import sys

class result():
    # internal class passed to by sub process
    def __init__(self, i, n):
        self.fp_ind = i # fingerprint index 
        self.pred_fp_ = numpy.zeros(n) # predicted fingerprints
        self.acc = 0 # cross validation accuracy
def internalCV_mp(kernel, labels, n_folds, select_c=False, n_p=8, prob=False):    
    """
    Internel cross validation using train data.

    Parameters:
    -----------
    kernel, numpy 2d array, n_train*n_train, training kernel

    labels, numpy 2d array, n_train*n_fingerprints, training labels

    n_folds, number of cross validations

    pred_f, string, file to store predicted fingerprints in the CV

    select_c, bool, whether doing C selection in CV

    n_p, int, number of processes to use

    prob, boolean, probability output if prob=True.

    Returns:
    --------

    pred_fp: numpy 2d array, cross validation predictions or probability of
             positive label if prob=True.

    Note:
    -----
    Wrtie the cross validation predition fingerprints in pred_f
    """
    
    (n_x, n_x) = kernel.shape
    (n_x, n_y) = labels.shape
    x = kernel
    if n_y < n_p:
        n_p = n_y

    #cv_accs = numpy.zeros(n_y)
    pred_fp = numpy.zeros((n_x, n_y))

    tags = _label_folds(n_x, n_folds)
    result_queue = multiprocessing.Queue(n_y)

    task_dict = {}
    for i in range(n_y):
        task_dict[i%n_p] = []
    for i in range(n_y):
        task_dict[i%n_p].append(i)

{'fraction_non_alphanumeric': 0.15005694760820046, 'fraction_numerical': 0.0028473804100227792, 'mean_word_length': 2.9120267260579062, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3286743', 'n_tokens_mistral': 1125, 'n_tokens_neox': 1066, 'n_words': 341}