<commit_before>from __future__ import print_function
from __future__ import unicode_literals

import datetime
import inspect
import logging
import os.path
import re
import time
from operator import itemgetter

import numpy


def number_string(number, singular_unit, plural_unit, format_string="{} {}"):
    return format_string.format(number, singular_unit if number == 1 else plural_unit)


class Timed(object):
    """Decorator for timing how long a function takes"""

    def __init__(self, func):
        self.func = func

    def __call__(self, *args, **kwargs):
        start_time = time.time()
        retval = self.func(*args, **kwargs)
        elapsed = time.time() - start_time

        hours, seconds = divmod(elapsed, 60 * 60)
        minutes = seconds / 60.
        time_string = number_string(minutes, "minute", "minutes", format_string="{:.1f} {}")
        if hours:
            time_string = ", ".join((number_string(hours, "hour", "hours"), time_string))

        print("{} took {}".format(self.func.__name__, time_string))

        return retval


class DataSet(object):
    """Thin wrapper to bundle common data vars"""
    def __init__(self, inputs, outputs, splits, feature_names, target_names, output_index):
        self.inputs = inputs
        self.outputs = outputs
        self.splits = splits
        self.feature_names = feature_names
        self.target_names = target_names
        self.output_index = output_index

    def select_features(self, num_features, feature_scores, higher_is_better=True, verbose=0):
        pairs = enumerate(feature_scores)
        pairs = sorted(pairs, key=itemgetter(1), reverse=higher_is_better)

        if num_features < 1:
            num_features = int(num_features * self.inputs.shape[1])

        if verbose >= 1:
            print([(self.feature_names[i], score) for i, score in pairs])
            print("Dropping", [(self.feature_names[i]) for i, _ in pairs[num_features:]])
<commit_msg>Adding support to have an ensemble of cross validations<commit_after>from __future__ import print_function
from __future__ import unicode_literals

import datetime
import inspect
import logging
import os.path
import re
import time
from operator import itemgetter

import numpy


def number_string(number, singular_unit, plural_unit, format_string="{} {}"):
    return format_string.format(number, singular_unit if number == 1 else plural_unit)


class Timed(object):
    """Decorator for timing how long a function takes"""

    def __init__(self, func):
        self.func = func

    def __call__(self, *args, **kwargs):
        start_time = time.time()
        retval = self.func(*args, **kwargs)
        elapsed = time.time() - start_time

        hours, seconds = divmod(elapsed, 60 * 60)
        minutes = seconds / 60.
        time_string = number_string(minutes, "minute", "minutes", format_string="{:.1f} {}")
        if hours:
            time_string = ", ".join((number_string(hours, "hour", "hours"), time_string))

        print("{} took {}".format(self.func.__name__, time_string))

        return retval


class DataSet(object):
    """Thin wrapper to bundle common data vars"""
    def __init__(self, inputs, outputs, splits, feature_names, target_names, output_index, split_map=None):
        self.inputs = inputs
        self.outputs = outputs
        self.splits = splits
        self.feature_names = feature_names
        self.target_names = target_names
        self.output_index = output_index

        # mapping of labels to cross validation splits to support multi CV
        self.split_map = split_map

    def select_features(self, num_features, feature_scores, higher_is_better=True, verbose=0):
        pairs = enumerate(feature_scores)
        pairs = sorted(pairs, key=itemgetter(1), reverse=higher_is_better)

        if num_features < 1:
            num_features = int(num_features * self.inputs.shape[1])

        if verbose >= 1:
            print([(self.feature_names[i], score) for i, score in pairs])
            print("Dropping", [(self.feature_names[i]) for i, _ in pairs[num_features:]])

{'fraction_non_alphanumeric': 0.10648714810281518, 'fraction_numerical': 0.006364749082007344, 'mean_word_length': 3.251821019771072, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1936810', 'n_tokens_mistral': 1230, 'n_tokens_neox': 1156, 'n_words': 332}