<commit_before># utils for interfacing with Scikit-Learn
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
logger = logging.getLogger('testlogger')

# TODO: allow specification of cross-validation params at init time
def init_classifier(classifier_type, args=None):
    if args is not None:
        return classifier_type(*args)
    return classifier_type()

def train_classifier(X, y, classifier):
    classifier.fit(X,y)

def token_classifiers(token_contexts, token_tags, classifier_type, classifier_args=None):
    classifier_map = {}
    for token, contexts in token_contexts.items():
        logger.info('training classifier for token: {}'.format(token.encode('utf-8')))
        token_classifier = init_classifier(classifier_type, classifier_args)
        token_classifier.fit(contexts, token_tags[token])
        classifier_map[token] = token_classifier

    return classifier_map
<commit_msg>update learning utils<commit_after># utils for interfacing with Scikit-Learn
import logging
import numpy as np
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
logger = logging.getLogger('testlogger')


# TODO: allow specification of cross-validation params at init time
def init_classifier(classifier_type, args=None):
    if args is not None:
        return classifier_type(*args)
    return classifier_type()


def train_classifier(X, y, classifier):
    classifier.fit(X, y)


def map_classifiers(all_contexts, tags, classifier_type, data_type='plain', classifier_args=None):
    if data_type == 'plain':
        assert(type(all_contexts) == np.ndarray or type(all_contexts) == list)
        logger.info('training classifier')
        classifier = init_classifier(classifier_type, classifier_args)
        classifier.fit(all_contexts, tags)
        return classifier
    elif data_type == 'token':
        assert(type(all_contexts) == dict)
        classifier_map = {}
        for token, contexts in all_contexts.items():
            logger.info('training classifier for token: {}'.format(token.encode('utf-8')))
            token_classifier = init_classifier(classifier_type, classifier_args)
            token_classifier.fit(contexts, tags[token])
            classifier_map[token] = token_classifier
        return classifier_map


def predict_all(test_features, classifier_map, data_type='plain'):
    if data_type == 'plain':
        predictions = classifier_map.predict(test_features)
        return predictions
    elif data_type == 'token':
        test_predictions = {}
        for key, features in test_features.iteritems():
            try:
                classifier = classifier_map[key]
                predictions = classifier.predict(features)
                test_predictions[key] = predictions
            except KeyError as e:
                print(key + " - is NOT in the classifier map")
                raise
        return test_predictions

{'fraction_non_alphanumeric': 0.09777478084962914, 'fraction_numerical': 0.0006743088334457181, 'mean_word_length': 3.621495327102804, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6797088', 'n_tokens_mistral': 873, 'n_tokens_neox': 831, 'n_words': 201}