<commit_before># -*- coding: UTF-8 -*-
__author__="<NAME>"
__license__="MIT License"

#
# This module defines a number of helper functions. 
#

from datetime import datetime
import logging
import numpy as np
import os
import sys


def short_name(model):
    """Return a simplified name for this model. A bit brittle."""
    # *TODO* make this method more general 
    # for a single model, this will work
    name = model.__class__.__name__
    try:
        if hasattr(model, 'steps'):
            # pipeline
            name = '-'.join( [ pair[0] for pair in model.steps ] )
        elif hasattr(model, 'best_estimator_'):
            if hasattr(model.estimator, 'steps'):
                # gridsearchcv
                name = 'gscv_' + '-'.join( [x[0] for x in model.estimator.steps ])
            elif hasattr(model.estimator, 'estimators'):
                # votingclassifier
                name = 'gscv_vc_' + '-'.join( [x[0] for x in model.estimator.estimators ])
        elif hasattr(model, 'base_estimator_'):
            # bagging
            name = 'bag_' + short_name(model.base_estimator)
    except AttributeError, e:
        logging.info('utils.short_name() couldnt generate quality name')
        # for a single model, this will work
        name = model.__class__.__name__
        logging.info('falling back to generic name={}'.format(name))
    return name 


def create_submission(predictions, sub_name, comment=None, team='DrJ'):
    """Include the specified array of image predictions in a 
    properly-formatted submission file.
    """
    now = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S')
    submission_name = '-'.join(sub_name.split())
    with open('submissions/{}_{}.submission'.format(
            now, submission_name), 'w') as f:

        f.write('#'*20 + ' Generated submission file\n')
        if comment is not None:
            f.write('# ' + comment + '\n')
        f.write('{}\n'.format(team))
        f.write('{}\n'.format(now))
        f.write('{}\n'.format(sub_name))
        for p in predictions:
            f.write('{}\n'.format(p))
    return True


def load_np_arrays():
    """Return numpy arrays for training dataset, training labels, 
    and test dataset (in that order). 
    """
    # nb: path assumes that we call this function from project root  
    #   - may not be appropriate for importing into code in bin/ ?
    X_train = np.load(os.path.join('data', 'train-images.npy'))
    y_train = np.load(os.path.join('data', 'train-labels.npy'))
    X_test = np.load(os.path.join('data', 'test-images.npy'))

    return (X_train, y_train, X_test)

<commit_msg>Include flag for original/expanded data in load_arrays() helper<commit_after># -*- coding: UTF-8 -*-
__author__="<NAME>"
__license__="MIT License"

#
# This module defines a number of helper functions. 
#

from datetime import datetime
import logging
import numpy as np
import os
import sys


def short_name(model):
    """Return a simplified name for this model. A bit brittle."""
    # *TODO* make this method more general 
    # for a single model, this will work
    name = model.__class__.__name__
    try:
        if hasattr(model, 'steps'):
            # pipeline
            name = '-'.join( [ pair[0] for pair in model.steps ] )
        elif hasattr(model, 'best_estimator_'):
            if hasattr(model.estimator, 'steps'):
                # gridsearchcv
                name = 'gscv_' + '-'.join( [x[0] for x in model.estimator.steps ])
            elif hasattr(model.estimator, 'estimators'):
                # votingclassifier
                name = 'gscv_vc_' + '-'.join( [x[0] for x in model.estimator.estimators ])
        elif hasattr(model, 'base_estimator_'):
            # bagging
            name = 'bag_' + short_name(model.base_estimator)
    except AttributeError, e:
        logging.info('utils.short_name() couldnt generate quality name')
        # for a single model, this will work
        name = model.__class__.__name__
        logging.info('falling back to generic name={}'.format(name))
    return name 


def create_submission(predictions, sub_name, comment=None, team='DrJ'):
    """Include the specified array of image predictions in a 
    properly-formatted submission file.
    """
    now = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S')
    submission_name = '-'.join(sub_name.split())
    with open('submissions/{}_{}.submission'.format(
            now, submission_name), 'w') as f:

        f.write('#'*20 + ' Generated submission file\n')
        if comment is not None:
            f.write('# ' + comment + '\n')
        f.write('{}\n'.format(team))
        f.write('{}\n'.format(now))
        f.write('{}\n'.format(sub_name))
        for p in predictions:
            f.write('{}\n'.format(p))
    return True


def load_np_arrays(files='original'):
    """
    Return numpy arrays for training dataset, training labels, 
    and test dataset (in that order). If files='original', 
    return the image data, as given. If files='expanded', 
    return the perturbed image files (~5 times larger). 
    
    files='original', 'expanded' 
    """
    # nb: path assumes that we call this function from project root  
    train_imgs_f = 'train-images.npy'
    train_labels_f = 'train-labels.npy'

    if files == 'expanded':
        train_imgs_f = 'expanded-' + train_imgs_f 
        train_labels_f = 'expanded-' + train_labels_f 

    X_train = np.load(os.path.join('data', train_imgs_f))
    y_train = np.load(os.path.join('data', train_labels_f))
    X_test = np.load(os.path.join('data', 'test-images.npy'))

    return (X_train, y_train, X_test)


{'fraction_non_alphanumeric': 0.13488952245188882, 'fraction_numerical': 0.002316464718460442, 'mean_word_length': 2.6142949130714745, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5636487', 'n_tokens_mistral': 1780, 'n_tokens_neox': 1722, 'n_words': 480}