<commit_before># utils.py
# provides utilities for learning a neural network model
from __future__ import absolute_import, division, print_function
import time
import numpy as np
import functools
import operator

import theano
import theano.tensor as T
import lasagne
from lasagne import objectives

from lasagne import layers
from sklearn.cross_validation import StratifiedKFold
from sklearn.utils import shuffle
import cv2
import cPickle as pickle
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import ImageGrid
import matplotlib.cm as cm
from os.path import join
import utool as ut
import six
#from six.moves import range, zip


# NOTE: can use pygments instead
class ANSI(object):
    RED     = '\033[91m'
    GREEN   = '\033[92m'
    BLUE    = '\033[94m'
    CYAN    = '\033[96m'
    WHITE   = '\033[97m'
    YELLOW  = '\033[93m'
    MAGENTA = '\033[95m'
    GREY    = '\033[90m'
    BLACK   = '\033[90m'
    # DEFAULT = '\033[99m'
    RESET   = '\033[0m'


RANDOM_SEED = None
# RANDOM_SEED = 42


def _update(kwargs, key, value):
    #if key not in kwargs.keys():
    if key not in kwargs:
        kwargs[key] = value


def testdata_imglist():
    import vtool as vt
    x = 32
    width = 32
    height = 32
    channels = 3
    img0 = np.arange(x ** 2 * 3, dtype=np.uint8).reshape(x, x, 3)
    img1 = vt.imread(ut.grab_test_imgpath('jeff.png'))
    img2 = vt.imread(ut.grab_test_imgpath('carl.jpg'))
    img3 = vt.imread(ut.grab_test_imgpath('lena.png'))
    img_list = [vt.padded_resize(img, (width, height)) for img in [img0, img1, img2, img3]]
    return img_list, width, height, channels


def convert_imagelist_to_data(img_list):
    """
    Converts a list of cv2-style images into a single numpy array of nonflat
    theano-style images.

    h=height, w=width, b=batchid, c=channel

    Args:
        img_list (list of ndarrays): a list of numpy arrays with shape [h, w, c]

    Returns:
        data: in the shape [b, (c x h x w)]

    CommandLine:
        python -m ibeis_cnn.utils --test-convert_imagelist_to_data

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis_cnn.utils import *  # NOQA
        >>> import vtool as vt
        >>> # build test data
        >>> # execute function
        >>> img_list, width, height, channels = testdata_imglist()
        >>> data = convert_imagelist_to_data(img_list)
        >>> data[0].reshape(3, 32, 32)[:, 0:2, 0:2]
        >>> subset = (data[0].reshape(3, 32, 32)[:, 0:2, 0:2])
        >>> #result = str(np.transpose(subset, (1, 2, 0)))
        >>> result = str(subset).replace('\n', '')
        >>> print(result)
        [[[  0   3]  [ 96  99]] [[  1   4]  [ 97 100]] [[  2   5]  [ 98 101]]]
    """
    #[img.shape for img in img_list]
    # format to [b, c, h, w]
    shape_list = [img.shape for img in img_list]
    assert ut.list_allsame(shape_list)
    theano_style_imgs = [np.transpose(img, (2, 0, 1))[None, :] for img in img_list]
    data = np.vstack(theano_style_imgs)
    #data = np.vstack([img[None, :] for img in img_list])
    return data


def convert_data_to_imglist(data, width, height, channels):
    r"""
    Args:
        data (ndarray): in the shape [b, (c x h x w)]
        width (?):
        height (?):
        channels (?):

    Returns:
        img_list (list of ndarrays): a list of numpy arrays with shape [h, w, c]

    CommandLine:
        python -m ibeis_cnn.utils --test-convert_data_to_imglist

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis_cnn.utils import *  # NOQA
        >>> # build test data
        >>> img_list, width, height, channels = testdata_imglist()
        >>> data = convert_imagelist_to_data(img_list)
        >>> img_list2 = convert_data_to_imglist(data, width, height, channels)
        >>> assert np.all(img_list == img_list2)
    """
    #num_imgs = data.shape[0]
    #newshape = (num_imgs, channels, width, height)
    #data_ = data.reshape(newshape)
    img_list = np.transpose(data, (0, 2, 3, 1))
    return img_list


def get_current_time():
    return time.strftime('%Y-%m-%d %H:%M:%S')


def train_test_split(X, y, eval_size):
    # take the data and label arrays, split them preserving the class distributions
    kf = StratifiedKFold(y, round(1. / eval_size))

    train_indices, valid_indices = six.next(iter(kf))
    X_train, y_train = X[train_indices], y[train_indices]
    X_valid, y_valid = X[valid_indices], y[valid_indices]

    return X_train, y_train, X_valid, y_valid


def load(data_file, labels_file=None, random_state=None):
    # Load X matrix (data)
    data = np.load(data_file, mmap_mode='r')
    # Load y vector (labels)
    labels = None
    if labels_file is not None:
        labels = np.load(labels_file, mmap_mode='r')
    # Return data
    return data, labels


def print_header_columns():
    print('''
[info]   Epoch |  Train Loss  |  Valid Loss  |  Train / Val  |  Valid Acc  |  Test Acc   |  Dur
[info] --------|--------------|--------------|---------------|-------------|-------------|------\
''')


def print_layer_info(output_layer):
    nn_layers = layers.get_all_layers(output_layer)
    print('\n[info] Network Structure:')
    for index, layer in enumerate(nn_layers):
        output_shape = layer.get_output_shape()
        print('[info]     {:>4}{:<18}\t{:<20}\tproduces {:>7,} outputs'.format(
            index,
            layer.__class__.__name__,
            str(output_shape),
            int(str(functools.reduce(operator.mul, output_shape[1:]))),
        ))
    print('[info] ...this model has {:,} learnable parameters\n'.format(
        layers.count_params(output_layer)
    ))


def print_epoch_info(train_loss, valid_loss, valid_accuracy, test_accuracy,
                     epoch, duration, best_train_loss, best_valid_loss,
                     best_valid_accuracy, best_test_accuracy, **kwargs):
    best_train      = train_loss == best_train_loss
    best_valid      = valid_loss == best_valid_loss
    best_valid_accuracy = valid_accuracy == best_valid_accuracy
    best_train_accuracy = test_accuracy == best_test_accuracy
    ratio           = train_loss / valid_loss
    unhealthy_ratio = ratio <= 0.5 or 2.0 <= ratio
    print('[info]  {:>5}  |  {}{:>10.6f}{}  |  {}{:>10.6f}{}  '
          '|  {}{:>11.6f}{}  |  {}{:>9}{}  |  {}{:>9}{}  |  {:>3.1f}s'.format(
              epoch,
              ANSI.BLUE if best_train else '',
              train_loss,
              ANSI.RESET if best_train else '',<commit_msg>Show convolutions<commit_after># utils.py
# provides utilities for learning a neural network model
from __future__ import absolute_import, division, print_function
import time
import numpy as np
import functools
import operator

import theano
import theano.tensor as T
import lasagne
from lasagne import objectives

from lasagne import layers
from sklearn.cross_validation import StratifiedKFold
from sklearn.utils import shuffle
import cv2
import cPickle as pickle
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import ImageGrid
import matplotlib.cm as cm
from os.path import join
import utool as ut
import six
#from six.moves import range, zip


# NOTE: can use pygments instead
class ANSI(object):
    RED     = '\033[91m'
    GREEN   = '\033[92m'
    BLUE    = '\033[94m'
    CYAN    = '\033[96m'
    WHITE   = '\033[97m'
    YELLOW  = '\033[93m'
    MAGENTA = '\033[95m'
    GREY    = '\033[90m'
    BLACK   = '\033[90m'
    # DEFAULT = '\033[99m'
    RESET   = '\033[0m'


RANDOM_SEED = None
# RANDOM_SEED = 42


def _update(kwargs, key, value):
    #if key not in kwargs.keys():
    if key not in kwargs:
        kwargs[key] = value


def testdata_imglist():
    import vtool as vt
    x = 32
    width = 32
    height = 32
    channels = 3
    img0 = np.arange(x ** 2 * 3, dtype=np.uint8).reshape(x, x, 3)
    img1 = vt.imread(ut.grab_test_imgpath('jeff.png'))
    img2 = vt.imread(ut.grab_test_imgpath('carl.jpg'))
    img3 = vt.imread(ut.grab_test_imgpath('lena.png'))
    img_list = [vt.padded_resize(img, (width, height)) for img in [img0, img1, img2, img3]]
    return img_list, width, height, channels


def convert_imagelist_to_data(img_list):
    """
    Converts a list of cv2-style images into a single numpy array of nonflat
    theano-style images.

    h=height, w=width, b=batchid, c=channel

    Args:
        img_list (list of ndarrays): a list of numpy arrays with shape [h, w, c]

    Returns:
        data: in the shape [b, (c x h x w)]

    CommandLine:
        python -m ibeis_cnn.utils --test-convert_imagelist_to_data

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis_cnn.utils import *  # NOQA
        >>> import vtool as vt
        >>> # build test data
        >>> # execute function
        >>> img_list, width, height, channels = testdata_imglist()
        >>> data = convert_imagelist_to_data(img_list)
        >>> data[0].reshape(3, 32, 32)[:, 0:2, 0:2]
        >>> subset = (data[0].reshape(3, 32, 32)[:, 0:2, 0:2])
        >>> #result = str(np.transpose(subset, (1, 2, 0)))
        >>> result = str(subset).replace('\n', '')
        >>> print(result)
        [[[  0   3]  [ 96  99]] [[  1   4]  [ 97 100]] [[  2   5]  [ 98 101]]]
    """
    #[img.shape for img in img_list]
    # format to [b, c, h, w]
    shape_list = [img.shape for img in img_list]
    assert ut.list_allsame(shape_list)
    theano_style_imgs = [np.transpose(img, (2, 0, 1))[None, :] for img in img_list]
    data = np.vstack(theano_style_imgs)
    #data = np.vstack([img[None, :] for img in img_list])
    return data


def convert_data_to_imglist(data, width, height, channels):
    r"""
    Args:
        data (ndarray): in the shape [b, (c x h x w)]
        width (?):
        height (?):
        channels (?):

    Returns:
        img_list (list of ndarrays): a list of numpy arrays with shape [h, w, c]

    CommandLine:
        python -m ibeis_cnn.utils --test-convert_data_to_imglist

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis_cnn.utils import *  # NOQA
        >>> # build test data
        >>> img_list, width, height, channels = testdata_imglist()
        >>> data = convert_imagelist_to_data(img_list)
        >>> img_list2 = convert_data_to_imglist(data, width, height, channels)
        >>> assert np.all(img_list == img_list2)
    """
    #num_imgs = data.shape[0]
    #newshape = (num_imgs, channels, width, height)
    #data_ = data.reshape(newshape)
    img_list = np.transpose(data, (0, 2, 3, 1))
    return img_list


def get_current_time():
    return time.strftime('%Y-%m-%d %H:%M:%S')


def train_test_split(X, y, eval_size):
    # take the data and label arrays, split them preserving the class distributions
    kf = StratifiedKFold(y, round(1. / eval_size))

    train_indices, valid_indices = six.next(iter(kf))
    X_train, y_train = X[train_indices], y[train_indices]
    X_valid, y_valid = X[valid_indices], y[valid_indices]

    return X_train, y_train, X_valid, y_valid


def load(data_file, labels_file=None, random_state=None):
    # Load X matrix (data)
    data = np.load(data_file, mmap_mode='r')
    # Load y vector (labels)
    labels = None
    if labels_file is not None:
        labels = np.load(labels_file, mmap_mode='r')
    # Return data
    return data, labels


def print_header_columns():
    print('''
[info]   Epoch |  Train Loss  |  Valid Loss  |  Train / Val  |  Valid Acc  |  Test Acc   |  Dur
[info] --------|--------------|--------------|---------------|-------------|-------------|------\
''')


def print_layer_info(output_layer):
    nn_layers = layers.get_all_layers(output_layer)
    print('\n[info] Network Structure:')
    for index, layer in enumerate(nn_layers):
        output_shape = layer.get_output_shape()
        print('[info]  {:>3}  {:<18}\t{:<20}\tproduces {:>7,} outputs'.format(
            index,
            layer.__class__.__name__,
            str(output_shape),
            int(str(functools.reduce(operator.mul, output_shape[1:]))),
        ))
    print('[info] ...this model has {:,} learnable parameters\n'.format(
        layers.count_params(output_layer)
    ))


def print_epoch_info(train_loss, valid_loss, valid_accuracy, test_accuracy,
                     epoch, duration, best_train_loss, best_valid_loss,
                     best_valid_accuracy, best_test_accuracy, **kwargs):
    best_train      = train_loss == best_train_loss
    best_valid      = valid_loss == best_valid_loss
    best_valid_accuracy = valid_accuracy == best_valid_accuracy
    best_train_accuracy = test_accuracy == best_test_accuracy
    ratio           = train_loss / valid_loss
    unhealthy_ratio = ratio <= 0.5 or 2.0 <= ratio
    print('[info]  {:>5}  |  {}{:>10.6f}{}  |  {}{:>10.6f}{}  '
          '|  {}{:>11.6f}{}  |  {}{:>9}{}  |  {}{:>9}{}  |  {:>3.1f}s'.format(
              epoch,
              ANSI.BLUE if best_train else '',
              train_loss,
              ANSI.RESET if best_train else '',
{'fraction_non_alphanumeric': 0.1410286246218292, 'fraction_numerical': 0.024358079280117913, 'mean_word_length': 2.6510903426791277, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 135, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '66753', 'n_tokens_mistral': 4739, 'n_tokens_neox': 4484, 'n_words': 1140}