<commit_before>"""Utilities for maniuplating, analyzing and plotting `pybrain` `Network` and `DataSet` objects

TODO:
    Incorporate into pybrain fork so pug doesn't have to depend on pybrain

"""
from __future__ import print_function
import os

import pandas as pd
from scipy import ndarray, reshape  # array, amin, amax, 
np = pd.np
from matplotlib import pyplot as plt
import pybrain.datasets
import pybrain.structure
import pybrain.supervised
import pybrain.tools
pb = pybrain
# from pybrain.supervised.trainers import Trainer
from pybrain.tools.customxml import NetworkReader
from pybrain.structure.parametercontainer import ParameterContainer
from pybrain.structure.connections.connection import Connection


#import pug.nlp.util as nlp

# print(os.path.realpath(__file__))
DATA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'data')


def build_ann(N_input=None, N_hidden=2, N_output=1, hidden_layer_type=pb.structure.LinearLayer, verbosity=1):
    """Build a neural net with the indicated input, hidden, and outout dimensions

    Arguments:
        params (dict or PyBrainParams namedtuple): 
            default: {'N_hidden': 6}
            (this is the only parameter that affects the NN build)

    Returns:
        FeedForwardNetwork with N_input + N_hidden + N_output nodes in 3 layers
    """
    N_input = N_input or 1
    N_output = N_output or 1

    nn = pb.structure.FeedForwardNetwork()

    # layers
    nn.addInputModule(pb.structure.BiasUnit(name='bias'))
    nn.addInputModule(pb.structure.LinearLayer(N_input, name='input'))
    try:
        for i, (Nhid, hidlaytype) in enumerate(zip(N_hidden, hidden_layer_type)):
            Nhid = int(Nhid)
            if isinstance(hidlaytype, basestring):
                hidlaytype = getattr(pb.structure, hidlaytype)
            nn.addModule(hidlaytype(Nhid, name=('hidden-{}'.format(i) if i else 'hidden')))
    except:
        N_hidden = int(N_hidden)
        if isinstance(hidden_layer_type, basestring):
            hidden_layer_type = getattr(pb.structure, hidlaytype)
        if N_hidden:
            nn.addModule(hidden_layer_type(N_hidden, name='hidden'))
    nn.addOutputModule(pb.structure.LinearLayer(N_output, name='output'))

    # connections
    nn.addConnection(pb.structure.FullConnection(nn['bias'],  nn['hidden'] if N_hidden else nn['output']))
    nn.addConnection(pb.structure.FullConnection(nn['input'], nn['hidden'] if N_hidden else nn['output']))
    try:
        for i, (Nhid, hidlaytype) in enumerate(zip(N_hidden[:-1], hidden_layer_type[:-1])):
            Nhid = int(Nhid)
            nn.addConnection(pb.structure.FullConnection(nn[('hidden-{}'.format(i) if i else 'hidden')], nn['hidden-{}'.format(i+1)]))
        nn.addConnection(pb.structure.FullConnection(nn['hidden-{}'.format(len(N_hidden)-1)], nn['output']))
    except:
        if N_hidden:
            nn.addConnection(pb.structure.FullConnection(nn['hidden'], nn['output']))

    nn.sortModules()
    if verbosity > 0:
        print(nn)
        print(nn.connections)
    return nn


def ann_from_ds(ds=None, N_input=3, N_hidden=0, N_output=1):
    N_input = getattr(ds, 'indim', N_input)
    N_output = getattr(ds, 'outdim', N_output)
    N_hidden = getattr(ds, 'paramdim', N_hidden + N_input + N_output) - N_hidden - N_output
<commit_msg>normalize_layer_type accepts shortened strings (without Layer suffix)<commit_after>"""Utilities for maniuplating, analyzing and plotting `pybrain` `Network` and `DataSet` objects

TODO:
    Incorporate into pybrain fork so pug doesn't have to depend on pybrain

"""
from __future__ import print_function
import os

import pandas as pd
from scipy import ndarray, reshape  # array, amin, amax, 
np = pd.np
from matplotlib import pyplot as plt
import pybrain.datasets
import pybrain.structure
import pybrain.supervised
import pybrain.tools
pb = pybrain
# from pybrain.supervised.trainers import Trainer
from pybrain.tools.customxml import NetworkReader
from pybrain.structure.parametercontainer import ParameterContainer
from pybrain.structure.connections.connection import Connection


#import pug.nlp.util as nlp

# print(os.path.realpath(__file__))
DATA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'data')


def normalize_layer_type(layer_type):
    if isinstance(layer_type, pb.structure.NeuronLayer):
        return layer_type
    try:
        return getattr(pb.structure, layer_type.strip())
    except:
        try:
            return getattr(pb.structure, layer_type.strip()+'Layer')
        except:
            pass
    return [normalize_layer_type(lt) for lt in layer_type]


def build_ann(N_input=None, N_hidden=2, N_output=1, hidden_layer_type='Linear', verbosity=1):
    """Build a neural net with the indicated input, hidden, and outout dimensions

    Arguments:
        params (dict or PyBrainParams namedtuple): 
            default: {'N_hidden': 6}
            (this is the only parameter that affects the NN build)

    Returns:
        FeedForwardNetwork with N_input + N_hidden + N_output nodes in 3 layers
    """
    N_input = N_input or 1
    N_output = N_output or 1
    N_hidden = N_hidden or tuple()
    if isinstance(N_hidden, (int, float, basestring)):
        N_hidden = (int(N_hidden),)

    hidden_layer_type = hidden_layer_type or tuple()
    hidden_layer_type = normalize_layer_type(hidden_layer_type)
    if isinstance(hidden_layer_type, pb.structure.NeuronLayer):
        hidden_layer_type = (hidden_layer_type,)

    nn = pb.structure.FeedForwardNetwork()

    # layers
    nn.addInputModule(pb.structure.BiasUnit(name='bias'))
    nn.addInputModule(pb.structure.LinearLayer(N_input, name='input'))
    for i, (Nhid, hidlaytype) in enumerate(zip(N_hidden, hidden_layer_type)):
        Nhid = int(Nhid)
        if isinstance(hidlaytype, basestring):
            hidlaytype = getattr(pb.structure, hidlaytype)
        nn.addModule(hidlaytype(Nhid, name=('hidden-{}'.format(i) if i else 'hidden')))
    nn.addOutputModule(pb.structure.LinearLayer(N_output, name='output'))

    # connections
    nn.addConnection(pb.structure.FullConnection(nn['bias'],  nn['hidden'] if N_hidden else nn['output']))
    nn.addConnection(pb.structure.FullConnection(nn['input'], nn['hidden'] if N_hidden else nn['output']))
    for i, (Nhid, hidlaytype) in enumerate(zip(N_hidden[:-1], hidden_layer_type[:-1])):
        Nhid = int(Nhid)
        nn.addConnection(pb.structure.FullConnection(nn[('hidden-{}'.format(i) if i else 'hidden')], nn['hidden-{}'.format(i+1)]))
    nn.addConnection(pb.structure.FullConnection(nn['hidden-{}'.format(len(N_hidden)-1)], nn['output']))

    nn.sortModules()
    if verbosity > 0:
        print(nn)
        print(nn.connections)
    return nn


def ann_from_ds(ds=None, N_input=3, N_hidden=0, N_output=1):
    N_input = getattr(ds, 'indim', N_input)
    N_output = getattr(ds, 'outdim', N_output)
    N_hidden = getattr(ds, 'paramdim', N_hidden + N_input + N_output) - N_hidden - N_output

{'fraction_non_alphanumeric': 0.11737761236759232, 'fraction_numerical': 0.00429430289149728, 'mean_word_length': 3.9413012729844414, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2589878', 'n_tokens_mistral': 2289, 'n_tokens_neox': 2221, 'n_words': 526}