<commit_before>#!/usr/bin/python
# -*- coding: UTF-8 -*-
import json
import os
from warnings import warn
from collections import OrderedDict

import numpy as np
import pandas as pd
from IPython import embed

def sigm_tf(x):
    return 1./(1 + np.exp(-1 * x))

#def sigm(x):
#    return 2./(1 + np.exp(-2 * x)) - 1

def flatten(l):
    return [item for sublist in l for item in sublist]

def nn_dict_to_matlab(json_file):
    newjs = {}
    for key, val in json_file.items():
        newjs[key.replace('/', '_').replace(':', '_')] = val
    matdict = {', '.join(newjs['target_names']): newjs}
    return matdict

class QuaLiKizComboNN():
    def __init__(self, target_names, nns, combo_func):
        self._nns = nns
        feature_names = nns[0]
        for nn in self._nns:
            if np.all(nn._feature_names.ne(feature_names)):
                Exception('Supplied NNs have different feature names')
        if np.any(self._feature_min > self._feature_max):
            raise Exception('Feature min > feature max')

        self._combo_func = combo_func
        self._target_names = pd.Series(target_names)
        if not self._target_names.index.is_unique:
            raise Exception('Non unique index for target_names!')
        self._target_min = pd.Series(
            self._combo_func(*[nn._target_min.values for nn in nns]),
            index=self._target_names)
        self._target_max = pd.Series(
            self._combo_func(*[nn._target_max.values for nn in nns]),
            index=self._target_names)

    def get_output(self, input, output_pandas=True, clip_low=False, clip_high=False, low_bound=None, high_bound=None, safe=True, **kwargs):
        nn_input, safe, clip_low, clip_high, low_bound, high_bound = \
            determine_settings(self, input, safe, clip_low, clip_high, low_bound, high_bound)
        output = self._combo_func(*[nn.get_output(nn_input, output_pandas=False, clip_low=False, clip_high=False, safe=False, **kwargs) for nn in self._nns])
        output = clip_to_bounds(output, clip_low=clip_low, clip_high=clip_high, low_bound=low_bound, high_bound=high_bound)
        if output_pandas is True:
            output = pd.DataFrame(output, columns=self._target_names)
        return output

    @property
    def _feature_names(self):
        return self._nns[0]._feature_names

    @property
    def _feature_max(self):
        feature_max = pd.Series(np.full_like(self._nns[0]._feature_max, np.inf),
                                index=self._nns[0]._feature_max.index)
        for nn in self._nns:
            feature_max = nn._feature_max.combine(feature_max, min)
        return feature_max

    @property
    def _feature_min(self):
        feature_min = pd.Series(np.full_like(self._nns[0]._feature_min, -np.inf),
                                index=self._nns[0]._feature_min.index)
        for nn in self._nns:
            feature_min = nn._feature_min.combine(feature_min, max)
        return feature_min
<commit_msg>QOL FFNN: raise on different feature_name, pass safe to output<commit_after>#!/usr/bin/python
# -*- coding: UTF-8 -*-
import json
import os
from warnings import warn
from collections import OrderedDict

import numpy as np
import pandas as pd
from IPython import embed

def sigm_tf(x):
    return 1./(1 + np.exp(-1 * x))

#def sigm(x):
#    return 2./(1 + np.exp(-2 * x)) - 1

def flatten(l):
    return [item for sublist in l for item in sublist]

def nn_dict_to_matlab(json_file):
    newjs = {}
    for key, val in json_file.items():
        newjs[key.replace('/', '_').replace(':', '_')] = val
    matdict = {', '.join(newjs['target_names']): newjs}
    return matdict

class QuaLiKizComboNN():
    def __init__(self, target_names, nns, combo_func):
        self._nns = nns
        for nn in self._nns:
            if np.any(nn._feature_names.ne(self._feature_names)):
                raise Exception('Supplied NNs have different feature names')
        if np.any(self._feature_min > self._feature_max):
            raise Exception('Feature min > feature max')

        self._combo_func = combo_func
        self._target_names = pd.Series(target_names)
        if not self._target_names.index.is_unique:
            raise Exception('Non unique index for target_names!')
        self._target_min = pd.Series(
            self._combo_func(*[nn._target_min.values for nn in nns]),
            index=self._target_names)
        self._target_max = pd.Series(
            self._combo_func(*[nn._target_max.values for nn in nns]),
            index=self._target_names)

    def get_output(self, input, output_pandas=True, clip_low=False, clip_high=False, low_bound=None, high_bound=None, safe=True, **kwargs):
        nn_input, safe, clip_low, clip_high, low_bound, high_bound = \
            determine_settings(self, input, safe, clip_low, clip_high, low_bound, high_bound)
        output = self._combo_func(*[nn.get_output(nn_input, output_pandas=False, clip_low=False, clip_high=False, safe=safe, **kwargs) for nn in self._nns])
        output = clip_to_bounds(output, clip_low=clip_low, clip_high=clip_high, low_bound=low_bound, high_bound=high_bound)
        if output_pandas is True:
            output = pd.DataFrame(output, columns=self._target_names)
        return output

    @property
    def _feature_names(self):
        return self._nns[0]._feature_names

    @property
    def _feature_max(self):
        feature_max = pd.Series(np.full_like(self._nns[0]._feature_max, np.inf),
                                index=self._nns[0]._feature_max.index)
        for nn in self._nns:
            feature_max = nn._feature_max.combine(feature_max, min)
        return feature_max

    @property
    def _feature_min(self):
        feature_min = pd.Series(np.full_like(self._nns[0]._feature_min, -np.inf),
                                index=self._nns[0]._feature_min.index)
        for nn in self._nns:
            feature_min = nn._feature_min.combine(feature_min, max)
        return feature_min

{'fraction_non_alphanumeric': 0.11135261661929444, 'fraction_numerical': 0.0045142952683497745, 'mean_word_length': 2.8174856413529037, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4698755', 'n_tokens_mistral': 1965, 'n_tokens_neox': 1935, 'n_words': 436}