<commit_before>#!/usr/bin/env python3
"""
Converter from Keras saved NN to JSON
"""

# import yaml
import argparse
import json
import h5py
import numpy as np

def _run():
    """Top level routine"""
    args = _get_args()
    with open(args.arch_file, 'r') as arch_file:
        arch = json.load(arch_file)
    with open(args.inputs_file, 'r') as inputs_file:
        inputs = json.load(inputs_file)
    with h5py.File(args.hdf5_file, 'r') as h5:
        out_dict = {
            'layers': _get_layers(arch, h5),
        }
        out_dict.update(_parse_inputs(inputs))
    print(json.dumps(out_dict, indent=2))

def _get_args():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('arch_file', help='architecture json file')
    parser.add_argument('inputs_file', help='input specifications json')
    parser.add_argument('hdf5_file')
    return parser.parse_args()

_activation_map = {
    'relu': 'rectified',
    'sigmoid': 'sigmoid',
}

def _get_layers(network, h5):
    layers = []
    in_layers = network['layers']
    n_out = h5['layer_0']['param_0'].shape[0]
    for layer_n in range(len(in_layers)):
        activation = _activation_map[in_layers[layer_n]['activation']]
        layer_group = h5['layer_{}'.format(layer_n)]
        weights = layer_group['param_0']
        bias = layer_group['param_1']
        assert weights.shape[1] == bias.shape[0]
        assert weights.shape[0] == n_out
        n_out = weights.shape[1]
        out_layer = {
            'activation': activation,
            'weights': np.asarray(weights).flatten('C').tolist(),
            'bias': np.asarray(bias).flatten('C').tolist()
        }
        layers.append(out_layer)
    return layers
<commit_msg>Add check to make sure Keras layer type is supported<commit_after>#!/usr/bin/env python3
"""
Converter from Keras saved NN to JSON
"""

# import yaml
import argparse
import json
import h5py
import numpy as np

def _run():
    """Top level routine"""
    args = _get_args()
    with open(args.arch_file, 'r') as arch_file:
        arch = json.load(arch_file)
    with open(args.inputs_file, 'r') as inputs_file:
        inputs = json.load(inputs_file)
    with h5py.File(args.hdf5_file, 'r') as h5:
        out_dict = {
            'layers': _get_layers(arch, h5),
        }
        out_dict.update(_parse_inputs(inputs))
    print(json.dumps(out_dict, indent=2))

def _get_args():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('arch_file', help='architecture json file')
    parser.add_argument('inputs_file', help='input specifications json')
    parser.add_argument('hdf5_file')
    return parser.parse_args()

_activation_map = {
    'relu': 'rectified',
    'sigmoid': 'sigmoid',
}

def _get_layers(network, h5):
    layers = []
    in_layers = network['layers']
    n_out = h5['layer_0']['param_0'].shape[0]
    for layer_n in range(len(in_layers)):
        layer_arch = in_layers[layer_n]
        layer_type = layer_arch['name']
        assert layer_type == 'Dense', '{} layers not supported'.format(
            layer_type)
        activation = _activation_map[layer_arch['activation']]

        layer_group = h5['layer_{}'.format(layer_n)]
        weights = layer_group['param_0']
        bias = layer_group['param_1']
        assert weights.shape[1] == bias.shape[0]
        assert weights.shape[0] == n_out
        n_out = weights.shape[1]
        out_layer = {
            'activation': activation,
            'weights': np.asarray(weights).flatten('C').tolist(),
            'bias': np.asarray(bias).flatten('C').tolist()
        }
        layers.append(out_layer)
    return layers

{'fraction_non_alphanumeric': 0.13730853391684902, 'fraction_numerical': 0.010940919037199124, 'mean_word_length': 3.1136107986501687, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6505165', 'n_tokens_mistral': 1219, 'n_tokens_neox': 1195, 'n_words': 244}