<commit_before>from __future__ import absolute_import
from __future__ import print_function
import sys
import glob

import time
import numpy as np
import pandas as pd
import os.path
import time
import datetime
import re

from keras.preprocessing import sequence
from keras.optimizers import SGD, RMSprop, Adagrad
from keras.utils import np_utils
from keras.models import Sequential, Graph, Model
from keras.models import model_from_json

from keras.layers import Input, merge, Flatten, Dense, Activation, Convolution1D, ZeroPadding1D

from convolutional_transpose import Convolution1D_Transpose_Arbitrary

#from keras.layers.core import Dense, Dropout, Activation, TimeDistributedDense, Flatten, Reshape, Permute, Merge, Lambda
#from keras.layers.convolutional import Convolution1D, MaxPooling1D, Convolution2D, MaxPooling2D, UpSampling1D, UpSampling2D, ZeroPadding1D
from keras.layers.advanced_activations import ParametricSoftplus, SReLU
from keras.callbacks import ModelCheckpoint, Callback
import matplotlib.pyplot as plt

path = "./training_data_large/"  # to make sure signal files are written in same directory as data files

def draw_model(model):
    from IPython.display import SVG
    from keras.utils.visualize_util import model_to_dot
    from keras.utils.visualize_util import plot

    #graph = to_graph(model, show_shape=True)
    #graph.write_png("UFCNN_1.png")

    SVG(model_to_dot(model).create(prog='dot', format='svg'))
    plot(model, to_file='UFCNN_1.png')

# DEPRECATED DUE TO INTRODUCTION OF model.summary()
def print_nodes_shapes(model):
# DEPRECATED DUE TO INTRODUCTION OF model.summary()
    print(model.summary())
#     for k, v in model.inputs.items():
#         print("{} : {} : {} : {}".format(k, type(v), v.input_shape, v.output_shape))
#
#     for k, v in model.nodes.items():
#         print("{} : {} : {} : {}".format(k, type(v), v.input_shape, v.output_shape))
#
#     for k, v in model.outputs.items():
#         print("{} : {} : {} : {}".format(k, type(v), v.input_shape, v.output_shape))
#
#
def print_layers_shapes(model):
# DEPRECATED DUE TO INTRODUCTION OF model.summary()
    print(model.summary())
#     for l in model.layers:
#         print("{} : {} : {}".format(type(l), l.input_shape, l.output_shape))
        

def save_neuralnet (model, model_name):

    json_string = model.to_json()
    open(path + model_name + '_architecture.json', 'w').write(json_string)
    model.save_weights(path + model_name + '_weights.h5', overwrite=True)

    yaml_string = model.to_yaml()
    with open(path + model_name + '_data.yml', 'w') as outfile:
        outfile.write( yaml_string)

def load_neuralnet(model_name):
    """ 
    reading the model from disk - including all the trained weights and the complete model design (hyperparams, planes,..)
    """

    arch_name = path + model_name + '_architecture.json'
    weight_name = path + model_name + '_weights.h5'

    if not os.path.isfile(arch_name) or not os.path.isfile(weight_name):
        print("model_name given and file %s and/or %s not existing. Aborting." % (arch_name, weight_name))
        sys.exit()

    print("Loaded model: ",model_name)

    model = model_from_json(open(arch_name).read())
    model.load_weights(weight_name)
    return model


def ufcnn_model_concat(sequence_length=5000,
                       features=1,
                       nb_filter=150,
                       filter_length=5,
                       output_dim=1,
                       optimizer='adagrad',
                       loss='mse',
                       regression = True,
                       class_mode=None,
                       activation="softplus",
                       init="lecun_uniform"):
    #model = Graph()
   
    #model.add_input(name='input', input_shape=(None, features))

    main_input = Input(name='input', shape=(None, features))

    #########################################################

    #model.add_node(ZeroPadding1D(2), name='input_padding', input='input') # to avoid lookahead bias

    input_padding = ZeroPadding1D(2)(main_input)  # to avoid lookahead bias

    #########################################################

    #model.add_node(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='valid', init=init, input_shape=(sequence_length, features)), name='conv1', input='input_padding')
    #model.add_node(Activation(activation), name='relu1', input='conv1')

    conv1 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='valid', init=init)(input_padding)
    relu1 = Activation(activation)(conv1)

    #########################################################
    #model.add_node(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init), name='conv2', input='relu1')
    #model.add_node(Activation(activation), name='relu2', input='conv2')

    conv2 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(relu1)
    relu2 = Activation(activation)(conv2)

    #########################################################
    #model.add_node(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init), name='conv3', input='relu2')
    #model.add_node(Activation(activation), name='relu3', input='conv3')


    conv3 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(relu2)
    relu3 = Activation(activation)(conv3)

    #########################################################
    #model.add_node(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init), name='conv4', input='relu3')
    #model.add_node(Activation(activation), name='relu4', input='conv4')

    conv4 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(relu3)
    relu4 = Activation(activation)(conv4)

    #########################################################
    #model.add_node(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init), name='conv5', input='relu4')
    #model.add_node(Activation(activation), name='relu5', input='conv5')

    conv5 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(relu4)
    relu5 = Activation(activation)(conv5)

    #########################################################
    #model.add_node(Convolution1D(nb_filter=nb_filter,filter_length=filter_length, border_mode='same', init=init),
    #                 name='conv6',
    #                 inputs=['relu3', 'relu5'],
    #                 merge_mode='concat', concat_axis=-1)
    #model.add_node(Activation(activation), name='relu6', input='conv6')


    merge6 = merge([relu3, relu5], mode='concat', concat_axis=1)
    conv6 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(merge6)
    relu6 = Activation(activation)(conv6)

    #########################################################
    #model.add_node(Convolution1D(nb_filter=nb_filter,filter_length=filter_length, border_mode='same', init=init),
    #                 name='conv7',
    #                 inputs=['relu2', 'relu6'],
    #                 merge_mode='concat', concat_axis=-1)
    #model.add_node(Activation(activation), name='relu7', input='conv7')

    merge7 = merge([relu2, relu6], mode='concat', concat_axis=1)
    conv7 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(merge7)
    relu7 = Activation(activation)(conv7)

    #########################################################
    #model.add_node(Convolution1D(nb_filter=nb_filter,filter_length=filter_length, border_mode='same', init=init),
    #                 name='conv8',
    #                 inputs=['relu1', 'relu7'],
    #                 merge_mode='concat', concat_axis=-1)
    #model.add_node(Activation(activation), name='relu8', input='conv8')

    merge8 = merge([relu1, relu7], mode='concat', concat_axis=1)
    conv8 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(merge8)
    relu8 = Activation(activation)(conv8)

    #########################################################
    if regression:
        #########################################################
        #model.add_node(Convolution1D(nb_filter=output_dim, filter_length=sequence_length, border_mode='same', init=init), name='conv9', input='relu8')
        #model.add_output(name='output', input='conv9')


        conv9 = Convolution1D(nb_filter=output_dim, filter_length=sequence_length, border_mode='same', init=init)(relu8)
        output = conv9
        #main_output = conv9.output

    else:
        #model.add_node(Convolution1D(nb_filter=output_dim, filter_length=sequence_length, border_mode='same', init=init), name='conv9', input='relu8')
        #model.add_node(Activation('softmax'), name='activation', input='conv9')
        #model.add_output(name='output', input='activation')

        conv9 = Convolution1D(nb_filter=output_dim, filter_length=sequence_length, border_mode='same', init=init)(relu8)
        activation = (Activation('softmax'))(conv9)
        #main_output = activation.output
        output = activation
    
    #model.compile(optimizer=optimizer, loss={'output': loss})

    model = Model(input=main_input, output=output)
    model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy', ])

    print(model.summary())<commit_msg>* Fixed merge params which produced wrong shape changes<commit_after>from __future__ import absolute_import
from __future__ import print_function
import sys
import glob

import time
import numpy as np
import pandas as pd
import os.path
import time
import datetime
import re

from keras.preprocessing import sequence
from keras.optimizers import SGD, RMSprop, Adagrad
from keras.utils import np_utils
from keras.models import Sequential, Graph, Model
from keras.models import model_from_json

from keras.layers import Input, merge, Flatten, Dense, Activation, Convolution1D, ZeroPadding1D

from convolutional_transpose import Convolution1D_Transpose_Arbitrary

#from keras.layers.core import Dense, Dropout, Activation, TimeDistributedDense, Flatten, Reshape, Permute, Merge, Lambda
#from keras.layers.convolutional import Convolution1D, MaxPooling1D, Convolution2D, MaxPooling2D, UpSampling1D, UpSampling2D, ZeroPadding1D
from keras.layers.advanced_activations import ParametricSoftplus, SReLU
from keras.callbacks import ModelCheckpoint, Callback
import matplotlib.pyplot as plt

path = "./training_data_large/"  # to make sure signal files are written in same directory as data files

def draw_model(model):
    from IPython.display import SVG
    from keras.utils.visualize_util import model_to_dot
    from keras.utils.visualize_util import plot

    #graph = to_graph(model, show_shape=True)
    #graph.write_png("UFCNN_1.png")

    SVG(model_to_dot(model).create(prog='dot', format='svg'))
    plot(model, to_file='UFCNN_1.png')

# DEPRECATED DUE TO INTRODUCTION OF model.summary()
def print_nodes_shapes(model):
# DEPRECATED DUE TO INTRODUCTION OF model.summary()
    print(model.summary())
#     for k, v in model.inputs.items():
#         print("{} : {} : {} : {}".format(k, type(v), v.input_shape, v.output_shape))
#
#     for k, v in model.nodes.items():
#         print("{} : {} : {} : {}".format(k, type(v), v.input_shape, v.output_shape))
#
#     for k, v in model.outputs.items():
#         print("{} : {} : {} : {}".format(k, type(v), v.input_shape, v.output_shape))
#
#
def print_layers_shapes(model):
# DEPRECATED DUE TO INTRODUCTION OF model.summary()
    print(model.summary())
#     for l in model.layers:
#         print("{} : {} : {}".format(type(l), l.input_shape, l.output_shape))
        

def save_neuralnet (model, model_name):

    json_string = model.to_json()
    open(path + model_name + '_architecture.json', 'w').write(json_string)
    model.save_weights(path + model_name + '_weights.h5', overwrite=True)

    yaml_string = model.to_yaml()
    with open(path + model_name + '_data.yml', 'w') as outfile:
        outfile.write( yaml_string)

def load_neuralnet(model_name):
    """ 
    reading the model from disk - including all the trained weights and the complete model design (hyperparams, planes,..)
    """

    arch_name = path + model_name + '_architecture.json'
    weight_name = path + model_name + '_weights.h5'

    if not os.path.isfile(arch_name) or not os.path.isfile(weight_name):
        print("model_name given and file %s and/or %s not existing. Aborting." % (arch_name, weight_name))
        sys.exit()

    print("Loaded model: ",model_name)

    model = model_from_json(open(arch_name).read())
    model.load_weights(weight_name)
    return model


def ufcnn_model_concat(sequence_length=5000,
                       features=1,
                       nb_filter=150,
                       filter_length=5,
                       output_dim=1,
                       optimizer='adagrad',
                       loss='mse',
                       regression = True,
                       class_mode=None,
                       activation="softplus",
                       init="lecun_uniform"):
    #model = Graph()
   
    #model.add_input(name='input', input_shape=(None, features))

    main_input = Input(name='input', shape=(None, features))

    #########################################################

    #model.add_node(ZeroPadding1D(2), name='input_padding', input='input') # to avoid lookahead bias

    input_padding = ZeroPadding1D(2)(main_input)  # to avoid lookahead bias

    #########################################################

    #model.add_node(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='valid', init=init, input_shape=(sequence_length, features)), name='conv1', input='input_padding')
    #model.add_node(Activation(activation), name='relu1', input='conv1')

    conv1 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='valid', init=init)(input_padding)
    relu1 = Activation(activation)(conv1)

    #########################################################
    #model.add_node(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init), name='conv2', input='relu1')
    #model.add_node(Activation(activation), name='relu2', input='conv2')

    conv2 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(relu1)
    relu2 = Activation(activation)(conv2)

    #########################################################
    #model.add_node(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init), name='conv3', input='relu2')
    #model.add_node(Activation(activation), name='relu3', input='conv3')


    conv3 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(relu2)
    relu3 = Activation(activation)(conv3)

    #########################################################
    #model.add_node(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init), name='conv4', input='relu3')
    #model.add_node(Activation(activation), name='relu4', input='conv4')

    conv4 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(relu3)
    relu4 = Activation(activation)(conv4)

    #########################################################
    #model.add_node(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init), name='conv5', input='relu4')
    #model.add_node(Activation(activation), name='relu5', input='conv5')

    conv5 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(relu4)
    relu5 = Activation(activation)(conv5)

    #########################################################
    #model.add_node(Convolution1D(nb_filter=nb_filter,filter_length=filter_length, border_mode='same', init=init),
    #                 name='conv6',
    #                 inputs=['relu3', 'relu5'],
    #                 merge_mode='concat', concat_axis=-1)
    #model.add_node(Activation(activation), name='relu6', input='conv6')


    merge6 = merge([relu3, relu5], mode='concat')
    conv6 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(merge6)
    relu6 = Activation(activation)(conv6)

    #########################################################
    #model.add_node(Convolution1D(nb_filter=nb_filter,filter_length=filter_length, border_mode='same', init=init),
    #                 name='conv7',
    #                 inputs=['relu2', 'relu6'],
    #                 merge_mode='concat', concat_axis=-1)
    #model.add_node(Activation(activation), name='relu7', input='conv7')

    merge7 = merge([relu2, relu6], mode='concat')
    conv7 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(merge7)
    relu7 = Activation(activation)(conv7)

    #########################################################
    #model.add_node(Convolution1D(nb_filter=nb_filter,filter_length=filter_length, border_mode='same', init=init),
    #                 name='conv8',
    #                 inputs=['relu1', 'relu7'],
    #                 merge_mode='concat', concat_axis=-1)
    #model.add_node(Activation(activation), name='relu8', input='conv8')

    merge8 = merge([relu1, relu7], mode='concat')
    conv8 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(merge8)
    relu8 = Activation(activation)(conv8)

    #########################################################
    if regression:
        #########################################################
        #model.add_node(Convolution1D(nb_filter=output_dim, filter_length=sequence_length, border_mode='same', init=init), name='conv9', input='relu8')
        #model.add_output(name='output', input='conv9')


        conv9 = Convolution1D(nb_filter=output_dim, filter_length=sequence_length, border_mode='same', init=init)(relu8)
        output = conv9
        #main_output = conv9.output

    else:
        #model.add_node(Convolution1D(nb_filter=output_dim, filter_length=sequence_length, border_mode='same', init=init), name='conv9', input='relu8')
        #model.add_node(Activation('softmax'), name='activation', input='conv9')
        #model.add_output(name='output', input='activation')

        conv9 = Convolution1D(nb_filter=output_dim, filter_length=sequence_length, border_mode='same', init=init)(relu8)
        activation = (Activation('softmax'))(conv9)
        #main_output = activation.output
        output = activation
    
    #model.compile(optimizer=optimizer, loss={'output': loss})

    model = Model(input=main_input, output=output)
    model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy', ])

    print(model.summary())
{'fraction_non_alphanumeric': 0.1865510900971894, 'fraction_numerical': 0.014657210401891253, 'mean_word_length': 4.556333917104495, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2943340', 'n_tokens_mistral': 6031, 'n_tokens_neox': 5771, 'n_words': 1044}