<jupyter_start><jupyter_text>Reference1. http://cs231n.github.io/convolutional-networks/conv to demonstrate dimension changesAccepts a volume of size W1×H1×D1Requires four hyperparameters:- Number of filters K,- their spatial extent F,- the stride S, (move number of pixals for each filter)- the amount of zero padding P.Produces a volume of size W2×H2×D2 where:- W2=(W1−F+2P)/S+1- H2=(H1−F+2P)/S+1 (i.e. width and height are computed equally by symmetry)- D2=KWith parameter sharing, it introduces F⋅F⋅D1 weights per filter, for a total of (F⋅F⋅D1)⋅K weights and K biases.In the output volume, the d-th depth slice (of size W2×H2) is the result of performing a valid convolution of the d-th filter over the input volume with a stride of S, and then offset by d-th bias.<jupyter_code># set optimization parameters
logger.log_this("compiling the OrbWeaver model ...")
network.compile(optimizer=Adadelta(), 
                loss=loss,
                metrics=['accuracy'])
import numpy as np
import scipy.stats as stats
import time
import pdb
import keras.backend as K

from keras.callbacks import Callback, EarlyStopping

class AuROC(Callback):

    def __init__(self, predtype, map_array, logger):

        self.prediction_type = predtype
        self.map_array = map_array
        self.logger = logger
        self.current_time = time.time()

    def on_train_begin(self, logs={}):
        self.values = []
        self.current_time = time.time()

    def on_epoch_end(self, epoch, logs={}):

        values = []
        prediction = self.model.predict(self.validation_data[0])
        Y = self.validation_data[1]
        import pdb; pdb.set_trace()
        
        if self.prediction_type=="cellgroup":
            prediction = np.dot(prediction, self.map_array)
            Y = np.dot(Y, self.map_array)
        


        mask = ~np.logical_or(Y.sum(1)==0, Y.sum(1)==Y.shape[1])

        
        for y,pred in zip(Y.T,prediction.T):
            pos = np.logical_and(mask, y==1)
            neg = np.logical_and(mask, y==0)
            try:
                U = stats.mannwhitneyu(pred[pos], pred[neg])[0]
                values.append(1.-U/(np.count_nonzero(pos)*np.count_nonzero(neg)))
            except ValueError:
                values.append(0.5)

        self.values.append(values)
        epoch_time = time.time()-self.current_time
        self.logger.log_this("epoch_%d_auroc: %s (%ds)"%(epoch, ' '.join(['%.4f'%v for v in self.values[-1]]), int(epoch_time)))
        self.current_time = time.time()
import pydot
pydot
import keras
reload(keras)
from keras.utils import plot_model
#import pydot
plot_model(network, to_file='model_plot.png', show_shapes=True, show_layer_names=True)
network.summary()
from importlib import reload  # Python 3.4+ only.

load=reload(callbacks)

# callbacks
early_stopping = callbacks.EarlyStopping(monitor='val_loss', patience=50)
auroc = callbacks.AuROC(options.prediction_type, cellgroup_map_array, logger)


# train model
logger.log_this("training the OrbWeaver model ...")
logger.log_this("cell types: %s"%(' '.join(cellnames)))
history = network.fit_generator(train_flow, \
                                samples_per_epoch=5, \
                                epochs=options.num_epochs, \
                                verbose=1, \
                                validation_data=(X_validation, Y_validation), \
                                callbacks=[auroc, early_stopping])

import load
reload(load)
# evaluate test accuracy
from importlib import reload  # Python 3.4+ only.

load=reload(load)
logger.log_this("loading test data ...")
test_data_generator = load.DataGenerator(test, genome_track)

test_flow = test_data_generator.flow(batch_size=len(test))
X_test, Y_test = test_flow.next()


logger.log_this("evaluating model on test data ...")
test_auc = compute_test_accuracy(X_test, Y_test, network, options.prediction_type, cellgroup_map_array)
logger.log_this("test auroc: %s"%(' '.join(['%.4f'%v for v in test_auc])))

genome_track.close()
logger.log_this("saving the model architecture and parameters ...")
# save model architecture
network_arch = network.to_json()
handle = open("%s.json"%options.model_prefix,'w')
handle.write(network_arch)
handle.close()



# save model parameters
network.save_weights("%s.h5"%options.model_prefix, overwrite=True)


# save TFs
his = history.history
handle = open("%s.tfs.pkl"%options.model_prefix,'wb')
pickle.dump(tfs,handle,protocol=2)
pickle.dump(history.history,handle,protocol=2)
pickle.dump(test_auc,handle,protocol=2)
handle.close()

logger.log_this("done.")<jupyter_output>done.
{'fraction_non_alphanumeric': 0.12205754141238012, 'fraction_numerical': 0.010897994768962511, 'mean_word_length': 3.353889943074004, 'pattern_counts': {'":': 1, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '79187', 'license': 'MIT', 'n_tokens_mistral': 1517, 'n_tokens_neox': 1489, 'n_words': 344, 'path': '.ipynb_checkpoints/test-checkpoint.ipynb', 'repo_name': 'biomystery/OrbWeaver', 'size': '4588'}