<jupyter_start><jupyter_text>Efficient Net<jupyter_code>import os
import csv
import numpy as np
import cv2
import scipy

import tensorflow as tf
import keras
import pickle as pkl

import efficientnet.keras as efn
import gc
from keras import backend as K

def fRecall(y_true, y_pred):
        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
        recall = true_positives / (possible_positives + K.epsilon())
        return recall

def fPrecision(y_true, y_pred):
        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
        precision = true_positives / (predicted_positives + K.epsilon())
        return precision

def fF1(y_true, y_pred):
    precision = fPrecision(y_true, y_pred)
    recall = fRecall(y_true, y_pred)
    return 2*((precision*recall)/(precision+recall+K.epsilon()))
from keras.preprocessing.image import ImageDataGenerator
train_gen = ImageDataGenerator(samplewise_center=True, samplewise_std_normalization=True)
val_gen = ImageDataGenerator(samplewise_center=True, samplewise_std_normalization=True)
strTrainDir = '/project/hackathon/hackers07/shared/patchExtraction/all_train/'
strValDir = '/project/hackathon/hackers07/shared/patchExtraction/all_val/'
# Cear GPU cache if needed i.e. run this before creating new model
keras.backend.clear_session()
gc.collect()

def fMyModel():
    pretrained_model = efn.EfficientNetB4(include_top=False, weights='imagenet', input_shape=(299, 299,3))
    #pretrained_model = keras.applications.resnet50.resnet50(input_shape=(512,512,1), weights=None, include_top = False)
    for layer in pretrained_model.layers:
        layer.trainable = False
    output = pretrained_model.output
    output = keras.layers.GlobalAveragePooling2D()(output)
    
    # customize codes here
    output = keras.layers.core.Dense(700, activation='relu')(output)
    output = keras.layers.Dropout(0.10)(output)
    output = keras.layers.core.Dense(32, activation='relu')(output)
    output = keras.layers.core.Dense(1, activation='sigmoid')(output)
    model = keras.models.Model(inputs=pretrained_model.inputs, outputs=output)
    #model = keras.utils.multi_gpu_model(model, gpus=2)
    
    return model
cModel = fMyModel()
cModel.summary()
myOptimizer = keras.optimizers.Nadam(lr=0.004)

cModel.compile(optimizer=myOptimizer, loss='binary_crossentropy', metrics=['accuracy',fF1, fPrecision, fRecall])
nTrain = 22449
nVal = 8527

nTrainNoTube=17564
nTrainTube=4882

aClassWeight = [nTrainTube/nTrainNoTube, 1]
for layer in cModel.layers:
    print(layer.output_shape)
cHistory = cModel.fit_generator(
    train_gen.flow_from_directory(strTrainDir, target_size=(299,299),class_mode='binary'),
    validation_data=val_gen.flow_from_directory(strValDir, target_size=(299,299), class_mode='binary'),
    steps_per_epoch=nTrain//32,
    epochs=1,
    verbose=1,
    validation_steps=nVal//32,
    class_weight=aClassWeight
)
a=train_gen.flow_from_directory(strTrainDir, target_size=(299,299),class_mode='binary')
a1,y1=next(a)
cModel.predict(a1),y1
for layer in cModel.layers:
    layer.trainable = True
from keras.callbacks import ModelCheckpoint
myOptimizer = keras.optimizers.Nadam(lr=0.00004)

cCallback=ModelCheckpoint('weights.{epoch:02d}-{val_loss:.2f}.hdf5', monitor='val_acc', verbose=0, save_best_only=True, save_weights_only=False, mode='auto', period=1)

cModel.compile(optimizer=myOptimizer, loss='binary_crossentropy', metrics=['accuracy',fF1, fPrecision, fRecall])
cHistory = cModel.fit_generator(
    train_gen.flow_from_directory(strTrainDir, target_size=(299,299),class_mode='binary'),
    validation_data=val_gen.flow_from_directory(strValDir, target_size=(299,299),class_mode='binary'),
    steps_per_epoch=nTrain//16,
    epochs=100,
    verbose=1,
    validation_steps=nVal//16,
    class_weight=aClassWeight,
    callbacks=[cCallback]
)
cModel.save('/project/hackathon/hackers07/shared/cEfficientNetB4.h5')
a=train_gen.flow_from_directory(strTrainDir, target_size=(299,299),class_mode='binary')
a1,y1=next(a)
b=cModel.predict(a1)
for i in range(len(y1)):
    print(b.flatten()[i], y1[i])
%matplotlib inline
import matplotlib.pyplot as plt
dHist=cModel.history.history
dHist
lsValues=[]
fig = plt.figure()
ax = plt.subplot(111)
for sKey in dHist.keys():
    if not 'val' in sKey and 'loss' not in sKey:
        ax.plot(range(len(dHist[sKey])), dHist[sKey], label=sKey)
        lsValues.append(sKey)
    else:
        continue
ax.legend(loc='upper center', bbox_to_anchor=(1.2, 0.8))
plt.show()
lsValues=[]
fig = plt.figure()
ax = plt.subplot(111)
for sKey in dHist.keys():
    if 'val' in sKey and 'loss' not in sKey:
        ax.plot(range(len(dHist[sKey])), dHist[sKey], label=sKey)
        lsValues.append(sKey)
    else:
        continue
ax.legend(loc='upper center', bbox_to_anchor=(1.2, 0.8))
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12920782100382988, 'fraction_numerical': 0.032049989921386815, 'mean_word_length': 5.361538461538461, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '213861', 'license': 'MIT', 'n_tokens_mistral': 1920, 'n_tokens_neox': 1798, 'n_words': 308, 'path': 'efficientNet/EfficientNet_b4.ipynb', 'repo_name': 'NCBI-Codeathons/NGcubed', 'size': '4961'}