<jupyter_start><jupyter_text>Augmentation<jupyter_code>import imgaug.augmenters as iaa
seq = iaa.Sequential([
    iaa.SomeOf((0,2),[
        iaa.Identity(),
        iaa.AverageBlur(k=((3, 5), (5, 7))),
        iaa.Rotate((-45,45)),
        iaa.Affine(scale=(0.5, 0.95)),    
        iaa.Multiply((0.50, 1.1))
        #,iaa.BlendAlphaRegularGrid(nb_rows=(4, 6), nb_cols=(1, 4),
        #                        foreground=iaa.Multiply(0.0))
        #,iaa.Cartoon()
        ,iaa.Cutout(nb_iterations=(1, 3), size=0.2, squared=False, cval=0)
        ,iaa.Affine(shear=(-48, 48))
        ,iaa.Affine(translate_px={"x": (-42, 42), "y": (-36, 36)})
        ,iaa.KeepSizeByResize(iaa.Resize({"height": (0.70, 0.90), "width": (0.70, 0.90)}))
        ,iaa.CropAndPad(percent=(-0.2, 0.2))
        #,iaa.PiecewiseAffine(scale=(0.01, 0.05))
        ,iaa.PerspectiveTransform(scale=(0.01, 0.1))
        #,iaa.WithPolarWarping(iaa.CropAndPad(percent=(-0.1, 0.1)))
        #,iaa.ElasticTransformation(alpha=(0, 3.0), sigma=0.5)
       ])
    #,iaa.SaveDebugImageEveryNBatches(folder_path, 100)    
], random_order=True)
def generator(features, labels, batch_size):
    while True:
        # Fill arrays of batch size with augmented data taken randomly from full passed arrays
        indexes = random.sample(range(len(features)), batch_size)
      
        # Transform X and y
        x_aug = seq(images =features[indexes])
        yield np.array(x_aug), np.array(labels[indexes])
images = generator(data_x,data_y, 9)
image = next(images)

plt.figure(figsize=(10, 10))
for i in range(9):
    ax = plt.subplot(3, 3, i + 1)
    #augmented_image = seq(images = x_train[:1])
    plt.imshow(image[0][i])
    plt.title(str(image[1][i]))
    plt.axis("off")<jupyter_output>Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).<jupyter_text>Transfer Learning<jupyter_code>x_train, x_val, y_train, y_val = train_test_split(data_x, data_y, test_size=0.20, random_state=42)
y_train = np.argmax(y_train,axis=1)
y_val = np.argmax(y_val,axis=1)
from tensorflow.keras import applications
import logging
logging.basicConfig(filename='EfficientNet-AutoML.log')

model_list = ['EfficientNetB0', 'EfficientNetB1', 'EfficientNetB2', 'EfficientNetB3', 'EfficientNetB4', 'EfficientNetB5'
              , 'EfficientNetB6', 'EfficientNetB7','InceptionResNetV2']

for m in model_list:
    base_model = getattr(applicationspip , m)(weights='imagenet', input_shape=(image_size, image_size, 3),  include_top=False)

    # Create new model on top.
    inputs = keras.Input(shape=(image_size, image_size, 3))
    x = base_model(inputs, training=False)
    x = keras.layers.GlobalAveragePooling2D()(x)
    outputs = keras.layers.Dense(4)(x)
    model = keras.Model(inputs, outputs)
    model.summary()
    
    
    # Warm up head
    adam = optimizers.Adam(learning_rate=0.0005)
    #lr_scheduler = tf.keras.callbacks.LearningRateScheduler(scheduler)
    reduce_lr = ReduceLROnPlateau(monitor='val_loss', 
                               factor=0.2,  
                               patience=3, 
                               min_lr=0.00005)
    early_stop = EarlyStopping(
        monitor='val_loss', 
        min_delta=0, 
        patience=6, 
        verbose=0, 
        mode='auto',
        baseline=None, 
        restore_best_weights=True
    )

    batch_size = 64
    #num_warmup_steps = 5

    # freeze pretrained weights
    model.layers[1].trainable = False

    model.compile(optimizer=adam, 
                  loss=tf.keras.losses.SparseCategoricalCrossentropy(),
                  metrics=[tf.keras.metrics.SparseCategoricalCrossentropy()])


    history = model.fit(generator(x_train, y_train, batch_size),  
                        shuffle=True,  
                        validation_data = (x_val, y_val),
                        callbacks = [reduce_lr,early_stop],                        
                        epochs=100,
                        steps_per_epoch=len(x_train)/batch_size ,
                        verbose=True
                       )
    print(model.optimizer.lr)
    
    # Train entire network
    adam = optimizers.Adam(learning_rate=0.0001)
    #lr_scheduler = tf.keras.callbacks.LearningRateScheduler(scheduler)
    reduce_lr = ReduceLROnPlateau(monitor='val_accuracy', 
                               factor=0.2,  
                               patience=5 , 
                               min_lr=0.00001)
    early_stop = EarlyStopping(
        monitor='val_accuracy', 
        min_delta=0, 
        patience=10, 
        verbose=0, 
        mode='auto',
        baseline=None, 
        restore_best_weights=True
    )

    # freeze pretrained weights
    model.layers[1].trainable = True

    model.compile(optimizer=adam, 
                  loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
                  metrics=['accuracy'])

    history2 = model.fit(generator(x_train, y_train, batch_size),  
                        shuffle=True,  
                        validation_data = (x_val, y_val),
                        callbacks = [reduce_lr,early_stop],                        
                        epochs=200,
                        steps_per_epoch=len(x_train)/batch_size ,
                        verbose=True
                       )
    model.save(f'models/{m}')
    from sklearn.metrics import confusion_matrix
    import seaborn as sn
    import pandas as pd
    import matplotlib.pyplot as plt

    labels = ['healthy', 'multiple_diseases', 'rust', 'scab']
    y_val_pred = np.argmax(model.predict(x_val), axis=1)
    mat = confusion_matrix(y_val, y_val_pred)

    df_cm = pd.DataFrame(mat, index = [i for i in labels],
                      columns = [i for i in labels])
    plt.figure(figsize = (10,7))
    sn.heatmap(df_cm, annot=True)
    
    # Train on all data before inference
    model.fit(generator(x_val, y_val, batch_size),            
                        epochs=2,
                        steps_per_epoch=len(x_train)/batch_size ,
                        verbose=True
                       )
    
    labels = ['healthy', 'multiple_diseases', 'rust', 'scab']
    y_val_pred = np.argmax(model.predict(x_val), axis=1)
    mat = confusion_matrix(y_val, y_val_pred)

    df_cm = pd.DataFrame(mat, index = [i for i in labels],
                      columns = [i for i in labels])
    plt.figure(figsize = (10,7))
    sn.heatmap(df_cm, annot=True)
    
    # Load test data and their filename for submission file
    test_x, test_id  = load_data("Test") 
    test_y = model.predict(test_x)
    test_pred = tf.nn.softmax(test_y).numpy()
    test_set = np.hstack((test_id, test_pred))
    test_set = test_set[test_set[:,1].astype('uint16').argsort()]
    test_DF = pd.DataFrame(test_set, index=test_set[:,1], columns=["image_id","id","healthy","multiple_diseases","rust","scab"])
    test_DF[["image_id","healthy","multiple_diseases","rust","scab"]].to_csv(f'submissions/{m}.csv', index=False)
    
    file = f'submissions/{m}.csv'
    comments = f'"{m} baseline {date.today()}"'
    !kaggle competitions submit -f$file  -m$comments plant-pathology-2020-fgvc7
    
    del model
    
    
!kaggle competitions submissions plant-pathology-2020-fgvc7<jupyter_output>Downloading data from https://storage.googleapis.com/keras-applications/efficientnetb0_notop.h5
16711680/16705208 [==============================] - 0s 0us/step
Model: "model"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_2 (InputLayer)         [(None, 224, 224, 3)]     0         
_________________________________________________________________
efficientnetb0 (Functional)  (None, 7, 7, 1280)        4049571   
_________________________________________________________________
global_average_pooling2d (Gl (None, 1280)              0         
_________________________________________________________________
dense (Dense)                (None, 4)                 5124      
=================================================================
Total params: 4,054,695
Trainable params: 4,012,672
Non-trainable params: 42,023
_________________[...]
{'fraction_non_alphanumeric': 0.13850616388687453, 'fraction_numerical': 0.031786318588349044, 'mean_word_length': 2.521276595744681, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '471911', 'license': 'MIT', 'n_tokens_mistral': 2860, 'n_tokens_neox': 2604, 'n_words': 455, 'path': '2. TL Baseline - EfficientNet-AutoML.ipynb', 'repo_name': 'louis-li/PlantDiseaseDetection', 'size': '8274'}