<jupyter_start><jupyter_text>Imports<jupyter_code>import numpy as np
import pandas as pd
import os
import random, re, math
import tensorflow as tf, tensorflow.keras.backend as K
from tensorflow.keras.layers import Dense
from tensorflow.keras.models import Model
from tensorflow.keras import optimizers
from kaggle_datasets import KaggleDatasets
from random import seed
from random import randint
import cv2
from matplotlib import pyplot as plt
import seaborn as sns
from keras import Sequential
from tensorflow.keras.layers import Dense, Dropout, Input
!pip install efficientnet
import efficientnet.tfkeras as efn<jupyter_output>Collecting efficientnet
  Downloading efficientnet-1.1.0-py3-none-any.whl (18 kB)
Requirement already satisfied: keras-applications<=1.0.8,>=1.0.7 in /opt/conda/lib/python3.7/site-packages (from efficientnet) (1.0.8)
Requirement already satisfied: scikit-image in /opt/conda/lib/python3.7/site-packages (from efficientnet) (0.16.2)
Requirement already satisfied: h5py in /opt/conda/lib/python3.7/site-packages (from keras-applications<=1.0.8,>=1.0.7->efficientnet) (2.10.0)
Requirement already satisfied: numpy>=1.9.1 in /opt/conda/lib/python3.7/site-packages (from keras-applications<=1.0.8,>=1.0.7->efficientnet) (1.18.1)
Requirement already satisfied: scipy>=0.19.0 in /opt/conda/lib/python3.7/site-packages (from scikit-image->efficientnet) (1.4.1)
Requirement already satisfied: matplotlib!=3.0.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-image->efficientnet) (3.2.1)
Requirement already satisfied: networkx>=2.0 in /opt/conda/lib/python3.7/site-packages (from sciki[...]<jupyter_text>TPU Setup<jupyter_code>AUTO = tf.data.experimental.AUTOTUNE
# Detect hardware, return appropriate distribution strategy

try:
    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.
    print('Running on TPU ', tpu.master())
except ValueError:
    tpu = None

if tpu:
    tf.config.experimental_connect_to_cluster(tpu)
    tf.tpu.experimental.initialize_tpu_system(tpu)
    strategy = tf.distribute.experimental.TPUStrategy(tpu)
else:
    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.

print("REPLICAS: ", strategy.num_replicas_in_sync)


# Data access
GCS_DS_PATH = KaggleDatasets().get_gcs_path()<jupyter_output>Running on TPU  grpc://10.0.0.2:8470
REPLICAS:  8<jupyter_text>Get DataDue to the overlap between the multiple diseases and the scab and rust categories we will try to compensate for the difficulty in differentiating between these categories by using a series of binary classifiers. Each classifier will be trained on a different subset of binary classifications from the original four. The important distinction between a one vs all approach is that we will have separate classifiers for rust, including multiple diseases, and rust only. We do the same for scab. This will hopefully beneficial because of the overlap between multiple diseases and each individual disease. Then we will use a final multiclass classifier that will use the output of the first six as input and will attempt to classifiy the actual class.<jupyter_code>CSV_PATH = "/kaggle/input/plant-pathology-2020-fgvc7/"
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
IMG_SIZE = 768
EPOCHS = 25
VERBOSE = 1
SHOW_GRAPHS = True
train_csv = pd.read_csv(CSV_PATH + 'train.csv')
test_csv = pd.read_csv(CSV_PATH + 'test.csv')
sub = pd.read_csv(CSV_PATH + 'sample_submission.csv')

# Note that we have to shuffle here and not anywhere else
train_csv = pd.read_csv(CSV_PATH + 'train.csv')
train_csv = train_csv.sample(frac=1).reset_index(drop=True)

test_csv = pd.read_csv(CSV_PATH + 'test.csv')

# Get full image paths
train_paths = train_csv.image_id.apply(lambda image: GCS_DS_PATH + '/images/' + image + '.jpg').values
test_paths = test_csv.image_id.apply(lambda image: GCS_DS_PATH + '/images/' + image + '.jpg').values
def decodeImage(filename, label=None, img_size=(IMG_SIZE, IMG_SIZE)):
    
    bits = tf.io.read_file(filename)
    image = tf.image.decode_jpeg(bits, channels = 3)
    image = (tf.cast(image, tf.float32) / 127.5) - 1
    
    # A few images are rotated
    if image.shape != [1365, 2048, 3]:
        image = tf.image.rot90(image)
    
    image = tf.image.resize(image, img_size)
    
    if label is None:
        return image
    else:
        return image, label
    
def dataAugment(image, label=None, seed=2020):
    
    image = tf.image.random_flip_left_right(image, seed=seed)
    image = tf.image.random_flip_up_down(image, seed=seed)
           
    if label is None:
        return image
    else:
        return image, label
    
def getClassifierData(paths, labels, batch_size=BATCH_SIZE):
    
    ds = (
        tf.data.Dataset
        .from_tensor_slices((train_paths, labels))
        .map(decodeImage, num_parallel_calls=AUTO)
        .map(dataAugment, num_parallel_calls=AUTO)
        #.repeat() # Using repeat leads to inconsistent results when predicting this set (counts are always off)
        .batch(BATCH_SIZE)
        .prefetch(AUTO)
    )
    
    return ds

def calcWeights(df, label):
    
    counts = df[label].value_counts()
    
    return {0 : counts[1] / sum(counts),
            1 : counts[0] / sum(counts)}
# C1 is healthy vs not healthy
train_csv['C1_Label'] = train_csv.apply(lambda row: 0 if row['healthy'] == 1 else 1, axis=1)
# C2 is rust vs not rust
train_csv['C2_Label'] = train_csv.apply(lambda row: 0 if row['rust'] == 1 else 1, axis=1)
# C3 is scab vs not scab
train_csv['C3_Label'] = train_csv.apply(lambda row: 0 if row['scab'] == 1 else 1, axis=1)
# C4 is both diseases vs one or none
train_csv['C4_Label'] = train_csv.apply(lambda row: 0 if row['multiple_diseases'] == 1 else 1, axis=1)
# C5 rust or both vs scab or none
train_csv['C5_Label'] = train_csv.apply(lambda row: 0 if row['multiple_diseases'] == 1 or row['rust'] == 1 else 1, axis=1)
# C6 rust or both vs scab or none
train_csv['C6_Label'] = train_csv.apply(lambda row: 0 if row['multiple_diseases'] == 1 or row['scab'] == 1 else 1, axis=1)

if SHOW_GRAPHS:
    train_csv.head(10)
test_dataset = (
    tf.data.Dataset
    .from_tensor_slices(test_paths)
    .map(decodeImage, num_parallel_calls=AUTO)
    .batch(BATCH_SIZE)
)
# Left stuff for validation data in in case we want to use later

training_dict = {'C1' : {'Train' : getClassifierData(train_paths, train_csv['C1_Label'].values.reshape(-1,1)),
                         #'Val' : getClassifierData(val_data, 'C1_Label', batch_size=BATCH_SIZE),
                         'Weights' : calcWeights(train_csv, 'C1_Label')},
                 
                 'C2' : {'Train' : getClassifierData(train_paths, train_csv['C2_Label'].values.reshape(-1,1)),
                         #'Val' : getClassifierData(val_data, 'C2_Label', batch_size=BATCH_SIZE),
                         'Weights' : calcWeights(train_csv, 'C2_Label')},
                 
                 'C3' : {'Train' : getClassifierData(train_paths, train_csv['C3_Label'].values.reshape(-1,1)),
                         #'Val' : getClassifierData(val_data, 'C3_Label', batch_size=BATCH_SIZE),
                         'Weights' : calcWeights(train_csv, 'C3_Label')},
                 
                 'C4' : {'Train' : getClassifierData(train_paths, train_csv['C4_Label'].values.reshape(-1,1)),
                         #'Val' : getClassifierData(val_data, 'C4_Label', batch_size=BATCH_SIZE),
                         'Weights' : calcWeights(train_csv, 'C4_Label')},
                 
                 'C5' : {'Train' : getClassifierData(train_paths, train_csv['C5_Label'].values.reshape(-1,1)),
                         #'Val' : getClassifierData(val_data, 'C5_Label', batch_size=BATCH_SIZE),
                         'Weights' : calcWeights(train_csv, 'C5_Label')},
                 
                 'C6' : {'Train' : getClassifierData(train_paths, train_csv['C6_Label'].values.reshape(-1,1)),
                         #'Val' : getClassifierData(val_data, 'C6_Label', batch_size=BATCH_SIZE),
                         'Weights' : calcWeights(train_csv, 'C6_Label')},
                 'Test' : test_dataset
                }<jupyter_output><empty_output><jupyter_text>Data Exploration<jupyter_code>seed(8)
IMG_PATH = CSV_PATH + 'images/'
STEPS = train_csv.shape[0] // BATCH_SIZE
SHOW_IMAGES = False
NUM_PER_COL = 2
NUM_PER_ROW = 5
def showImages(df, num_per_col, num_per_row):
    
    fig = plt.figure(figsize=(num_per_row*30, num_per_col*30))

    for i in range(0, num_per_col * num_per_row):
        
        plt.subplot(num_per_col, num_per_row, i+1)
        
        image_id = df.iloc[randint(0, len(df)-1)]['image_id']
        image = cv2.imread(IMG_PATH + image_id + '.jpg', cv2.IMREAD_UNCHANGED)
        image = cv2.resize(image, (int(IMG_SIZE / 2), int(IMG_SIZE / 2)))
        
        plt.imshow(image)
        plt.axis('off')
        
    plt.tight_layout()
    plt.show()
healthy_plants = train_csv[train_csv['healthy'] == 1]

if SHOW_IMAGES:
    showImages(healthy_plants, NUM_PER_COL, NUM_PER_ROW)
scab_plants = train_csv[train_csv['scab'] == 1]

if SHOW_IMAGES:
    showImages(scab_plants, NUM_PER_COL, NUM_PER_ROW)
rust_plants = train_csv[train_csv['rust'] == 1]

if SHOW_IMAGES:
    showImages(rust_plants, NUM_PER_COL, NUM_PER_ROW)
multiple_plants = train_csv[train_csv['multiple_diseases'] == 1]

if SHOW_IMAGES:
    showImages(multiple_plants, NUM_PER_COL, NUM_PER_ROW)
if SHOW_GRAPHS:
    fig = plt.figure()
    ax = fig.add_axes([0,0,1,1])
    labels = ['healthy', 'rust', 'scab', 'multiple_diseases']
    counts = [train_csv['healthy'].value_counts()[1], 
              train_csv['rust'].value_counts()[1],
              train_csv['scab'].value_counts()[1], 
              train_csv['multiple_diseases'].value_counts()[1]]
    ax.bar(labels, counts)
    plt.show()
if SHOW_GRAPHS:
    fig = plt.figure(figsize=(15,35))

    plt.subplot(621)
    ax = sns.countplot(x='C1_Label', data=train_csv, order=[0, 1])
    ax.set_title('C1 Train')
    ax.set_xticklabels(['Healthy', 'Not Healthy'])
    ax.set(xlabel='Disease')

    plt.subplot(622)
    ax = sns.countplot(x='C1_Label', data=train_csv, order=[0, 1])
    ax.set_title('C1 Test')
    ax.set_xticklabels(['Healthy', 'Not Healthy'])
    ax.set(xlabel='Disease')

    plt.subplot(623)
    ax = sns.countplot(x='C2_Label', data=train_csv, order=[0, 1])
    ax.set_title('C2 Train')
    ax.set_xticklabels(['Rust', 'Not Rust'])
    ax.set(xlabel='Disease')

    plt.subplot(624)
    ax = sns.countplot(x='C2_Label', data=train_csv, order=[0, 1])
    ax.set_title('C2 Test')
    ax.set_xticklabels(['Rust', 'Not Rust'])
    ax.set(xlabel='Disease')

    plt.subplot(625)
    ax = sns.countplot(x='C3_Label', data=train_csv, order=[0, 1])
    ax.set_title('C3 Train')
    ax.set_xticklabels(['Scab', 'Not Scab'])
    ax.set(xlabel='Disease')

    plt.subplot(626)
    ax = sns.countplot(x='C3_Label', data=train_csv, order=[0, 1])
    ax.set_title('C3 Test')
    ax.set_xticklabels(['Scab', 'Not Scab'])
    ax.set(xlabel='Disease')

    plt.subplot(627)
    ax = sns.countplot(x='C4_Label', data=train_csv, order=[0, 1])
    ax.set_title('C4 Train')
    ax.set_xticklabels(['Multiple', 'Not Multiple'])
    ax.set(xlabel='Disease')

    plt.subplot(628)
    ax = sns.countplot(x='C4_Label', data=train_csv, order=[0, 1])
    ax.set_title('C4 Test')
    ax.set_xticklabels(['Multiple', 'Not Multiple'])
    ax.set(xlabel='Disease')

    plt.subplot(629)
    ax = sns.countplot(x='C5_Label', data=train_csv, order=[0, 1])
    ax.set_title('C5 Train')
    ax.set_xticklabels(['Rust or Multiple', 'Scab or None'])
    ax.set(xlabel='Disease')

    plt.subplot(6,2,10)
    ax = sns.countplot(x='C5_Label', data=train_csv, order=[0, 1])
    ax.set_title('C5 Test')
    ax.set_xticklabels(['Rust or Multiple', 'Scab or None'])
    ax.set(xlabel='Disease')

    plt.subplot(6,2,11)
    ax = sns.countplot(x='C6_Label', data=train_csv, order=[0, 1])
    ax.set_title('C6 Train')
    ax.set_xticklabels(['Scab or Multiple', 'Rust or None'])
    ax.set(xlabel='Disease')

    plt.subplot(6,2,12)
    ax = sns.countplot(x='C6_Label', data=train_csv, order=[0, 1])
    ax.set_title('C6 Test')
    ax.set_xticklabels(['Scab or Multiple', 'Rust or None'])
    ax.set(xlabel='Disease')

    plt.show()<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>LR_START = 0.00001
LR_MAX = 0.0001 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 15
LR_SUSTAIN_EPOCHS = 3
LR_EXP_DECAY = .8

def lrfn(epoch):
    
    if epoch < LR_RAMPUP_EPOCHS:
        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START
    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
        lr = LR_MAX
    else:
        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN
        
    return lr
    
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)

if SHOW_GRAPHS:
    rng = [i for i in range(EPOCHS)]
    y = [lrfn(x) for x in rng]
    plt.plot(rng, y)
    print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
# We have to disable the steps due to the way the training datasets are set up now

def get_model():
    
    base_model =  efn.EfficientNetB7(weights='imagenet', include_top=False, 
                                     pooling='avg', input_shape=(IMG_SIZE, IMG_SIZE, 3))
    
    x = base_model.output
    output = Dense(1, activation="sigmoid")(x)
    
    return Model(inputs=base_model.input, outputs=output)

def fitModel(label, training_dict, lr_callback=lr_callback, epochs=EPOCHS, steps=STEPS, verbose=VERBOSE):
    
    with strategy.scope():
        model = get_model()

    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
    
    model.fit(
        training_dict[label]['Train'],
        #steps_per_epoch = steps,
        callbacks = [lr_callback],
        #class_weight = training_dict[label]['Weights'] # Class weights still aren't working
        epochs = epochs,
        verbose = verbose
    )
    
    return model

def getPredictions(label, training_dict, steps=STEPS, verbose=VERBOSE):
    
    model = fitModel(label, training_dict)
    
    train_predictions = model.predict(training_dict[label]['Train'], #steps = steps,
                                use_multiprocessing = True, verbose = verbose)
    
    test_predictions = model.predict(training_dict['Test'], #steps = steps,
                                use_multiprocessing = True, verbose = verbose)
    
    # Releases tpu memory from model so other models can run
    tf.tpu.experimental.initialize_tpu_system(tpu)
    
    return train_predictions, test_predictions
predictions = [getPredictions('C1', training_dict),
               getPredictions('C2', training_dict),
               getPredictions('C3', training_dict),
               getPredictions('C4', training_dict),
               getPredictions('C5', training_dict),
               getPredictions('C6', training_dict)]
train_predictions, test_predictions = zip(*predictions)
train_predictions = np.hstack(train_predictions)
test_predictions = np.hstack(test_predictions)
inputs = Input(shape=(train_predictions.shape[1],))
dense1 = Dense(200, activation='relu')(inputs)
dropout = Dropout(0.3)(dense1)
dense2 = Dense(50, activation='relu')(dropout)
outputs = Dense(4, activation='softmax')(dense2)
model_final = Model(inputs=inputs, outputs=outputs)

model_final.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

model_final.fit(
    train_predictions,
    train_csv[['healthy', 'multiple_diseases', 'rust', 'scab']].values,
    epochs = EPOCHS
)<jupyter_output>Epoch 1/25
57/57 [==============================] - 0s 4ms/step - loss: 0.5367 - accuracy: 0.9198
Epoch 2/25
57/57 [==============================] - 0s 4ms/step - loss: 0.0295 - accuracy: 0.9995
Epoch 3/25
57/57 [==============================] - 0s 4ms/step - loss: 0.0070 - accuracy: 1.0000
Epoch 4/25
57/57 [==============================] - 0s 4ms/step - loss: 0.0038 - accuracy: 1.0000
Epoch 5/25
57/57 [==============================] - 0s 4ms/step - loss: 0.0020 - accuracy: 1.0000
Epoch 6/25
57/57 [==============================] - 0s 5ms/step - loss: 0.0014 - accuracy: 1.0000
Epoch 7/25
57/57 [==============================] - 0s 4ms/step - loss: 0.0013 - accuracy: 1.0000
Epoch 8/25
57/57 [==============================] - 0s 4ms/step - loss: 7.1795e-04 - accuracy: 1.0000
Epoch 9/25
57/57 [==============================] - 0s 4ms/step - loss: 5.5469e-04 - accuracy: 1.0000
Epoch 10/25
57/57 [==============================] - 0s 5ms/step - loss: 5.4226e-04 - accuracy: 1.0000
Epoch 1[...]<jupyter_text>Predictions<jupyter_code>sub_predictions = model_final.predict(test_predictions)
sub_predictions = pd.DataFrame(data=sub_predictions, columns=['healthy', 'multiple_diseases', 'rust', 'scab'])
sub_predictions = pd.concat([test_csv, sub_predictions], axis=1)
sub_predictions
sub_predictions.to_csv('submission.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14657127900889896, 'fraction_numerical': 0.033094864188914094, 'mean_word_length': 3.638251955759374, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '38252', 'license': 'Unlicense', 'n_tokens_mistral': 6449, 'n_tokens_neox': 6026, 'n_words': 1278, 'path': 'projects/plant-pathology.ipynb', 'repo_name': 'phogya/KaggleProjects', 'size': '17193'}