<jupyter_start><jupyter_text><jupyter_code># for data reading
import pandas as pd
import numpy as np
from keras_preprocessing.image import ImageDataGenerator

# for model architecture
from keras.models import Model
from keras.layers import Input, Dense, Flatten, Conv2D, MaxPooling2D
from keras.optimizers import Adam

# for plotting
from matplotlib import pyplot as plt

# for prediction evaluation
import math
from scipy import stats
from sklearn.metrics import mean_squared_error

# for saving files
from pandas import DataFrame
#### DATA PREPARATION
training_dir = '/content/drive/MyDrive/SoybeanIDC_CNN_Data/Valley711_RIL/' ##NEED TO refine directory for your own case
testing_dir = '/content/drive/MyDrive/SoybeanIDC_CNN_Data/Valley711_NAM/'

training_label_dir ='/content/drive/MyDrive/SoybeanIDC_CNN_Data/Valley711_RIL.csv'
testing_label_dir = '/content/drive/MyDrive/SoybeanIDC_CNN_Data/Valley711_NAM.csv'
training_label = pd.read_csv(training_label_dir, delimiter=',', header=0, names=['filename', 'IDC'])
testing_label = pd.read_csv(testing_label_dir, delimiter=',', header=0, names=['filename', 'IDC'])

testing_y = np.array(testing_label['IDC']) #true IDC scores for testing dataset

img_width, img_height = 224, 448 

## Read data using flow_from_dataframe()
datagen=ImageDataGenerator(rescale=1./255.,validation_split=0.2)
test_datagen=ImageDataGenerator(rescale=1./255.)

train_generator=datagen.flow_from_dataframe(
    dataframe=training_label,
    directory=training_dir,
    x_col='filename', 
    y_col='IDC',
    subset="training",
    batch_size=32,
    seed=22,
    shuffle=True,
    class_mode='raw',
    target_size=(img_height,img_width))

valid_generator=datagen.flow_from_dataframe(
    dataframe=training_label,
    directory=training_dir,
    x_col='filename', 
    y_col='IDC',
    subset="validation",
    batch_size=32,
    seed=22,
    shuffle=True,
    class_mode="raw",
    target_size=(img_height,img_width))

test_generator=test_datagen.flow_from_dataframe(
    dataframe=testing_label,
    directory=testing_dir,
    x_col="filename",
    y_col=None,
    batch_size=1,
    seed=22,
    shuffle=False,
    class_mode=None,
    target_size=(img_height,img_width))
#### Define Model
inChannel = 3
input_img = Input(shape = (img_height, img_width, inChannel))

def regres(input_img):
    x = Conv2D(filters=16, kernel_size=3, activation='relu', padding='same')(input_img) 
    x = MaxPooling2D(pool_size=(2, 2))(x) 
    
    x = Conv2D(filters=32, kernel_size=3, activation='relu', padding='same')(x) 
    x = MaxPooling2D(pool_size=(2, 2))(x) 
     
    x = Flatten()(x)
    x = Dense(units=100, activation='relu')(x)

    #The output layer. Use 'softmax' or 'sigmoid' activation if your task is classification problem
    pred = Dense(units=1, activation='linear')(x) 
    return pred 


full_model = Model(input_img,regres(input_img))

full_model.summary()

full_model.compile(loss='mean_squared_error', metrics=['mean_squared_error'], optimizer=Adam(lr=0.0001))
#### TRAIN model
STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size
STEP_SIZE_VALID=valid_generator.n//valid_generator.batch_size

regressor = full_model.fit_generator(generator=train_generator,
                                     steps_per_epoch=STEP_SIZE_TRAIN,
                                     validation_data=valid_generator,
                                     validation_steps=STEP_SIZE_VALID,
                                     epochs=100)

## plot training process
loss = regressor.history['loss']
val_loss = regressor.history['val_loss']
epochs = range(len(loss))
plt.figure()
plt.plot(epochs, loss, 'b', label='Training loss')
plt.plot(epochs, val_loss, 'r', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()
#### PREDICTION
STEP_SIZE_TEST=test_generator.n//test_generator.batch_size
test_generator.reset()

predict=full_model.predict_generator(test_generator,steps=STEP_SIZE_TEST,verbose=1)

# plot results
pred = predict[:,0]
true = testing_y
plt.plot(pred,true, 'g^')

# r2 and rmse 
slope, intercept, r_value, p_value, std_err = stats.linregress(true, pred) #x,y
mse = mean_squared_error(true, pred)
rmse = math.sqrt(mse)
print("R2: %f" % r_value**2)
print("RMSE: %f" % rmse)

# ## save
# df = DataFrame([true,pred])
# df = df.T #transpose
# df.to_excel('PATH TO EXPECTED DIRECTORY.xlsx', index=False)<jupyter_output>12/164 [=>............................] - ETA: 0s
{'fraction_non_alphanumeric': 0.12341078474353354, 'fraction_numerical': 0.017755370451556333, 'mean_word_length': 4.008781558726674, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '582512', 'license': 'MIT', 'n_tokens_mistral': 1670, 'n_tokens_neox': 1565, 'n_words': 282, 'path': 'CNN.ipynb', 'repo_name': 'JiatingLi/CNN_SoybeanIDC', 'size': '4562'}