<jupyter_start><jupyter_text><jupyter_code># -*- coding: utf-8 -*-
"""
Created on Sat Oct 24 15:46:07 2020

@author: fbasatemur
"""

from keras.models import Sequential
from keras.layers import Dense, Dropout, BatchNormalization
from keras.callbacks import ModelCheckpoint
from keras.losses import SparseCategoricalCrossentropy
from keras.models import model_from_json
from keras.models import load_model
from keras import layers, activations
from matplotlib import pyplot as plt


 
#%% ANN model
def createModel():
      
      model = Sequential()
      
      model.add(Dense(100, activation="relu", input_shape=(625,)))  # 25x25 FFT
      model.add(BatchNormalization()) 
      
      model.add(Dense(100, activation="relu"))
      model.add(BatchNormalization())
      
      model.add(Dense(100, activation="relu"))
      model.add(BatchNormalization())
      
      model.add(Dense(100, activation="relu"))
      model.add(BatchNormalization())
      
      model.add(Dense(100, activation="relu"))
      model.add(BatchNormalization())
      
      model.add(Dense(100, activation="relu"))
      model.add(BatchNormalization())

      model.add(Dense(100, activation="relu"))
      model.add(BatchNormalization())
      
      model.add(Dense(20, activation="relu"))
      model.add(BatchNormalization())
      
      model.add(Dense(1, activation="sigmoid"))   # unipolar sigmoid (0-1)
      
      model.compile(optimizer='Adam', loss='mean_squared_error', metrics=['mae'])
      
      return model

def showAccuracy(history):
      
      for key in history.history.keys():  
            plt.plot(history.history[key], label = key)
    
      plt.xlabel("epochs")
      plt.ylabel("values")
      
      plt.legend()
      plt.show() 
      


# drive access

from google.colab import drive
drive.mount('/content/drive')

!ls
#%% read file 

import numpy as np

fftValues = []
varianceValues = []
imagePaths = []

# drive_path_FFT = "drive/My Drive/Colab Notebooks/FFT_Variance_ANN_Tasarim_Donemi/25_NORM_FFT.txt" 
# drive_path_VAR = "drive/My Drive/Colab Notebooks/FFT_Variance_ANN_Tasarim_Donemi/100_Variance.txt" 
# drive_path_IMGPATHS = "drive/My Drive/Colab Notebooks/FFT_Variance_ANN_Tasarim_Donemi/Image_Paths.txt" 

drive_path_FFT = "drive/My Drive/Colab Notebooks/FFT_Variance_ANN_Tasarim_Donemi/25_NORM_FFT.txt" 
drive_path_VAR = "drive/My Drive/Colab Notebooks/FFT_Variance_ANN_Tasarim_Donemi/Variance.txt" 
drive_path_IMGPATHS = "drive/My Drive/Colab Notebooks/FFT_Variance_ANN_Tasarim_Donemi/Image_Paths.txt" 

with open(drive_path_FFT, "r") as fileFFT:
    fftValues = [line.split() for line in fileFFT]

with open(drive_path_VAR, "r") as fileVar:
    for line in fileVar:
        varianceValues.append(line.split()[0])

with open(drive_path_IMGPATHS, "r") as filePaths:
    for line in filePaths:
        imagePaths.append(line.split('\\')[-1])

fileFFT.close()
fileVar.close()
filePaths.close()

fftValues = np.array(fftValues)                          
fftValues = fftValues.astype('float')

varianceValues = np.array(varianceValues)                          
varianceValues = varianceValues.astype('float')

filePaths = np.array(filePaths)

print(type(fftValues[0][0]))
#%% varianceValues tags
# varianceValues labeled with images index

labeledVarianceData = np.arange(len(varianceValues))
labeledVarianceData = np.vstack((varianceValues, labeledVarianceData)).T
#%% 
from sklearn.model_selection import train_test_split

x_train, x_test, y_train, y_test = train_test_split(fftValues, labeledVarianceData, test_size=0.1, random_state=0)

print("Total images: " + str(len(varianceValues)))
print("Total train images: " + str(y_train.T[0].shape[0]))
print("Total test images: " + str(y_test.T[0].shape[0]))

#%%

model = createModel()
history = model.fit(x_train, y_train.T[0], epochs=1000, batch_size=25)

#%%

showAccuracy(history)
print(model.evaluate(x_test, y_test.T[0]))

#%% y_test_predict print

y_test_predict = model.predict(x_test)

print(" ----Predict----      -------Real-------    -----Images Tags - Paths----- ")
for i in range(len(y_test_predict)):
      print( str(i) + " : "+ str(y_test_predict[i]) + "              " + str(y_test.T[0][i]) + "             " + str(int(y_test.T[1][i])) + " - " + imagePaths[int(y_test.T[1][i])])

#%% model save .json

model_json = model.to_json()

with open("drive/My Drive/Colab Notebooks/FFT_Variance_ANN_Tasarim_Donemi/model_100_25_FFT.json", "w") as json_file:
    json_file.write(model_json)

# serialize weights to HDF5
model.save_weights("drive/My Drive/Colab Notebooks/FFT_Variance_ANN_Tasarim_Donemi/model_100_25_FFT.h5")


#%% model load

json_file = open('drive/My Drive/Colab Notebooks/FFT_Variance_ANN_Tasarim_Donemi/model_100_25_FFT.json', 'r')

loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)

loaded_model.load_weights("drive/My Drive/Colab Notebooks/FFT_Variance_ANN_Tasarim_Donemi/model_100_25_FFT.h5")


#%% loaded_model predict

loaded_model_predict = loaded_model.predict(x_test)

print(" ----Predict----      -------Real-------    -----Images Tags - Paths----- ")
for i in range(len(loaded_model_predict)):
     print( str(i) + " : "+ str(loaded_model_predict[i]) + "      " + str(y_test.T[0][i]) + "         " + str(int(y_test.T[1][i])) + " - " + imagePaths[int(y_test.T[1][i])])


#%% print min and max values

print("Min value:" + str(np.amin(loaded_model_predict)) + " index: " + str(np.argmin(loaded_model_predict, axis=0)))
print("Max value:" + str(np.amax(loaded_model_predict)) + " index: " + str(np.argmax(loaded_model_predict, axis=0)))<jupyter_output>----Predict----      -------Real-------    -----Images Tags - Paths----- 
0 : [0.61262256]      0.6         681 - test7_205.bmp

1 : [0.11936939]      0.1         764 - test7_288.bmp

2 : [0.8734704]      0.85         214 - 10X80_63.bmp

3 : [0.23797113]      0.25         14 - 10X70_14.bmp

4 : [0.69830555]      0.7         187 - 10X80_36.bmp

5 : [0.11160514]      0.1         761 - test7_285.bmp

6 : [0.6892598]      0.7         79 - 10X70_79.bmp

7 : [0.586354]      0.6         680 - test7_204.bmp

8 : [0.83297193]      0.9         643 - test7_167.bmp

9 : [0.52501816]      0.5         531 - test7_55.bmp

10 : [0.33242023]      0.35         31 - 10X70_31.bmp

11 : [0.8961381]      0.9         235 - 10X80_84.bmp

12 : [0.9191019]      0.9         392 - 20X80_51.bmp

13 : [0.85558033]      0.85         583 - test7_107.bmp

14 : [0.14860907]      0.15         739 - test7_263.bmp

15 : [0.81825924]      0.8         204 - 10X80_53.bmp

16 : [0.7179094]      0.75         85 - 10X70_85.bmp[...]
{'fraction_non_alphanumeric': 0.14578931482040447, 'fraction_numerical': 0.06715967401146997, 'mean_word_length': 3.1496556042579837, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '138926', 'license': 'MIT', 'n_tokens_mistral': 2732, 'n_tokens_neox': 2469, 'n_words': 417, 'path': 'FFT_Variance_ANN_Keras_model.ipynb', 'repo_name': 'fbasatemur/Image_Quality_Assessment_with_ANN', 'size': '6626'}