<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')
!cd /content/drive/My Drive/Colab Notebooks/images

! ls
cd 'My Drive'
cd images/ 
ls<jupyter_output><empty_output><jupyter_text>**Data Cleaning**<jupyter_code># example of loading an image with the Keras API
from keras.preprocessing.image import load_img, img_to_array, array_to_img
import numpy as np
import tensorflow as tf
import pathlib
import glob
import pandas as pd
target = '*'
NORMAL = glob.glob('//content/drive/My Drive/Colab Notebooks/images/Normal/' + target, recursive = True)
FIRE = glob.glob('//content/drive/My Drive/Colab Notebooks/images/Fire/' + target, recursive = True)
norm_count = len(NORMAL)
fire_cout= len(FIRE)


print(norm_count)
print(fire_cout)
image_data = []

for path in NORMAL:
    img=load_img(path).resize((400,400))
    image_data.append((img_to_array(img).reshape(400,400,3), 0))
    
for path in FIRE:
    img=load_img(path).resize((400,400))
    image_data.append((img_to_array(img).reshape(400,400,3), 1))
image_data[0][0].shape
df = pd.DataFrame(image_data)
df.columns = ["img_data", 'target']

df.tail()

df.iloc[90,0].shape<jupyter_output><empty_output><jupyter_text>**Data Processing**<jupyter_code># Import all modules
import time
import matplotlib.pyplot as plt
import numpy as np
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten

from keras.constraints import maxnorm
from keras.optimizers import SGD
from keras.layers import Activation
from keras.layers.normalization import BatchNormalization
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras.utils import np_utils
from sklearn.metrics import accuracy_score
from keras import backend as K
if K.backend()=='tensorflow':
    K.set_image_dim_ordering("th")
    
from sklearn.model_selection import train_test_split
from keras.utils import to_categorical


# Import Tensorflow with multiprocessing
import tensorflow as tf
# import multiprocessing as mp

# # Loading the CIFAR-10 datasets
# from keras.datasets import cifar10

# fix random seed for reproducibility
np.random.seed(1010)
X_train, X_test, y_train, y_test = train_test_split(df['img_data'], df['target'], test_size=0.33, random_state=42)<jupyter_output><empty_output><jupyter_text>The output variables are one-hot-encoded as a vector of integers from 0 to 1 for each class. This transforms the output into into a binary matrix in order to best model the classification problem of sorting images into 10 "buckets". We know there are 10 classes for this problem, so we can expect the binary matrix to have a width of 10.<jupyter_code>y_train = to_categorical(y_train, num_classes=2)
y_test = to_categorical(y_test, num_classes=2)
X_train[0]
# X_train = X_train.astype('float32')
# X_test = X_test.astype('float32')

X_train  /= 255
X_test /= 255
shaper = X_train[0].shape
print(shaper)
num_classes =2
import keras
batch_size =16
epochs =10
print(X_train[0].shape)
print(y_train[0])
import numpy as np<jupyter_output>[1. 0.]<jupyter_text>**Create Model 1**<jupyter_code>model = Sequential()

model.add(Conv2D(32, kernel_size=(3, 3),
                 activation='relu',
                 input_shape=(400, 400, 3)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))

model.compile(loss=keras.losses.categorical_crossentropy,
              optimizer=keras.optimizers.Adadelta(),
              metrics=['accuracy'])

model.fit(np.array([X_train[0]]), np.array([y_train[0]]),
          batch_size=batch_size,
          epochs=epochs,
          verbose=1)
# score = model.evaluate(X_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
model.summary()<jupyter_output><empty_output><jupyter_text>**Second Attempt at a model**<jupyter_code>import numpy as np

#Split df into train and test based on label column
df_train
df_test

def data_gen(df, batch_size):
    while True:
        x_batch = np.zeros((batch_size, 3, 1024, 1024))
        y_batch = np.zeros((batch_size, 1))
        for j in range(len(df/batch_size)):
            b = 0
            for m, k in zip(df['filename'].values[j*batch_size:(j+1)*batch_size], df['label'].values[j*batch_size:(j+1)*batch_size]):
                img = Image.open('{}/{}'.format(df['Folder'][b], m))
                image_red = img.resize((1024, 1024))
                x_batch[b] = img_to_array(image_red)
                y_batch[b] = k
                b += 1
            yield (x_batch, y_batch)


model.fit_generator(generator=data_gen(df_train, batch_size=batch_size), steps_per_epoch=len(df_train) // batch_size, epochs=epochs)
target = '*'<jupyter_output><empty_output><jupyter_text>***Milad's Way... Please work!!***<jupyter_code>import glob
import pandas as pd
import numpy as np
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
import keras
from PIL import Image
from keras.preprocessing.image import img_to_array
target = '*'
NORMAL = glob.glob('//content/drive/My Drive/Colab Notebooks/images/Normal/' + target, recursive = True)
FIRE = glob.glob('//content/drive/My Drive/Colab Notebooks/images/Fire/' + target, recursive = True)
norm_count = len(NORMAL)
fire_cout= len(FIRE)


print(norm_count)
print(fire_cout)

ls_fire = []
for i in FIRE:
    ls_fire.append(['Fire', i.split("/")[-1], '1'])
pwd
ls_fire[0]
ls_normal = []
for i in NORMAL:
    ls_normal.append(['Normal', i.split("/")[-1], '1'])
ls_all = ls_fire + ls_normal


len(ls_fire)
len(ls_normal)
ls_all[:10]
df_fire = pd.DataFrame(ls_fire, columns=['Folder', 'filename', 'label'])
df_normal = pd.DataFrame(ls_normal, columns=['Folder', 'filename', 'label'])
df_all = pd.DataFrame(ls_all, columns=['Folder', 'filename', 'label'])
input_shape = (128, 128, 3)
print(input_shape)
num_classes = 1
batch_size = 12
epochs = 10
def data_gen(df, batch_size):
    while True:
        x_batch = np.zeros((batch_size, 128, 128, 3))
        y_batch = np.zeros((batch_size, 1))
        for j in range(int(len(df)/batch_size)):
            b = 0
            for m, k, n in zip(df['filename'].values[j*batch_size:(j+1)*batch_size], df['label'].values[j*batch_size:(j+1)*batch_size], df['Folder'].values[j*batch_size:(j+1)*batch_size]):
                img = Image.open('{}/{}'.format(n, m)).convert("RGB")
#                 print(m)
                image_red = img.resize((128, 128))
                X = img_to_array(image_red)
                X = np.array(X)
                X = X.astype('float32')
                X /= 255
                x_batch[b] = X
                if k == '1':
                	y_batch[b] = 1.0
                b += 1
            yield (x_batch, y_batch)
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3),
                 activation='relu',
                 input_shape=input_shape))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='sigmoid'))

model.compile(loss=keras.losses.binary_crossentropy,
              optimizer=keras.optimizers.Adadelta(),
              metrics=['accuracy'])

model.fit_generator(generator=data_gen(df_all, batch_size=batch_size), steps_per_epoch=len(df_all) // batch_size,
                    epochs=epochs, verbose=1)

model_2 = Sequential()
# Changed Drop out rate and densityh


model_2.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape))
model_2.add(Conv2D(32, (3, 3), activation='relu'))
model_2.add(MaxPooling2D(pool_size=(2, 2)))
# model_2.add(Dropout(0.25))
model_2.add(Flatten())
model_2.add(Dense(32, activation='relu'))
# model_2.add(Dropout(0.25))
model_2.add(Dense(num_classes, activation='sigmoid'))

model_2.compile(loss=keras.losses.binary_crossentropy,
              optimizer=keras.optimizers.Adadelta(),
              metrics=['accuracy'])
model_2.fit_generator(generator=data_gen(df_all, batch_size=batch_size), steps_per_epoch=len(df_all) // batch_size,
                    epochs=epochs, verbose=1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13963328631875882, 'fraction_numerical': 0.028913963328631876, 'mean_word_length': 4.334796238244514, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '558507', 'license': 'MIT', 'n_tokens_mistral': 3176, 'n_tokens_neox': 2995, 'n_words': 631, 'path': 'final_project_work.ipynb', 'repo_name': 'capt-alien/100-pandas-puzzles', 'size': '8508'}