<jupyter_start><jupyter_text>Using Tf to form a model<jupyter_code>import pandas as pd
import tensorflow as tf
import tensorflow.keras as keras
import numpy as np
import cv2
import os
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
# from tensorflow.keras.models import Sequential
# from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D
# from tensorflow.keras.optimizers import Adam
from tensorflow.keras.applications import VGG16
test_dataset = pd.read_csv("data/test.csv")
train_dataset = pd.read_csv("data/train.csv")
columns = ["img_paths", "labels"]
test_dataset = test_dataset.sample(frac=1)
train_dataset = train_dataset.sample(frac=1)
# test_dataset.drop(columns=["Unnamed: 0"], inplace=True)
# train_dataset.drop(columns=["Unnamed: 0"], inplace=True)
test_dataset.reset_index(inplace=True, drop=True)
train_dataset.reset_index(inplace=True, drop=True)
img = cv2.imread("data/data/test/0/103277.png", cv2.IMREAD_GRAYSCALE)
img.shape
import time
def load_dataset(df):   
    ##############Getting the image Paths#################
#     df = pd.read_csv(path)
    img_paths = df.img_paths
    labels = df.labels
    ######################################################
    ##############Reading the images Grayscale############
    imgs = []
    for img in img_paths:
#         start = time.time()
        temp = cv2.imread(img, cv2.IMREAD_GRAYSCALE)
        imgs.append(temp)
#         end = time.time()
#         print(end-start)
        
    ######################################################
    #######Converting to numpy array for faster calculations#####
    labels = np.asarray(labels).astype('uint8')
    imgs = np.asarray(imgs)
    return labels, imgs
y_train, x_train = load_dataset(train_dataset)
y_test, x_test = load_dataset(test_dataset)
import pickle
filenames = ["dump/x_train.txt","dump/x_test.txt", "dump/y_train.txt","dump/y_test.txt"]
listings = [x_train, x_test, y_train, y_test]
for listing, filename in zip(listings, filenames):
    f = open(filename, 'wb')
    pickle.dump(listing, f)<jupyter_output><empty_output><jupyter_text>Load Data (Alternative starting point)<jupyter_code>import pandas as pd
import tensorflow as tf
import tensorflow.keras as keras
import numpy as np
import cv2
import os
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from tensorflow.keras.models import Sequential
from tensorflow.keras import layers
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.applications import VGG16
from tensorflow.keras.models import Model
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import LabelEncoder 
import pickle
filenames = ["dump/x_train.txt","dump/x_test.txt", "dump/y_train.txt","dump/y_test.txt"]
f = open(filenames[0], 'rb')
x_train  = pickle.load(f)
f = open(filenames[1], 'rb')
x_test = pickle.load(f)
f = open(filenames[2], 'rb')
y_train = pickle.load(f)
f = open(filenames[3], 'rb')
y_test = pickle.load(f)
def oneHotEncode(data):
    # binary encode
    onehot_encoder = OneHotEncoder(sparse=False) # disable sparse return type
    
    # reshape the array
    data = data.reshape(len(data), 1)
    data = onehot_encoder.fit_transform(data) 
    return data
y_test = oneHotEncode(y_test)
y_train = oneHotEncode(y_train)
x_train = x_train.reshape(-1, 32, 32, 1)
x_test = x_test.reshape(-1, 32, 32, 1)
x_test = x_test/255
x_train = x_train/255
np.min(x_test)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>def create_model(img_height, img_width, img_ch, num_classes):
    model = Sequential([
#         layers.experimental.preprocessing.Rescaling(1./255, input_shape=(img_height, img_width, img_ch)),
        layers.Conv2D(16, 3,input_shape=(img_height, img_width, img_ch), padding='same', activation='relu'),
        layers.MaxPooling2D(),
        layers.Conv2D(32, 3, padding='same', activation='relu'),
        layers.MaxPooling2D(),
        layers.Conv2D(64, 3, padding='same', activation='relu'),
        layers.MaxPooling2D(),
        layers.Flatten(),
        layers.Dense(128, activation='relu'),
        layers.Dense(num_classes)        
    ])
    model.compile(loss='mse', optimizer=Adam(learning_rate=0.0001), metrics=['accuracy'])
    return model

def create_model2(img_height, img_width, img_ch, num_classes):
    model = Sequential()
    model.add(Conv2D(32, kernel_size=(3, 3),
                     activation='relu',
                     input_shape=(img_height, img_width, img_ch)))
    model.add(Conv2D(64, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))
    model.add(Flatten())
    model.add(Dense(128, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(num_classes, activation='softmax'))
    
    model.compile(loss='mse', optimizer=Adam(learning_rate=0.0001), metrics=['accuracy'])
    return model
model = create_model(32,32,1,46)
model = create_model2(32,32,1,46)
model.summary()
checkpoint_cb = keras.callbacks.ModelCheckpoint("Conv2d.h5", save_best_only=True)
early_stopping_cb = keras.callbacks.EarlyStopping(patience=5, restore_best_weights=True)
history = model.fit(x_train, y_train, validation_data=(x_test, y_test), batch_size=100, epochs=5)
plt.imshow(x_train[0])
# plt.imshow(y_train[0])
np.argmax(y_train[0])<jupyter_output><empty_output><jupyter_text>Visualizing the model<jupyter_code>acc = history.history['accuracy']
val_acc = history.history['val_accuracy']

loss = history.history['loss']
val_loss = history.history['val_loss']

epochs_range = range(100)

plt.figure(figsize=(8, 8))
plt.subplot(1, 2, 1)
plt.plot(epochs_range, acc, label='Training Accuracy')
plt.plot(epochs_range, val_acc, label='Validation Accuracy')
plt.legend(loc='lower right')
plt.title('Training and Validation Accuracy')

plt.subplot(1, 2, 2)
plt.plot(epochs_range, loss, label='Training Loss')
plt.plot(epochs_range, val_loss, label='Validation Loss')
plt.legend(loc='upper right')
plt.title('Training and Validation Loss')
plt.show()<jupyter_output><empty_output><jupyter_text>Saving Model<jupyter_code>model.save("models/Conv2d.h5")
del model<jupyter_output><empty_output><jupyter_text>Loading the model<jupyter_code>model = tf.keras.models.load_model("models/Conv2d.h5")
import tensorflowjs as tfjs

tfjs.converters.save_keras_model(model, "model_js")
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])<jupyter_output>Test loss: 0.002279395703226328
Test accuracy: 0.9288405776023865
{'fraction_non_alphanumeric': 0.15142428785607195, 'fraction_numerical': 0.027286356821589204, 'mean_word_length': 5.546614327772326, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '173256', 'license': 'MIT', 'n_tokens_mistral': 2459, 'n_tokens_neox': 2324, 'n_words': 448, 'path': 'Model.ipynb', 'repo_name': 'Siddharthm10/ClientSideDevnagriPrediction', 'size': '6670'}