<jupyter_start><jupyter_text>Importowanie Biblotek<jupyter_code>import pandas as pd
import numpy as np
import os
import datetime
import tensorflow as tf

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPool2D, Dense, Flatten, Dropout
from tensorflow.keras.utils import to_categorical

%load_ext  tensorboard

import matplotlib.pyplot as plt
from skimage import color, exposure

from sklearn.metrics import accuracy_score<jupyter_output><empty_output><jupyter_text>Wczytanie danych<jupyter_code>cd 'drive/My Drive/Colab Notebooks/matrix/dw_matrix_road_sign'
train = pd.read_pickle('data/train.p')
test = pd.read_pickle('data/test.p')

X_train, y_train = train['features'], train['labels']
X_test, y_test = test['features'], test['labels']
y_train
to_categorical( y_train )[0]
 if (y_train.ndim == 1):
     y_train = to_categorical(y_train)

 if (y_test.ndim == 1):
     y_test = to_categorical(y_test)
input_shape = X_train.shape[1:]
num_classes = y_train.shape[1]
model = Sequential([
    Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu', input_shape = input_shape ),
    
    Flatten(),
    Dense(num_classes, activation = 'softmax'),
])

# model.summary()

model.compile(loss = 'categorical_crossentropy', optimizer = 'Adam', metrics = ['accuracy'])
model.fit(X_train, y_train)
def get_ccn_v1(input_shape, num_classes):
    return Sequential([
        Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu', input_shape = input_shape ),
        
        Flatten(),
        Dense(num_classes, activation = 'softmax'),
    ])

def train_model(model, X_train, y_train, params_fit = {}):
    model.compile(loss = 'categorical_crossentropy', optimizer = 'Adam', metrics = ['accuracy'])

    logdir = os.path.join("logs", datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
    tensorboard_callback = tf.keras.callbacks.TensorBoard(logdir, histogram_freq = 1)
    model.fit(
        X_train, 
        y_train,
        batch_size = params_fit.get('batch_size', 128),
        epochs = params_fit.get('epochs', 5),
        verbose = params_fit.get('verbose', 1),
        validation_data = params_fit.get('validation_data', (X_train, y_train)),
        callbacks = [ tensorboard_callback ]
    )

    return model

model = get_ccn_v1(input_shape, num_classes)
model_trained = train_model(model, X_train, y_train)
df = pd.read_csv('data/signnames.csv')
labels_dict = df.to_dict()['b']
def predict(model_trained, X_test, y_test, scoring = accuracy_score):
    y_test_norm = np.argmax(y_test, axis = 1)

    y_pred_prob = model_trained.predict(X_test)
    y_pred = np.argmax( y_pred_prob, axis = 1)

    return scoring(y_test_norm, y_pred)

predict(model_trained, X_test, y_test)
def train_and_predict(model):
    model_trained = train_model(model, X_train, y_train)

    return predict(model_trained, X_test, y_test)
def get_ccn_v2(input_shape, num_classes):
    return Sequential([
        Conv2D(filters = 32, kernel_size = (3, 3), activation = 'relu', input_shape = input_shape ),
        MaxPool2D(),
        Dropout(0.3),

        Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu'),
        MaxPool2D(),
        Dropout(0.3),

        Flatten(),
        
        Dense(1024, activation = 'relu'),
        Dropout(0.3),

        Dense(num_classes, activation = 'softmax'),
    ])


train_and_predict( get_ccn_v2(input_shape, num_classes) )
def get_ccn_v3(input_shape, num_classes):
    return Sequential([
        Conv2D(filters = 32, kernel_size = (3, 3), activation = 'relu', input_shape = input_shape ),
        Conv2D(filters = 32, kernel_size = (3, 3), activation = 'relu'),
        MaxPool2D(),
        Dropout(0.3),

        Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu'),
        Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu'),
        MaxPool2D(),
        Dropout(0.3),

        Flatten(),
        
        Dense(1024, activation = 'relu'),
        Dropout(0.3),

        Dense(num_classes, activation = 'softmax'),
    ])


train_and_predict( get_ccn_v3(input_shape, num_classes) )
def get_ccn_v4(input_shape, num_classes):
    return Sequential([
        Conv2D(filters = 32, kernel_size = (3, 3), activation = 'relu', input_shape = input_shape ),
        Conv2D(filters = 32, kernel_size = (3, 3), activation = 'relu', padding = 'same'),
        MaxPool2D(),
        Dropout(0.3),

        Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu', padding = 'same'),
        Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu'),
        MaxPool2D(),
        Dropout(0.3),

        Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu', padding = 'same'),
        Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu'),
        MaxPool2D(),
        Dropout(0.3),

        Flatten(),
        
        Dense(1024, activation = 'relu'),
        Dropout(0.3),

        Dense(num_classes, activation = 'softmax'),
    ])

train_and_predict( get_ccn_v4(input_shape, num_classes) )
def get_ccn_v5(input_shape, num_classes):
    return Sequential([
        Conv2D(filters = 32, kernel_size = (3, 3), activation = 'relu', input_shape = input_shape ),
        Conv2D(filters = 32, kernel_size = (3, 3), activation = 'relu', padding = 'same'),
        MaxPool2D(),
        Dropout(0.3),

        Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu', padding = 'same'),
        Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu'),
        MaxPool2D(),
        Dropout(0.3),

        Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu', padding = 'same'),
        Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu'),
        MaxPool2D(),
        Dropout(0.3),

        Flatten(),
        
        Dense(1024, activation = 'relu'),
        Dropout(0.3),
        
        Dense(1024, activation = 'relu'),
        Dropout(0.3),

        Dense(num_classes, activation = 'softmax'),
    ])

train_and_predict( get_ccn_v5(input_shape, num_classes) )
X_train_gray = color.rgb2gray(X_train).reshape(-1, 32, 32, 1)
X_test_gray = color.rgb2gray(X_test).reshape(-1, 32, 32, 1)
model = get_ccn_v5((32, 32, 1), num_classes)
model_trained =  train_model(model, X_train_gray, y_train, params_fit = {})
predict(model_trained, X_test_gray, y_test)
plt.imshow( color.rgb2gray(X_train[0]), cmap = plt.get_cmap('gray') )
plt.imshow(X_test[400])
!git add<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1407751937984496, 'fraction_numerical': 0.03255813953488372, 'mean_word_length': 3.514345696291113, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '304527', 'license': 'MIT', 'n_tokens_mistral': 2429, 'n_tokens_neox': 2277, 'n_words': 470, 'path': 'dw_matrix_three/day4.ipynb', 'repo_name': 'jedrzejd/dw_matrix_road_sign', 'size': '6450'}