<jupyter_start><jupyter_text>Read Data set<jupyter_code>#------- Read training images---------------------------------------------------
NUM_CLASSES = 43 # 43 for GTSRB, 164 for European data set
IMAGE_DIMS = (48,48,3)
file_name = 'X_train_german.h5' # File name for saving your training images

try:
    with  h5py.File(file_name) as hf: 
        X, Y = hf['imgs'][:], hf['labels'][:]
    print("Loaded images from {}".format(file_name))
    
except (IOError,OSError, KeyError):  
    print("Error in reading {}. Processing all images...".format(file_name))
    root_dir = '../../Datasets/Traffic_signs/GTSRB/Final_Training/Images' # changed to your needs
    imgs = []
    labels = []

    # Read all image paths with extension ppm
    all_img_paths = sorted(glob.glob(os.path.join(root_dir, '*/*.ppm')) )
    np.random.seed(42)
    np.random.shuffle(all_img_paths)
    
    # Read images and preprocess them
    print("[INFO] loading images...")
    for img_path in all_img_paths:
        try:
            img = preprocess_img(io.imread(img_path))            
            label = get_class(img_path)            
            
            # Save images and labels in lists
            imgs.append(img)
            labels.append(label)

            if len(imgs)%1000 == 0: print("Processed {}/{}".format(len(imgs), len(all_img_paths)))
        except (IOError, OSError):
            print('missed', img_path)
            pass

    X = np.array(imgs, dtype='float32')
    Y = np.eye(NUM_CLASSES, dtype='uint8')[labels]
    # Save the training dictionary
    with h5py.File(file_name,'w') as hf:
        hf.create_dataset('imgs', data=X)
        hf.create_dataset('labels', data=Y)



#--------------- Read test images-------------------------------
file_name = 'X_test_german.h5' # File name for saving your testing images

try:
    with  h5py.File(file_name) as hf: 
        X_test, y_test = hf['imgs'][:], hf['labels'][:]
    print("Loaded images from {}".format(file_name))
    
except (IOError,OSError, KeyError):  
    print("Error in reading {}. Processing all images...".format(file_name))
    root_dir = '../../Datasets/Traffic_signs/GTSRB/Final_Test/Images2'
    X_test = []
    y_test = []

    # Read all image paths with extension ppm
    all_img_paths = sorted(glob.glob(os.path.join(root_dir, '*/*.ppm')) )
    np.random.seed(42)
    np.random.shuffle(all_img_paths)

    # Read images and preprocess them    
    for img_path in all_img_paths:
        try:
            img = preprocess_img(io.imread(img_path))            
            label = get_class(img_path)
            
            # Save images and labels in lists
            X_test.append(img)
            y_test.append(label)

            if len(X_test)%1000 == 0: print("Processed {}/{}".format(len(X_test), len(all_img_paths)))
        except (IOError, OSError):
            print('missed', img_path)
            pass

    X_test = np.array(X_test, dtype='float32')
    y_test = np.array(y_test, dtype='uint8')
    # Save testing dictionary
    with h5py.File(file_name,'w') as hf:
        hf.create_dataset('imgs', data=X_test)
        hf.create_dataset('labels', data=y_test)

# ---------------------- Split dataset into train and validation -----------------------------
# partition the data into training and validation sets using 90% - 10% ratio
split = train_test_split(X, Y, test_size=0.1, random_state=42)
(X_train, X_val, y_train, y_val) = split
# --------------------- Normalize the data ----------------------------------
normalize = 0
# Subtract the mean image
if normalize:#
    mean_image = np.mean(X_train, axis=0)
    X_train -= mean_image
    X_test -= mean_image
    X_val -= mean_image

print('Train data shape: ', X_train.shape)
print('Train classes labels shape: ', y_train.shape)
print('Validation data shape: ', X_val.shape)
print('Validation classes labels shape: ', y_val.shape)
print('Test data shape: ', X_test.shape)
print('Test classes labels shape: ', y_test.shape)<jupyter_output>Train data shape:  (35288, 48, 48, 3)
Train classes labels shape:  (35288, 43)
Validation data shape:  (3921, 48, 48, 3)
Validation classes labels shape:  (3921, 43)
Test data shape:  (12630, 48, 48, 3)
Test classes labels shape:  (12630,)<jupyter_text>CNN 8-layers model<jupyter_code># ----------------- Initialization of weights -------------------------------
from keras import backend as K
import numpy as np

def my_init(shape, name=None):
    value = np.random.random(shape)
    return K.variable(value, name=name)
# --------------- Define Keras model ---------------------------------- 
from model.TS_Net_proposal import TrafficSignsNet

model = TrafficSignsNet.build(48, 48,
    numClasses=43,
    finalAct="softmax")

model.summary() # Print model information
#--------------------- Initialize model ------------------------
EPOCHS = 40
INIT_LR = 1e-3
BS = 128

opt = Adam(lr=INIT_LR, decay=INIT_LR / EPOCHS)

model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

def lr_schedule(epoch):
    return lr * (0.1 ** init(epoch/10))
#---------------------- Train model -----------------------
t1=time()
history = model.fit(X_train, y_train, batch_size=BS, 
                            nb_epoch=EPOCHS,
                            validation_data=(X_val, y_val),
                            shuffle = True
         )
t2 = time()

print ('Time taken to train the model', t2-t1)<jupyter_output>/home/citlalli/.local/lib/python3.5/site-packages/ipykernel_launcher.py:6: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.<jupyter_text>Test on the Test Dataset<jupyter_code>t1=time()
y_proba = model.predict(X_test)
t2=time()
print  ('Took {} sec to predict {} images'.format(t2-t1, len(X_test)) )

pred_labels = []
pred_probas = []
for i in range(0,len(y_proba)):
    classId = y_proba[i].argmax()
    pred_labels.append(classId)
    proba = max(y_proba[i])
    pred_probas.append(proba)

pred_labels = np.array(pred_labels, dtype='uint8')
pred_probas = np.array(pred_probas)

acc = np.mean(pred_labels==y_test)
print("Test accuracy = {}".format(acc*100))
model.save('GTSRB_weights/german_model_98.52.h5')<jupyter_output><empty_output><jupyter_text>Train with data augmentation<jupyter_code>datagen = ImageDataGenerator(featurewise_center=False, 
                            featurewise_std_normalization=False, 
                            width_shift_range=0.1,
                            height_shift_range=0.1,
                            zoom_range=0.2,
                            shear_range=0.1,
                            rotation_range=10.,)
datagen.fit(X_train)

nb_epoch = 50
history2=model.fit_generator(datagen.flow(X_train, y_train, batch_size=BS),
                            samples_per_epoch=X_train.shape[0],
                            nb_epoch=nb_epoch,
                            validation_data=(X_val, y_val),
                            callbacks=[ReduceLROnPlateau('val_loss', factor=0.2, patience=20, verbose=1, mode='auto'), 
                                       ModelCheckpoint('GTSRB_weights/german_aug_test.h5',save_best_only=True)]
                           )

# Evaluate the model
t1=time()
y_proba = model.predict(X_test)
t2=time()
print  ('Took {} sec to predict {} images'.format(t2-t1, len(X_test)) )

pred_labels = []
pred_probas = []
for i in range(0,len(y_proba)):
    classId = y_proba[i].argmax()
    pred_labels.append(classId)
    proba = max(y_proba[i])
    pred_probas.append(proba)

pred_labels = np.array(pred_labels, dtype='uint8')
pred_probas = np.array(pred_probas)

acc = np.mean(pred_labels==y_test)
print("Test accuracy = {}".format(acc*100))
model.save('GTSRB/german_aug-99.37.h5')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17066528336143172, 'fraction_numerical': 0.021916742316171703, 'mean_word_length': 2.433659839715049, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '526506', 'license': 'Apache-2.0', 'n_tokens_mistral': 2638, 'n_tokens_neox': 2516, 'n_words': 529, 'path': 'models/cnn_8-layers_improved/train_test.ipynb', 'repo_name': 'citlag/European-Traffic-Sings', 'size': '7711'}