<jupyter_start><jupyter_text>Step 2: Pre-process the Data Set (normalization, grayscale, etc.)  Shuffle data<jupyter_code>X_train, y_train = shuffle(X_train, y_train)<jupyter_output><empty_output><jupyter_text>HIstogram color equilibration<jupyter_code>def hisEqulColor(img):
    ycrcb=cv2.cvtColor(img,cv2.COLOR_RGB2YCR_CB)
    channels=cv2.split(ycrcb)
    cv2.equalizeHist(channels[0],channels[0])
    cv2.merge(channels,ycrcb)
    cv2.cvtColor(ycrcb,cv2.COLOR_YCR_CB2RGB,img)
    return img<jupyter_output><empty_output><jupyter_text>Contrast correction<jupyter_code>def cv2contrast_normalize(x_in):
    dst = np.zeros(x_in.shape) # setup destination matrix shape
    
    # Use list comprehension, adjust each image contrast to be between [0, 255]
    l = [cv2.normalize(x, dst, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8U) for x in x_in]
    
    return np.asarray(l)<jupyter_output><empty_output><jupyter_text>Color enhancement preview<jupyter_code>plt.figure(figsize=(22, 12))

signnames = [row[1] for row in csv.reader(open("signnames.csv"))][1:]

for i in range(0,24,2):
    n = np.random.randint(globalData['n_clasees'])
    
    ## Original
    plt.subplot(5, 8, 1+i)
    plt.yticks([])
    plt.xticks([])
    plt.imshow(X_train[n], interpolation='nearest')
    plt.xlabel(signnames[y_train[n]])
    
    ## Mod
    plt.subplot(5, 8, 2+i)
    plt.yticks([])
    plt.xticks([])

    plt.xlabel(signnames[y_train[n]])
    plt.imshow(hisEqulColor(cv2contrast_normalize(X_train[n])), interpolation='nearest')
    plt.xlabel('Enhanced')
    
print("Image enhancement:")

# Helper to adapt images to full width
plt.show()
X_train =  np.array([hisEqulColor(cv2contrast_normalize(xi)) for xi in X_train])
X_valid = np.array([hisEqulColor(cv2contrast_normalize(xi)) for xi in X_valid])
X_test = np.array([hisEqulColor(cv2contrast_normalize(xi)) for xi in X_test])<jupyter_output><empty_output><jupyter_text>Normalize images [-1, 1] The data should be normalized so that the data has mean zero and equal variance. For image data, `(pixel - 128)/ 128` is a quick way to approximately normalize the data and can be used in this project.<jupyter_code>X_train = np.divide(np.add(X_train, -128),128)
X_test = np.divide(np.add(X_test, -128),128)
X_valid = np.divide(np.add(X_valid, -128),128)
pickle.dump( [X_train, y_train, X_valid, y_valid, X_test, y_test], open( "pickle_data/prep_data.p", "wb" ) )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13701431492842536, 'fraction_numerical': 0.02903885480572597, 'mean_word_length': 5.522666666666667, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '456668', 'license': 'MIT', 'n_tokens_mistral': 1003, 'n_tokens_neox': 947, 'n_words': 178, 'path': '2. Preprocesing data.ipynb', 'repo_name': 'fizcris/CarND-Traffic-Sign-Classifier-Project', 'size': '2445'}