<jupyter_start><jupyter_text>Imports and Setup<jupyter_code>%matplotlib inline
import os
from os.path import join

import numpy as np
import pandas as pd
import dask.dataframe as dd
import matplotlib.pyplot as plt
import cv2

import tensorflow as tf

from pathlib import Path
from tensorflow.keras import layers, Input, models
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier 
from sklearn.model_selection import KFold 
from sklearn.model_selection import cross_val_score

from sklearn.model_selection import train_test_split

import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings("ignore")
root_path = Path('/home/ec2-user/SageMaker/defect_detection/')#.resolve()

data_path = root_path / "data/MIR-WM811K/"<jupyter_output><empty_output><jupyter_text>Data Load<jupyter_code>input_df = dd.from_pandas(
    pd.read_pickle(
        data_path / "LSWMD.pkl"
    ).astype(
        {"waferIndex": "int32"}
    ),
    npartitions=100)
input_df.info(verbose=True)
input_df.head()<jupyter_output><empty_output><jupyter_text>Data Analysis<jupyter_code>def plot_wafer_index_distr(data, y_min=31000, y_max=33000, figsize=None):
    figsize = figsize if figsize is not None else (15, 10)
    plot_data = data.groupby('waferIndex').size().compute()
    ax = plot_data.plot.bar(align='center', figsize=figsize)
    ax.figure.suptitle(" wafer Index distribution")
    ax.set_xlabel("index #")
    ax.set_ylabel("frequency")
    ax.axhline(y=plot_data.mean(), color='red', label='mean')
    ax.set_ylim(y_min,y_max)
    return ax
plot_wafer_index_distr(input_df);<jupyter_output><empty_output><jupyter_text>Data Transformation  Cleansing<jupyter_code>clean_df = input_df.drop('waferIndex', axis=1)
clean_df['waferMapDim'] = clean_df.waferMap.apply(lambda x: x.shape, meta=pd.Series({'waferMapDim': [(0, 0)]}))
clean_df[(clean_df.waferMapDim == (26, 26)) & (clean_df.failureType.apply(lambda x: len(x) > 0, meta=pd.Series({'x': [True]})))]<jupyter_output><empty_output><jupyter_text>Drop all maps with unequal dimensions, to avoid distortion when resizing<jupyter_code>clean_df = clean_df[clean_df.waferMapDim.apply(lambda x: x[0] == x[1])]<jupyter_output><empty_output><jupyter_text>Drop all unlabeled maps<jupyter_code>clean_df['label'] = clean_df.failureType.apply(lambda x: x[0, 0] if (isinstance(x, np.ndarray) and x.shape[0] > 0) else 'unknown')
clean_df = clean_df[clean_df.label != "unknown"]<jupyter_output><empty_output><jupyter_text>Generate X and y<jupyter_code>x = np.stack(clean_df.apply(
    lambda x: x.waferMap.reshape(
        (1, 26, 26)
    ) if x.waferMapDim[0] == 26 else cv2.resize(
        x.waferMap.reshape(x.waferMapDim[0], x.waferMapDim[1]), (26, 26)
    ).reshape(1, 26, 26), axis=1, meta=pd.Series({'x': [np.zeros((1, 26, 26))]})).compute().values)
y = clean_df.label.compute().values
print(f'x shape : {x.shape}, y shape : {y.shape}')<jupyter_output>x shape : (22894, 1, 26, 26), y shape : (22894,)<jupyter_text>Plot a sample of the results so far<jupyter_code>label_dist = clean_df.groupby('label').size().compute()
inv_prob_label = {k: v for (k, v) in ((1/(label_dist/label_dist.sum()) )/((1/(label_dist/label_dist.sum()) ).sum())).iteritems()}
fig, ax = plt.subplots(nrows=10, ncols=10, figsize=(20, 20))
ax = ax.ravel(order="C")
for i, img_index in enumerate(np.random.choice(list(i for i, _ in enumerate(y)), 100, [inv_prob_label[label] for label in y])):
    img = x[img_index][0]
    ax[i].imshow(img)
    ax[i].set_title(f"{i}: {y[img_index]}", fontsize=12)
fig.tight_layout()
fig.show()<jupyter_output><empty_output><jupyter_text>Even with upsampling, the good wafers completely dominage the picture - the bar chart below shows why. While that's good for business, a good model needs more balanced classes.<jupyter_code>label_dist.plot.bar(title="Examples per defect class", figsize=(15, 10));
x = x.reshape((-1, 26, 26, 1))
def hot_encode(img_arr):
    new_arr = np.zeros((676, 3))
    for x in range(676):
        new_arr[x, img_arr[x]] = 1
    return new_arr.reshape((26, 26, 3))
    
new_x = np.apply_along_axis(hot_encode, axis=1, arr=x.reshape(-1, 26 * 26))<jupyter_output><empty_output><jupyter_text>Data Augmentation<jupyter_code>def create_autoencoder(data, epochs, batch_size):
    input_shape = (26, 26, 3)
    input_tensor = Input(input_shape)
    encode = layers.Conv2D(64, (3,3), padding='same', activation='relu')(input_tensor)

    latent_vector = layers.MaxPool2D()(encode)

    # Decoder
    decode_layer_1 = layers.Conv2DTranspose(64, (3,3), padding='same', activation='relu')
    decode_layer_2 = layers.UpSampling2D()
    output_tensor = layers.Conv2DTranspose(3, (3,3), padding='same', activation='sigmoid')

    # connect decoder layers
    decode = decode_layer_1(latent_vector)
    decode = decode_layer_2(decode)

    ae = models.Model(input_tensor, output_tensor(decode))
    ae.compile(optimizer = 'Adam',
                  loss = 'mse',
                 )
#     ae.fit(data, data,
#        batch_size=batch_size,
#        epochs=epochs,
#        verbose=2)
    
    decoder_input = Input((13, 13, 64))
    decoder = decode_layer_1(decoder_input)
    decoder = decode_layer_2(decoder)
    
    return ae, models.Model(input_tensor, latent_vector), models.Model(decoder_input, output_tensor(decoder))
ae, encoder, decoder = create_autoencoder(new_x, epochs=15, batch_size=1024)
new_x.shape
encoded_x = encoder.predict(new_x)
noised_encoded_x = encoded_x + np.random.normal(loc=0, scale=0.1, size = (len(encoded_x), 13, 13, 64))
fig, ax = plt.subplots(nrows=1, ncols=3, figsize=(21, 7))

ax[0].imshow(np.argmax(new_x[3], axis=2));
ax[0].set_title("Original map")
noised_gen_x = np.argmax(decoder.predict(noised_encoded_x), axis=3)
ax[1].imshow(noised_gen_x[3])
ax[1].set_title("Map with Noise")
gen_x = np.argmax(ae.predict(new_x), axis=3)
ax[2].imshow(gen_x[3])
ax[2].set_title("Reconstructed map")
fig.show()
def gen_data(wafer, label, n_examples=2000):
    # Encode input wafer
    print(f"There are {len(wafer)} examples for {label}")
    encoded_x = encoder.predict(wafer)
    
    # dummy array for collecting noised wafer
    
    additional_example_batches = n_examples // wafer.shape[0] + 1
    temp_noised = []
    for i in range(additional_example_batches):
        temp_noised.append(encoded_x + np.random.normal(loc=0, scale=0.1, size = (len(encoded_x), 13, 13, 64)))
    noised_encoded_x = np.vstack(temp_noised)
    gen_x = decoder.predict(noised_encoded_x[1:])
    # also make label vector with same length
    gen_y = np.full((len(gen_x), ), label)

    print(f"Returning {n_examples - len(wafer)} new examples") 
    return gen_x[1:n_examples+1], gen_y[1:n_examples+1]
label_classes = {l: i for (i, l) in enumerate(label_dist.index.values)}

def augment_data(x, y, labels):
    aug_x = x.copy()
    aug_y = y.copy()
    for f in labels: 
        # skip none case
        if f in {'none', 'unknown'} : 
            continue
        else:
            print(f'Generating {f}')

        gen_x, gen_y = gen_data(x[np.where(y==f)[0]], f, 15000)
        aug_x = np.concatenate((aug_x, gen_x), axis=0)
        aug_y = np.concatenate((aug_y, gen_y))
    return aug_x, aug_y
aug_x, aug_y = augment_data(new_x, y, list(label_classes.keys()))<jupyter_output>Generating Center
There are 803 examples for Center
(15257, 13, 13, 64)
Returning 14197 new examples
Generating Donut
There are 14 examples for Donut
(15008, 13, 13, 64)
Returning 14986 new examples
Generating Edge-Loc
There are 680 examples for Edge-Loc
(15640, 13, 13, 64)
Returning 14320 new examples
Generating Edge-Ring
There are 1093 examples for Edge-Ring
(15302, 13, 13, 64)
Returning 13907 new examples
Generating Loc
There are 634 examples for Loc
(15216, 13, 13, 64)
Returning 14366 new examples
Generating Near-full
There are 32 examples for Near-full
(15008, 13, 13, 64)
Returning 14968 new examples
Generating Random
There are 282 examples for Random
(15228, 13, 13, 64)
Returning 14718 new examples
Generating Scratch
There are 165 examples for Scratch
(15015, 13, 13, 64)
Returning 14835 new examples<jupyter_text>Training<jupyter_code>aug_y = to_categorical([label_classes[label] for label in aug_y])

x_train, x_test, y_train, y_test = train_test_split(aug_x, aug_y, test_size=0.33, random_state=42)
def create_model():
    input_shape = (26, 26, 3)
    input_tensor = Input(input_shape)

    conv_1 = layers.Conv2D(16, (3,3), activation='relu', padding='same')(input_tensor)
    conv_2 = layers.Conv2D(64, (3,3), activation='relu', padding='same')(conv_1)
    conv_3 = layers.Conv2D(128, (3,3), activation='relu', padding='same')(conv_2)

    flat = layers.Flatten()(conv_3)

    dense_1 = layers.Dense(512, activation='relu')(flat)
    dense_2 = layers.Dense(128, activation='relu')(dense_1)
    output_tensor = layers.Dense(9, activation='softmax')(dense_2)

    model = models.Model(input_tensor, output_tensor)
    model.compile(optimizer='Adam',
                 loss='categorical_crossentropy',
                 metrics=['accuracy'])

    return model
model = KerasClassifier(build_fn=create_model, epochs=10, batch_size=1024, verbose=2)
kfold = KFold(n_splits=3, shuffle=True, random_state=42)
results = cross_val_score(model, x_train, y_train, cv=kfold)
print(f"CV Score: {np.mean(results)}")
history = model.fit(x_train, y_train,
         validation_data=[x_test, y_test],
         epochs=7,
         batch_size=1024,
         )
# accuracy plot 
import seaborn as sns
sns.set_style(style='darkgrid')
plt.figure(figsize=(15, 10))
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('model accuracy', fontsize=20)
plt.ylabel('accuracy', fontsize=14)
plt.xlabel('epoch', fontsize=14)
plt.legend(['train', 'test'], loc='upper left', fontsize=12)
plt.show()
# loss plot
plt.figure(figsize=(15, 10))
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss', fontsize=20)
plt.ylabel('loss', fontsize=14)
plt.xlabel('epoch', fontsize=14)
plt.legend(['train', 'test'], loc='upper right', fontsize=12)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14199723702388, 'fraction_numerical': 0.04440497335701599, 'mean_word_length': 5.050746268656717, 'pattern_counts': {'":': 1, '<': 43, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '455420', 'license': 'MIT', 'n_tokens_mistral': 4088, 'n_tokens_neox': 3724, 'n_words': 787, 'path': 'notebooks/WM-811K/CNN with augmentation.ipynb', 'repo_name': 'bobbruno/defect_detection', 'size': '10134'}