<jupyter_start><jupyter_text>1. Extract features<jupyter_code>train_datagen = image.ImageDataGenerator()
val_datagen = image.ImageDataGenerator()

train_generator = train_datagen.flow_from_directory(
    directory=train_dir_path,
    target_size=(img_height, img_width),
    seed=seed, 
    follow_links=True
)
val_generator = val_datagen.flow_from_directory(
    directory=val_dir_path,
    target_size=(img_height, img_width),
    seed=seed,
    follow_links=True
)

print("number of classes: {}".format(train_generator.nb_class))

print("number of images for training: {}".format(train_generator.nb_sample))
for i in train_generator:
    x, y = i
    print("shape of train x: {}, y: {}".format(x.shape, y.shape))
    break

# assumption: 151 images, therefore generator will output 32 * 151 images? 
print("number of images for validation: {}".format(val_generator.nb_sample))
model = ResNet50(
            include_top=False,
            weights="imagenet",
            input_tensor=Input(
                shape=(
                    img_height, 
                    img_width, 
                    img_dimension
                )
            )
        )
for layer in model.layers:
    layer.trainable = False
def __extract_features(generator, model, batch_size, set_type):
    feature_dirpath = "./features/"
    print("create a feature directory to store saved features: {}".format(feature_dirpath))
    if not os.path.exists(feature_dirpath):
        os.makedirs(feature_dirpath)

    print("extract features from convolutional model based on data")
    print("generator: {}_generator".format(set_type))
    print("batch_size: {}".format(batch_size))
    features = model.predict_generator(
        generator,
        batch_size
    )

    time_now = datetime.datetime.now().strftime("%Y%m%d_%H_%M_%S")
    features_filepath = feature_dirpath + "REALDEAL_{}_{}_{}_features_{}.npy".format(
        dataset_type,
        conv_model_type,
        set_type,
        time_now
    )
    print("save features to {}".format(features_filepath))
    np.save(open(features_filepath, "wb"), features)

    return features, features_filepath
train_features, train_features_path = __extract_features(train_generator, model, batch_size, "train")
#save features to ./features/REALDEAL_DGCarsDataset_ResNet50_train_features_20170214_23_34_31.npy
train_feature_path = "./features/REALDEAL_DGCarsDataset_ResNet50_train_features_20170214_23_34_31.npy"
train_features = np.load(open(train_feature_path, "rb"))<jupyter_output><empty_output><jupyter_text>2. Train classifier based on features<jupyter_code>checkpoint_dirpath = "./checkpoints/"
print("create a checkpoint directory to store saved checkpoints: {}".format(checkpoint_dirpath))
if not os.path.exists(checkpoint_dirpath):
    os.makedirs(checkpoint_dirpath)

checkpoint_filepath = \
    checkpoint_dirpath + \
    "{}_{}_features_".format(dataset_type, "classifier") + \
    "{epoch:02d}_{val_acc:.8f}.npy"

checkpoint = ModelCheckpoint(
    checkpoint_filepath, 
    monitor="val_acc", 
    save_best_only=True, 
    mode="max"
)
callbacks_list = [checkpoint]
nb_classes = train_generator.nb_class
nb_features = model.output_shape[-1] # same as train_features.shape[-1]
nb_hidden_layers = int(round(np.mean([nb_features, nb_classes])))
print("{} -> [hidden layer {}] -> {}\n".format(nb_features, nb_hidden_layers, nb_classes))

top_model = Sequential()
top_model.add(Dense(nb_hidden_layers, activation="relu", input_shape=train_features.shape[1:]))
top_model.add(Flatten())
top_model.add(Dense(nb_classes, activation="softmax")) 
top_model.compile(
    loss="categorical_crossentropy",
    optimizer="adam",
    metrics=["accuracy"]
)
count = 0
for i in train_generator:
    x, y = i
    count = count + 1
    print("count: {}, x.shape: {}, y.shape: {}".format(count, x.shape, y.shape))
    print("y: {}".format(y))
    break
count = 0
for x in train_features:
    count = count + 1
    print("count: {}, x.shape: {}".format(count, x.shape))
# x.shape: (1, 224, 224, 3), y.shape: (1, 3)
# batch_size = 1
class_dictionary = train_generator.class_indices

for key, value in class_dictionary.items():
    print(key, value)
count = 0
for i in train_generator.classes:
    count = count + 1
    print("count: {}, i: {}".format(count, i))
import threading
class threadsafe_iter:
    """Takes an iterator/generator and makes it thread-safe by
    serializing call to the `next` method of given iterator/generator.
    """
    def __init__(self, it):
        self.it = it
        self.lock = threading.Lock()

    def __iter__(self):
        return self

    def next(self):
        with self.lock:
            return self.it.next()


def threadsafe_generator(f):
    """A decorator that takes a generator function and makes it thread-safe.
    """
    def g(*a, **kw):
        return threadsafe_iter(f(*a, **kw))
    return g

@threadsafe_generator
def __create_generator_from_features(feature, generator):  # write the definition of your data generator
    for feature, class_index in zip(features, generator.classes):
        label = np.zeros(generator.nb_class)
        label[class_index] = 1
        yield (feature, label)
    
    """
    (X_train, y_train), (X_test, y_test) = mnist.load_data()
    y_train = np_utils.to_categorical(y_train,10)
    X_train = X_train.reshape(X_train.shape[0], 1, 28, 28)
    X_test = X_test.reshape(X_test.shape[0], 1, 28, 28)
    X_train = X_train.astype('float32')
    X_test = X_test.astype('float32')
    X_train /= 255
    X_test /= 255
    while 1:
        for i in range(1875):
            yield X_train[i*32:(i+1)*32], y_train[i*32:(i+1)*32]
        # print("Came here")
    """

#@threadsafe_generator
from keras.datasets import mnist
from keras.utils import np_utils
def mygenerator():  # write the definition of your data generator
    (X_train, y_train), (X_test, y_test) = mnist.load_data()
    y_train = np_utils.to_categorical(y_train,10)
    print("y_train.shape: {}".format(y_train.shape))
    X_train = X_train.reshape(X_train.shape[0], 1, 28, 28)
    print("X_train.shape: {}".format(X_train.shape))
    X_test = X_test.reshape(X_test.shape[0], 1, 28, 28)
    X_train = X_train.astype('float32')
    print("X_train.shape: {}".format(X_train.shape))
    X_test = X_test.astype('float32')
    X_train /= 255
    print("X_train.shape: {}".format(X_train.shape))
    X_test /= 255
    while 1:
        for i in range(1875):
            print(i)
            yield X_train[i*32:(i+1)*32], y_train[i*32:(i+1)*32]

count = 0
for i in mygenerator():
    x, y = i
    print("count: {}, x.shape: {}, y.shape: {}".format(count, x.shape, y.shape))
    #print(">> x: {}".format(x))
    #print(">> y: {}".format(y)'break)
    break

        
#train_feature_generator = __create_generator_from_features(train_features, train_generator)

"""
count = 0
for i in train_feature_generator:
    x, y = i
    count = count + 1
    print("count: {}, x.shape: {}, y.shape: {}".format(count, x.shape, y.shape))
    print("type of y: {}".format(type(y)))
"""

labels = np.zeros((194, 3))
count = 0
for i, class_index in zip(range(0, 194), train_generator.classes):
    labels[i][class_index] = 1
    count = count + 1
    #print("count: {}, i: {}, class_index: {}, labels[{}]: {}".format(count, i, class_index, i, labels[i]))
print(train_features[0])
# 194 = train_generator.nb_sample
# 32 = batch_size


def get_train_feature_generator():
    # must return x in the form of (32, 1, 28, 28)
    # must return y in the form of (32, 3)
    while True:
        for i in range(int(194/32)):
            yield train_features[i*32:(i+1)*32], labels[i*32:(i+1)*32]
train_features_generator = get_train_feature_generator()

top_model.fit_generator(
    generator=train_features_generator,
    samples_per_epoch=32,
    nb_epoch=10, 
    #callbacks=callbacks_list,
    verbose=2
)
top_model.fit_generator(
    generator=train_features_generator,
    samples_per_epoch=32,
    nb_epoch=10, 
    callbacks=callbacks_list,
    verbose=2
)
val_features, val_features_path = __extract_features(val_generator, model, batch_size, "validation")

val_labels = np.zeros((32, 3))

for i, class_index in zip(range(0, 32), val_generator.classes):
    val_labels[i][class_index] = 1

print("val_features: {}, val_labels: {}".format(val_features.shape, val_labels.shape))

score = top_model.evaluate(
    x=val_features,
    y=val_labels,
    batch_size=batch_size,
)

print("{}: {}".format(
    top_model.metrics_names[1],
    score[1]
))
print(val_features.shape)
val_labels<jupyter_output><empty_output><jupyter_text>3. Train entire model with data<jupyter_code>from keras.models import Model                              

model = ResNet50(
            include_top=False,
            weights="imagenet",
            input_tensor=Input(
                shape=(
                    img_height, 
                    img_width, 
                    img_dimension
                )
            )
        )

for layer in model.layers:
    layer.trainable = True
combined_model = Model(input=model.input, output=top_model(model.output))
combined_model.compile(
    loss="categorical_crossentropy",
    optimizer="adam",
    metrics=["accuracy"]
)
combined_model.fit_generator(
    generator=train_generator,
    samples_per_epoch=32,
    nb_epoch=10, 
    #callbacks=callbacks_list,
    verbose=2
)
score = combined_model.evaluate_generator(
    generator=val_generator,
    val_samples=val_generator.nb_sample
)

print("{}: {}".format(
    combined_model.metrics_names[1],
    score[1]
))<jupyter_output>acc: 0.3112582793298936
{'fraction_non_alphanumeric': 0.13175216800752274, 'fraction_numerical': 0.02476230278967715, 'mean_word_length': 3.624154589371981, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '378856', 'license': 'Apache-2.0', 'n_tokens_mistral': 3427, 'n_tokens_neox': 3235, 'n_words': 654, 'path': 'pelops/training/CNN Retrainer.ipynb', 'repo_name': 'dave-lab41/pelops', 'size': '9571'}