<jupyter_start><jupyter_text>Reset input directoryshutil.rmtree(base_dir)<jupyter_code>def safe_create(dirname):
    if not os.path.exists(dirname):
        print("Creating '{}'".format(dirname))
        os.mkdir(dirname)
    else:
        print("'{}' already exists".format(dirname))

safe_create(base_dir)
safe_create(train_dir)
safe_create(validation_dir)
safe_create(test_dir)
# Randomize the list of files
input_files = os.listdir(original_dataset_dir)
np.random.shuffle(input_files)

BUCKET_SIZE = 8500

def copy_bucket_of_maps(start_bucket,num_buckets,dest_dir):
    if len(os.listdir(dest_dir)) == 0:
        print("Copying {} files into '{}'".format(num_buckets*BUCKET_SIZE, dest_dir))
        start = start_bucket * BUCKET_SIZE
        end = (start_bucket + num_buckets) * BUCKET_SIZE
        for fname in input_files[start:end]:
            src = os.path.join(original_dataset_dir, fname)
            dst = os.path.join(dest_dir, fname)
            shutil.copyfile(src, dst)
    else:
        print("Skipped copying files into '{}'".format(dest_dir))

# Copy two buckets worth of metamaps images to train_dir
copy_bucket_of_maps(0,2,train_dir)

# Copy next bucket of metamaps to validation_cats_dir
copy_bucket_of_maps(2,1,validation_dir)
    
# Copy next bucket of metamaps to test_cats_dir
copy_bucket_of_maps(3,1,test_dir)

NUM_REAL_TRAIN_MAPS = 2 * BUCKET_SIZE
NUM_REAL_VALIDATION_MAPS = BUCKET_SIZE
NUM_REAL_TEST_MAPS = BUCKET_SIZE
# Verify number of files in input directories
show_number_of_files_in_dir(train_dir)
show_number_of_files_in_dir(validation_dir)
show_number_of_files_in_dir(test_dir)
#Create model and add layers
from keras import layers
from keras import models

model = models.Sequential()
model.add(layers.Conv2D(32, (3, 3), activation='relu',
                        input_shape=(64, 64, len(metamaps.EncodingDim))))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Flatten())
model.add(layers.Dense(512, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
model.summary()
from keras import optimizers

model.compile(loss='binary_crossentropy',
              optimizer=optimizers.RMSprop(lr=1e-4),
              metrics=['acc'])
BATCH_SIZE = 20

TRAINING_STEPS = int((NUM_REAL_TRAIN_MAPS * 2) / BATCH_SIZE)
VALIDATION_STEPS = int((NUM_REAL_VALIDATION_MAPS * 2) / BATCH_SIZE)

print("Training steps: {}".format(TRAINING_STEPS))
print("Validation steps: {}".format(VALIDATION_STEPS))
train_sequence = metamaps.MetaMapsSequence(train_dir,BATCH_SIZE)
validation_sequence = metamaps.MetaMapsSequence(validation_dir,BATCH_SIZE)
history = model.fit_generator(
      train_sequence,
      steps_per_epoch=TRAINING_STEPS, # number of batches to train with
      epochs=1, # this will get tweaked by looking at the overfitting curves
      validation_data=validation_sequence,
      validation_steps=VALIDATION_STEPS) # number of batches to validate with
model.save('real_vs_fake_metamaps.h5')
import matplotlib.pyplot as plt

acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(len(acc))

plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.figure()

plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12143241787829832, 'fraction_numerical': 0.014270328486806677, 'mean_word_length': 5.47212543554007, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '254522', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 1345, 'n_tokens_neox': 1294, 'n_words': 251, 'path': 'MetaMapClassifier.ipynb', 'repo_name': 'davidaramant/tiledriver-ml', 'size': '3714'}