<jupyter_start><jupyter_text>Dataset is stored on shared google drive.<jupyter_code>from google.colab import drive

drive.mount("/content/drive")<jupyter_output>Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount("/content/drive", force_remount=True).<jupyter_text>Create .txt files with image filenames. Separate file for original and fake images in train and test datasets.<jupyter_code>!ls /content/drive/Shareddrives/datasets/animals/data/train/original/*.jpg > /content/drive/Shareddrives/datasets/animals/originals_train.txt
!ls /content/drive/Shareddrives/datasets/animals/data/train/fake/*.jpg > /content/drive/Shareddrives/datasets/animals/fakes_train.txt

!ls /content/drive/Shareddrives/datasets/animals/data/test/original/*.jpg > /content/drive/Shareddrives/datasets/animals/originals_test.txt
!ls /content/drive/Shareddrives/datasets/animals/data/test/fake/*.jpg > /content/drive/Shareddrives/datasets/animals/fakes_test.txt

path = "/content/drive/Shareddrives/datasets/animals/"
original_train_filenames_path = path + "originals_train.txt"
fake_train_filenames_path = path + "fakes_train.txt"
original_test_filenames_path = path + "originals_test.txt"
fake_test_filenames_path = path + "fakes_test.txt"<jupyter_output><empty_output><jupyter_text>Paths to folders that include images. Current structure (after copying from previous notebook):data/    -train/          -original/          -fake/    -test/          -original/          -fake/<jupyter_code>original_train_path = path + "data/train/original"
fake_train_path = path + "data/train/fake"
original_test_path = path + "data/test/original"
fake_test_path = path + "data/test/fake"<jupyter_output><empty_output><jupyter_text>Function that creates array of filenames<jupyter_code>def get_array_of_filenames(original_train_filenames_path, original_test_filenames_path, fake_train_filenames_path, fake_test_filenames_path, main_path):
    ori_train_paths = []
    fake_train_paths = []
    ori_test_paths = []
    fake_test_paths = []

    with open(original_train_filenames_path, 'r') as file:
        ori_train_paths = [main_path + read.strip() for read in file.readlines()]

    with open(fake_train_filenames_path, 'r') as file:
        fake_train_paths = [main_path + read.strip() for read in file.readlines()]

    with open(original_test_filenames_path, 'r') as file:
        ori_test_paths = [main_path + read.strip() for read in file.readlines()]

    with open(fake_test_filenames_path, 'r') as file:
        fake_test_paths = [main_path + read.strip() for read in file.readlines()]

    return ori_train_paths, fake_train_paths, ori_test_paths, fake_test_paths
ori_train, fakes_train, ori_test, fakes_test = get_array_of_filenames(original_train_filenames_path, fake_train_filenames_path, original_test_filenames_path, fake_test_filenames_path, "")<jupyter_output><empty_output><jupyter_text><jupyter_code>def transformation_to_magnitude(image):
    image = np.mean(image, axis=2)
    dft = np.fft.fft2(image)
    dft_shift = np.fft.fftshift(dft)
    magnitude = np.abs(dft_shift)
    magnitude = magnitude/np.max(magnitude)
    result = np.zeros((magnitude.shape[0], magnitude.shape[1],3))
    result[:,:,0] = magnitude
    result[:,:,1] = magnitude
    result[:,:,2] = magnitude
    return result
def transformation_to_phase(image):

    image = np.mean(image, axis=2)
    dft = np.fft.fft2(image)
    dft_shift = np.fft.fftshift(dft)
    angle = np.angle(dft_shift)
    angle = (angle+np.pi)/(2*np.pi)
    result = np.zeros((angle.shape[0], angle.shape[1],3))
    result[:,:,0] = angle
    result[:,:,1] = angle
    result[:,:,2] = angle
    return result

def transformation_to_everything(image):
    image = np.mean(image, axis=2)
    dft = np.fft.fft2(image)
    dft_shift = np.fft.fftshift(dft)
    magnitude = np.abs(dft_shift)
    magnitude = magnitude/np.max(magnitude)
    angle = np.angle(dft_shift)
    angle = (angle+np.pi)/(2*np.pi)
    result = np.zeros((magnitude.shape[0], magnitude.shape[1],3))
    result[:,:,0] = image
    result[:,:,1] = angle
    result[:,:,2] = magnitude
    return result<jupyter_output><empty_output><jupyter_text>Main analisys<jupyter_code>K.clear_session()
n_classes = 2
img_width, img_height = 512, 512
train_data_dir = '/content/drive/Shareddrives/datasets/animals/data/train'
validation_data_dir = '/content/drive/Shareddrives/datasets/animals/data/test'
nb_train_samples = 1414 #75750
nb_validation_samples = 142 #25250
batch_size = 16
transformation_function = transformation_to_phase

train_datagen = ImageDataGenerator(
    rescale=1. / 255)
    #preprocessing_function=transformation_function)

test_datagen = ImageDataGenerator(
    rescale=1. / 255)
    #preprocessing_function=transformation_function)

train_generator = train_datagen.flow_from_directory(
    train_data_dir,
    target_size=(img_height, img_width),
    batch_size=batch_size,
    class_mode='categorical')

validation_generator = test_datagen.flow_from_directory(
    validation_data_dir,
    target_size=(img_height, img_width),
    batch_size=batch_size,
    class_mode='categorical')


inception = InceptionV3(weights="imagenet", include_top=False, classes=2)
x = inception.output
x = GlobalAveragePooling2D()(x)
x = Dense(128,activation='relu')(x)
x = Dropout(0.2)(x)

predictions = Dense(2,kernel_regularizer=regularizers.l2(0.005), activation='softmax')(x)

model = Model(inputs=inception.input, outputs=predictions)
model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy', metrics=['accuracy'])
checkpointer = ModelCheckpoint(filepath='/content/drive/Shareddrives/datasets/results/best_model_animals_batch_16_imagenet.hdf5', verbose=1, save_best_only=True)
csv_logger = CSVLogger('/content/drive/Shareddrives/datasets/results/history_animals_batch_16_imagenet.log')

history = model.fit_generator(train_generator,
                    steps_per_epoch = nb_train_samples // batch_size,
                    validation_data=validation_generator,
                    validation_steps=nb_validation_samples // batch_size,
                    epochs=30,
                    verbose=1,
                    callbacks=[csv_logger, checkpointer])

model.save('/content/drive/Shareddrives/datasets/results/model_trained_animals_batch_16_imagenet.hdf5')<jupyter_output>Found 1414 images belonging to 2 classes.
Found 142 images belonging to 2 classes.
Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/inception_v3/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5
87916544/87910968 [==============================] - 0s 0us/step
{'fraction_non_alphanumeric': 0.1197893152746426, 'fraction_numerical': 0.01851015801354402, 'mean_word_length': 4.955197132616488, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 27, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '494784', 'license': 'MIT', 'n_tokens_mistral': 2376, 'n_tokens_neox': 2260, 'n_words': 351, 'path': 'notebooks/AO_2.ipynb', 'repo_name': 'pkorytowski/photo-authenticity-detector', 'size': '6645'}