<jupyter_start><jupyter_text>Notebook to preprocess the images into encodings (VGG19 fc2 produces 4096 array)<jupyter_code>import pandas as pd
import numpy as np

from image_encoding import encode_images
from shared_module import *<jupyter_output>Using TensorFlow backend.<jupyter_text>Use existing VGG19 model and to encode the imagesNote that the images are preprocessed and VGG19 accepts 244 x 244 images  Start encoding the imagesLoad the small training and dev set<jupyter_code>train_small_file_path = get_path(INPUT_DIR + '/' + 'labels_train_small', ANCHORS_10, VGG19_4096)

df_train_small = pd.read_csv(train_small_file_path('source'))

df_train_small_vgg19_4096 = encode_images(df_train_small, INPUT_DIR, IMAGE_DIR, VGG19_4096)
df_train_small_vgg19_4096.to_csv(train_small_file_path('encoding'), index=False)
dev_small_file_path = get_path(INPUT_DIR + '/' + 'labels_dev_small', ANCHORS_10, VGG19_4096)

df_dev_small = pd.read_csv(dev_small_file_path('source'))

df_dev_small_vgg19_4096 = encode_images(df_dev_small, INPUT_DIR, IMAGE_DIR, VGG19_4096)
df_dev_small_vgg19_4096.to_csv(dev_small_file_path('encoding'), index=False)<jupyter_output><empty_output><jupyter_text>Load the full training set and encode images<jupyter_code>train_full_file_path = get_path(INPUT_DIR + '/' + 'labels_train_full', ANCHORS_10, VGG19_4096)

df_train_full = pd.read_csv(train_full_file_path('source'))

df_train_full_vgg19_4096 = encode_images(df_train_full, INPUT_DIR, IMAGE_DIR, VGG19_4096)
df_train_full_vgg19_4096.to_csv(train_full_file_path('encoding'), index=False)

dev_full_file_path = get_path(INPUT_DIR + '/' + 'labels_dev_full', ANCHORS_10, VGG19_4096)

df_dev_full = pd.read_csv(dev_full_file_path('source'))

df_dev_full_vgg19_4096 = encode_images(df_dev_full, INPUT_DIR, IMAGE_DIR, VGG19_4096)
df_dev_full_vgg19_4096.to_csv(dev_full_file_path('encoding'), index=False)
train_labels_file_path = get_path(INPUT_DIR + '/' + 'labels', ANCHORS_12, VGG19_4096)

df_train = pd.read_csv(train_labels_file_path('source'))

df_train_vgg19_4096 = encode_images(df_train, INPUT_DIR, IMAGE_DIR, VGG19_4096)
df_train_vgg19_4096.to_csv(train_labels_file_path('encoding'), index=False)
df_test = pd.read_csv('labels_test.csv')

df_test_vgg19_4096 = encode_images(df_test, 'input/test-2/', 'input/test-2/', VGG19_4096)
df_test_vgg19_4096.to_csv('labels_test_vgg19_4096.csv')<jupyter_output>input/test-2//vgg19_4096 already exist
Encoding already exist: input/test-2//vgg19_4096/266490a2b053a4a3bf68e779f28fb5a9.txt. Skipping.
Encoding already exist: input/test-2//vgg19_4096/dae9fd18836d8df20989ef0101771d4b.txt. Skipping.
Encoding already exist: input/test-2//vgg19_4096/079a40f22b0fad9a66f76797b62bb930.txt. Skipping.
Encoding already exist: input/test-2//vgg19_4096/ab8462f9a501d362041d47582fddc9ad.txt. Skipping.
Encoding already exist: input/test-2//vgg19_4096/8895089b432e020cc42e6262b5a1c2dd.txt. Skipping.
Encoding already exist: input/test-2//vgg19_4096/985ae7fc271ac4a0d131e546c0f95bb6.txt. Skipping.
Encoding already exist: input/test-2//vgg19_4096/77c1ab99cffb817d54fb14f229aeb118.txt. Skipping.
Encoding already exist: input/test-2//vgg19_4096/ad7e50abca9b7c6a88b63946c5bbe493.txt. Skipping.
Encoding already exist: input/test-2//vgg19_4096/d61a09cecf56666cdb315ebce4b93c43.txt. Skipping.
Encoding already exist: input/test-2//vgg19_4096/cb120c57904e8c0694bbe259c58a873c.txt. S[...]
{'fraction_non_alphanumeric': 0.09698403311649911, 'fraction_numerical': 0.13187463039621525, 'mean_word_length': 8.610795454545455, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '43533', 'license': 'Apache-2.0', 'n_tokens_mistral': 1617, 'n_tokens_neox': 1390, 'n_words': 194, 'path': 'doggie_img_encoding.ipynb', 'repo_name': 'alechfho/dog_breed', 'size': '3382'}