<jupyter_start><jupyter_text>Load Data from Disk<jupyter_code>#Load the data frame... nothing that special here
raw_df = pd.read_json(os.path.join(data_dir,'downloads/master_cleaned.json'))
    
transposed_df = raw_df.T
transposed_df.head()
image_df = transposed_df[['thumbnail_file','num_followers']]
image_df = image_df[~image_df['thumbnail_file'].str.contains('NA', na=False)]
image_df.fillna(value=0, inplace=True)
image_df = image_df[image_df["num_followers"] > 2000]
print(len(image_df))
image_df.head()<jupyter_output>9619<jupyter_text>Setup Labels (Followers)<jupyter_code>y = list(image_df.num_followers)

#TODO cleverly reshape y? better than this? multiple classes/categories?
#Reshape y to "above or below average" followers
y = [1 if f > 10000 else 0 for f in y]

print(f'{len(y)} artists with {sum(y)/len(y)} tag ratio')<jupyter_output>9619 artists with 0.4175070173614721 tag ratio<jupyter_text>Setup Images<jupyter_code>image_count = len(list(glob.glob(f'{data_dir}/downloads/thumbs/*.jpg')))
print(f'{image_count} thumbnail images')

#View a single image (as a test)
PIL.Image.open(os.path.join(os.getcwd(), "..", str(image_df.thumbnail_file[100])))
batch_size = 32
img_height = 200
img_width = 200

images = []
for image_path in list(image_df.thumbnail_file):
    full_path = os.path.join(os.getcwd(),"..",image_path)
    image = tf.keras.preprocessing.image.load_img(full_path,color_mode='rgb',target_size=(img_height,img_width))
    input_arr = tf.keras.preprocessing.image.img_to_array(image)
    images.append(input_arr)

x = images
plt.figure(figsize=(10, 10))
for i in range(9):
    ax = plt.subplot(3, 3, i + 1)
    plt.imshow(x[i].astype("uint8"))
    plt.title(y[i])
    plt.axis("off")<jupyter_output><empty_output><jupyter_text>Model and Train<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.1, random_state=42)
model = tf.keras.Sequential([
  tf.keras.layers.Conv2D(64, 3, activation='relu'),
  tf.keras.layers.MaxPooling2D(),
  tf.keras.layers.Conv2D(64, 3, activation='relu'),
  tf.keras.layers.MaxPooling2D(),
  tf.keras.layers.Conv2D(64, 3, activation='relu'),
  tf.keras.layers.MaxPooling2D(),
  tf.keras.layers.BatchNormalization(),
  tf.keras.layers.Flatten(),
  tf.keras.layers.Dense(128, activation='tanh'),
  tf.keras.layers.Dense(128, activation='tanh'),
  tf.keras.layers.Dense(1, activation='sigmoid')
])


model.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['accuracy'])
                       
    
modelcheckpoint = tf.keras.callbacks.ModelCheckpoint(filepath=output_dir+
                                  "/weights.{epoch:02d}.hdf5")

model.fit(np.array(X_train), np.array(y_train),
          batch_size=64,
          validation_data=(np.array(X_test),np.array(y_test)),
          epochs=10,
          callbacks=[modelcheckpoint])
score = model.evaluate(np.array(X_test), np.array(y_test)) 
print(f'Test Score: {score[0]} Test Accuracy:, {score[1]}')<jupyter_output>31/31 [==============================] - 0s 11ms/step - loss: 0.6835 - accuracy: 0.5665
Test Score: 0.6834844350814819 Test Accuracy:, 0.5665280818939209<jupyter_text>Load Weights, Test Model and Save<jupyter_code>model.load_weights(output_dir+"/weights.05.hdf5") # NOT zero-indexed
from random import randrange
i=randrange(len(y))

full_path = os.path.abspath(os.path.join(os.getcwd(), 
    "..", 
    list(image_df.thumbnail_file)[i]))
image = tf.keras.preprocessing.image.load_img(full_path,color_mode='rgb',target_size=(img_height,img_width))
input_arr = tf.keras.preprocessing.image.img_to_array(image)

plt.figure(figsize=(1, 1))
plt.imshow(input_arr.astype("uint8"))
plt.title(y[i])
plt.axis("off")

pred_followers = model.predict(np.array([input_arr]))
print(pred_followers[0][0])
model.save(output_dir+'/model_v001')<jupyter_output>WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/tracking/tracking.py:111: Model.state_updates (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.
Instructions for updating:
This property should not be used in TensorFlow 2.0, as updates are applied automatically.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/tracking/tracking.py:111: Layer.updates (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.
Instructions for updating:
This property should not be used in TensorFlow 2.0, as updates are applied automatically.
INFO:tensorflow:Assets written to: /tf/train/model_output/alpha-score/model_v001/assets
{'fraction_non_alphanumeric': 0.15020531661984007, 'fraction_numerical': 0.03695699157121245, 'mean_word_length': 5.454672245467225, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '87860', 'license': 'MIT', 'n_tokens_mistral': 1806, 'n_tokens_neox': 1681, 'n_words': 277, 'path': '3_ML/training_with_images.ipynb', 'repo_name': 'predbrad/bootcamp', 'size': '4627'}