<jupyter_start><jupyter_text>!pip install aicrowd-cli==0.1 API_KEY = ""!aicrowd login --api-key $API_KEY !aicrowd dataset download --challenge rover-classification -j 3<jupyter_code>import pandas as pd
import os
df_train = pd.read_csv(os.path.join(data_folder, "train.csv"))
df_val = pd.read_csv(os.path.join(data_folder, "val.csv"))
df_train['ImageID'] = df_train['ImageID'].astype(str)+".jpg"
df_val['ImageID'] = df_val['ImageID'].astype(str)+".jpg"
df_train.head()
df_val.head()
from tensorflow.keras.preprocessing.image import ImageDataGenerator
datagen=ImageDataGenerator(rescale=1./255.)

train_generator=datagen.flow_from_dataframe(
dataframe=df_train,
directory="../data/train/",
x_col="ImageID",
y_col="label",
batch_size=32,
seed=42,
shuffle=True,
class_mode="categorical",
target_size=(128,128))
val_generator=datagen.flow_from_dataframe(
dataframe=df_val,
directory="../data/val/",
x_col="ImageID",
y_col="label",
batch_size=32,
seed=42,
shuffle=True,
class_mode="categorical",
target_size=(128,128))
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation, Conv2D, Flatten, Dropout, MaxPooling2D, BatchNormalization
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras import regularizers, optimizers
import os
import numpy as np
import pandas as pd
class CustomAugment(object):
    def __call__(self, image):        
        # Random flips and grayscale with some stochasticity
        img = self._random_apply(tf.image.flip_left_right, image, p=0.5)
        img = self._random_apply(self._color_drop, img, p=0.8)
        return img

    def _color_drop(self, x):
        image = tf.image.rgb_to_grayscale(x)
        image = tf.tile(x, [1, 1, 1, 3])
        return x
    
    def _random_apply(self, func, x, p):
        return tf.cond(
          tf.less(tf.random.uniform([], minval=0, maxval=1, dtype=tf.float32),
                  tf.cast(p, tf.float32)),
          lambda: func(x),
          lambda: x)
data_augmentation = tf.keras.Sequential(
  [
     tf.keras.layers.Lambda(CustomAugment()),
    tf.keras.layers.experimental.preprocessing.RandomFlip("horizontal", 
                                                 input_shape=(128, 
                                                              128,
                                                              3)),
    tf.keras.layers.experimental.preprocessing.RandomRotation(0.1),
    tf.keras.layers.experimental.preprocessing.RandomZoom(0.1),
  ]
)
model = Sequential()
model.add(data_augmentation)
model.add(tf.keras.applications.MobileNetV2(input_shape=(128, 128, 3),
                                               include_top=False,
                                               weights='imagenet'))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(2, activation='softmax'))
model.compile(optimizers.RMSprop(lr=0.0001/10), loss="categorical_crossentropy", metrics=["Recall", "Precision"])
model.layers[1].trainable = False
import cv2
img = cv2.imread('../data/train/0.jpg')
img.shape # Start learning at 128 then change to 256
next(train_generator)[0].shape
STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size
STEP_SIZE_VAL=val_generator.n//train_generator.batch_size
model.fit(train_generator, steps_per_epoch=STEP_SIZE_TRAIN, validation_data=val_generator, validation_steps=STEP_SIZE_VAL, epochs=1)
model.summary()
model.layers[1].trainable = True
for layer in model.layers[1].layers[:100]:
    layer.trainable =  False
model.summary()
model.fit(train_generator, steps_per_epoch=STEP_SIZE_TRAIN, validation_data=val_generator, validation_steps=STEP_SIZE_VAL, epochs=1)
model<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12821874170427397, 'fraction_numerical': 0.02548447040084948, 'mean_word_length': 3.7336683417085426, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '205957', 'license': 'MIT', 'n_tokens_mistral': 1301, 'n_tokens_neox': 1227, 'n_words': 209, 'path': 'nbs/02_mobile_net.ipynb', 'repo_name': 'antonai91/rover_classification', 'size': '3767'}