<jupyter_start><jupyter_text>Import the google drive<jupyter_code>from google.colab import drive
drive.mount('/gdrive')<jupyter_output>Mounted at /gdrive<jupyter_text>Copy the data and code to /content - gdrive<jupyter_code># ! cp -r /gdrive/MyDrive/img-quality-assesment/nima/ /content/
# ! cp -r /gdrive/MyDrive/nima-dataset/ /content/
# ! unzip /content/nima-dataset/nima-dataset.zip -d /content/nima-dataset
! rm /content/nima-dataset/nima-dataset.zip
# sync model to local folder
from shutil import copy2
def  sync_model_to_drive(model_file):
    dest_dir = '/gdrive/MyDrive/img-quality-assesment/nima/weights/'
    copy2(model_file, dest_dir)

DATASET_DIR = '/content/nima-dataset/content/nima-dataset/'
TID_DATASET = os.path.join(DATASET_DIR, 'tid2013')
TID_IMAGES = os.path.join(TID_DATASET,'distorted_images')
AVA_DATASET = os.path.join(DATASET_DIR, 'ava')
AVA_IMAGES = os.path.join(AVA_DATASET,'images')

! ls nima-dataset/<jupyter_output>dataset-metadata.json  nima-dataset.zip<jupyter_text>Main code beigins here  Import the libraries<jupyter_code>! pip install -q livelossplot

import pandas as pd 
import numpy as np 
import matplotlib.pyplot as plt
import seaborn as sns
import os

import nima as nima
from nima.utils.ava_dataset_utils import load_data, get_rating_columns
from nima.config import INPUT_SHAPE, DATASET_DIR, CROP_SHAPE, MODEL_BUILD_TYPE, print_msg, WEIGHTS_DIR, TID_DATASET_DIR
from nima.model.data_generator import TrainDataGenerator, TestDataGenerator
from nima.model.loss import earth_movers_distance
from nima.model.model_builder import NIMA, model_weight_name<jupyter_output><empty_output><jupyter_text>Train the model on TID data<jupyter_code>p_model_name = 'inception'
p_dataset_dir = AVA_DATASET
p_sample_size = None
p_weight_path = WEIGHTS_DIR
p_batch_size=128
p_metrics=['accuracy']
p_epochs=25
p_verbose=1
ava_dataset_dir = p_dataset_dir #os.path.join(p_dataset_dir, 'AVA')
ava_images_dir = os.path.join(ava_dataset_dir, 'images')
img_format = 'jpg'
print_msg(f'Images directory {ava_images_dir}')

# Load the dataset
x_col, y_cols = 'image_id', get_rating_columns()
df_train, df_valid, df_test = load_data(p_dataset_dir, p_sample_size)
assert len(df_train) > 0 and len(df_valid) > 0 and len(df_test) > 0, 'Empty dataframe'
train_batch_size = valid_batch_size = p_batch_size
test_batch_size = min(p_batch_size, 32, len(df_test))

# Form the NIMA Aesthetic Model
nima_aesthetic_cnn = NIMA(base_model_name=p_model_name, model_weights=p_weight_path,
                            loss=earth_movers_distance, input_shape=INPUT_SHAPE, metrics=p_metrics)

# Build the model for training
nima_aesthetic_cnn.build()
nima_aesthetic_cnn.compile()
nima_aesthetic_cnn.model.summary()
# Get the generator
train_generator = TrainDataGenerator(df_train, ava_images_dir, x_col=x_col, y_col=y_cols,
                                        img_format=img_format, num_classes=10,
                                        preprocess_input=nima_aesthetic_cnn.get_preprocess_function(),
                                        batch_size=train_batch_size, input_size=INPUT_SHAPE, crop_size=CROP_SHAPE)
valid_generator = TrainDataGenerator(df_valid, ava_images_dir, x_col=x_col, y_col=y_cols,
                                        img_format=img_format, num_classes=10,
                                        preprocess_input=nima_aesthetic_cnn.get_preprocess_function(),
                                        batch_size=train_batch_size, input_size=INPUT_SHAPE, crop_size=CROP_SHAPE)


    # Train the model
    print_msg("Training Aesthetic Model...")
    train_result_df, train_weights_file = nima_aesthetic_cnn.train_model(train_generator, valid_generator,
                                                                         epochs=p_epochs, verbose=p_verbose)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08719560094265515, 'fraction_numerical': 0.0054988216810683424, 'mean_word_length': 4.134408602150538, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '132481', 'license': 'Apache-2.0', 'n_tokens_mistral': 1340, 'n_tokens_neox': 1319, 'n_words': 236, 'path': 'notebooks/ava_train_model.ipynb', 'repo_name': 'maindolaamit/nima', 'size': '3819'}