<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import numpy as np
import io
import gc
import time
import PIL.Image as Image
import matplotlib.pylab as plt

import tensorflow as tf
import tensorflow_hub as hub

# settings
import warnings
warnings.filterwarnings("ignore")
gc.enable()
# connect to google drive
from google.colab import drive
drive.mount('/content/drive')
gDriveTrainImages = '/content/drive/MyDrive/Datasets/Hackerearth_vehicle_insurance_claim/dataset/trainImages/'
gDriveTestImages = '/content/drive/MyDrive/Datasets/Hackerearth_vehicle_insurance_claim/dataset/testImages/'
gDrivePath = '/content/drive/MyDrive/Datasets/Hackerearth_vehicle_insurance_claim/dataset/'
df_train = pd.read_csv(gDrivePath+'train.csv')
df_train.head()
df_train.isnull().any()
# df_train['full_image_path'] = df_train['Image_path']
# for i in range(len(df_train['Image_path'])):
#     df_train['full_image_path'][i] = gDriveTrainImages + df_train['Image_path'][i]
# df_train.head()
# print(df_train['full_image_path'][1])<jupyter_output><empty_output><jupyter_text>Download the headless model Create the feature extractor. Use `trainable=False` to freeze the variables in the feature extractor layer, so that the training only modifies the new classifier layer.<jupyter_code># !mkdir -p Hackerearth_vehicle_insurance_claim/feature-vectors
base_model = tf.keras.applications.InceptionResNetV2(include_top=False, weights='imagenet', pooling='max', input_shape=(299, 299, 3))
preprocessor = tf.keras.applications.inception_resnet_v2.preprocess_input
IMAGE_SIZE = (299, 299)
# image_path = df_train['full_mage_path'][0]
# create empty csv with column names
csv_columns = ['image_name']
no_of_extracted_features = 1536

for i in range(1, no_of_extracted_features+1):
  col_name = "{}{}".format('feature_', i)
  csv_columns.append(col_name)

empty_df = pd.DataFrame(columns=csv_columns)
empty_df.to_csv('train_image_features.csv', index=False)
# returns feature vector
def extract_image_features(image_path, model, preprocessor):
  img = tf.keras.preprocessing.image.load_img(image_path, target_size=IMAGE_SIZE)
  x = tf.keras.preprocessing.image.img_to_array(img)
  x = np.expand_dims(x, axis=0)
  x = preprocessor(x)

  features = base_model.predict(x)
  return features
%%time

for index, row in df_train.iterrows():
  image_path = row['Image_path']
  full_image_path = gDriveTrainImages + image_path
  
  print("working on image : {}".format(image_path))
  
  features = extract_image_features(full_image_path, base_model, preprocessor)
  df = pd.DataFrame(features)
  df.insert(loc=0, column='image_name', value=image_path)
  df.to_csv('train_image_features.csv', mode='a', header=False)

# %%time
# features = extract_image_features(image_path, base_model, preprocessor)
# df = pd.DataFrame(features)
# df.insert(loc=0, column='image_name', value=df_train['Image_path'][1])
# df.to_csv('train_image_features.csv', mode='a', header=False)
# df.head()
df_train_image_features = pd.read_csv('train_image_features.csv')
df_train_image_features.reset_index(inplace = True, drop=True)
df_train_image_features.head()
# test image feature extraction
df_test = pd.read_csv(gDrivePath+'test.csv')


# create empty csv with column names
csv_columns = ['image_name']
no_of_extracted_features = 1536

for i in range(1, no_of_extracted_features+1):
  col_name = "{}{}".format('feature_', i)
  csv_columns.append(col_name)

empty_df = pd.DataFrame(columns=csv_columns)
empty_df.to_csv('test_image_features.csv', index=False)
%%time
for index, row in df_test.iterrows():
  image_path = row['Image_path']
  full_image_path = gDriveTestImages + image_path
  
  print("working on image : {}".format(image_path))
  
  features = extract_image_features(full_image_path, base_model, preprocessor)
  df = pd.DataFrame(features)
  df.insert(loc=0, column='image_name', value=image_path)
  df.to_csv('test_image_features.csv', mode='a', header=False)
df_test_image_features = pd.read_csv('test_image_features.csv')
df_test_image_features.reset_index(inplace = True, drop=True)
df_test_image_features.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12138868657441126, 'fraction_numerical': 0.008254430687059966, 'mean_word_length': 7.3232323232323235, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '265069', 'license': 'MIT', 'n_tokens_mistral': 1466, 'n_tokens_neox': 1433, 'n_words': 263, 'path': 'Hackerearth-Predict_condition_and_insurance_amount/vehicle_insurance_image_feature_extraction.ipynb', 'repo_name': 'chiranjeet14/ML_Projects', 'size': '4119'}