<jupyter_start><jupyter_text>HackerEarth Deep Learning Challenge 1 : Predicting the Product Category<jupyter_code>%matplotlib inline
## Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from scipy.misc import *
import os,sys
from PIL import Image
import cv2 #OpenCV library
from tqdm import tqdm
DATA_HOME_DIR = os.getcwd()
%pwd<jupyter_output><empty_output><jupyter_text>Reading the image labels and exploring the datasets<jupyter_code>train = pd.read_csv("/data/train.csv")
test = pd.read_csv("/data/test.csv")
train.head()
train.groupby(by = "label").count()
## set path for images
TRAIN_PATH = '/data/train_img/'
TEST_PATH = '/data/test_img/'<jupyter_output><empty_output><jupyter_text>Let's check out a few images from the training set along with their labels. Resizing pixelates the text part of the image.<jupyter_code># function to read images as arrays
def read_image(img_path):
    img = cv2.imread(img_path, cv2.IMREAD_COLOR)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    img = cv2.resize(img, (64,64))
    return img
# Resizing train and test images to 64X64
train_data = []
test_data = []
train_labels = train['label'].values

for img in tqdm(train['image_id'].values):
    train_data.append(read_image(TRAIN_PATH + '{}.png'.format(img)))
    
for img in tqdm(test['image_id'].values):
    test_data.append(read_image(TEST_PATH + '{}.png'.format(img)))
i = np.random.choice(np.arange(len(train_data)))
fig, ax = plt.subplots(nrows=1, ncols=2)

plt.subplot(1,2,1)
plt.title('{}.png - {} (Resized)'.format(train['image_id'].values[i], train_labels[i]))
plt.imshow(train_data[i])

plt.subplot(1,2,2)
plt.title('{}.png - {}'.format(train['image_id'].values[i], train_labels[i]))
plt.imshow(Image.open(TRAIN_PATH +'{}.png'.format(train.image_id[i])))
# Scale the images
X_train = np.array(train_data, np.float32) / 255.
X_test = np.array(test_data, np.float32) / 255.
# Encoding image labels into target variables (categorical)
from sklearn.preprocessing import LabelEncoder,OneHotEncoder
le = LabelEncoder()
y_train = le.fit_transform(train_labels)

ohe = OneHotEncoder(categorical_features = 'all')
y_train = ohe.fit_transform(y_train.reshape(-1,1)).toarray()
X_train.shape, y_train.shape<jupyter_output><empty_output><jupyter_text>Let's train simple Keras CNN models<jupyter_code># Importing the Keras libraries and packages
import keras
from keras.models import Sequential
from keras.layers import *
from keras.optimizers import SGD
from keras.constraints import maxnorm
from keras.callbacks import EarlyStopping
from keras.preprocessing import image<jupyter_output>Using Theano backend.
Using cuDNN version 5110 on context None
Preallocating 10867/11439 Mb (0.950000) on cuda
Mapped name None to device cuda: Tesla K80 (0000:00:1E.0)<jupyter_text>Model 1 : Deep CNN architecture with/without Dropout<jupyter_code>model = Sequential()
model.add(Convolution2D(32, (3,3), activation='relu', padding='same',input_shape = (64,64,3)))
model.add(Convolution2D(32, (3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))

model.add(Convolution2D(64, (3,3), activation='relu', padding='same'))
model.add(Convolution2D(64, (3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
#model.add(Dropout(0.25))

model.add(Convolution2D(128, (3,3), activation='relu', padding='same'))
model.add(Convolution2D(128, (3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
#model.add(Dropout(0.25))

model.add(Flatten())
model.add(Dense(256, activation='relu'))
#model.add(Dropout(0.5))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(y_train.shape[1], activation='softmax'))
opt = SGD(lr = 0.01)
model.compile(loss = 'categorical_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
model.summary()
early_stops = EarlyStopping(patience=5, monitor='val_acc')
# Training the model
model.fit(X_train, y_train, batch_size=64, epochs=2, validation_split=0.2, callbacks=[early_stops])
model.optimizer.lr = 0.0005<jupyter_output><empty_output><jupyter_text>Model heavily overfitted (20 epochs). Let's see the predictions on these and then we'll move to pre-trained models (different script).  Making predictions on test set<jupyter_code>predictions = model.predict(X_test)
predictions = np.argmax(predictions, axis= 1)
predictions[13:21]
unique_labels = np.unique(train['label'].tolist())
# Converting predicted category numbers to predicted labels
pred_labels = unique_labels[predictions]
## lets see what our classifier predicts on random test images
# Random predictions
i = np.random.choice(np.arange(len(test_data)))
print('I see this product is {}'.format(pred_labels[i]))
plt.title('{}.png'.format(test['image_id'].values[i]))
plt.imshow(Image.open(TEST_PATH +'{}.png'.format(test.image_id[i])))
# Prepare submission file
subm = pd.DataFrame({'image_id':test.image_id, 'label':pred_labels})
subm.to_csv('results.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14518072289156628, 'fraction_numerical': 0.03072289156626506, 'mean_word_length': 7.125611745513866, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '129176', 'license': 'MIT', 'n_tokens_mistral': 1853, 'n_tokens_neox': 1761, 'n_words': 378, 'path': 'Deep Learning Challenge #1 - Predict the Product/Identify the Product.ipynb', 'repo_name': 'sandeepnair2812/HackerEarth-problems', 'size': '4980'}