<jupyter_start><jupyter_text>Stackline - DS Assignment  Import functions<jupyter_code>import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
from sklearn.ensemble import VotingClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import SGDClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import LabelEncoder
from keras.utils import np_utils
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras.layers import ZeroPadding2D
from keras.layers import Convolution2D
import h5py
from keras import applications
from keras.utils.np_utils import to_categorical
from keras.models import Model
import urllib
import os
import math<jupyter_output><empty_output><jupyter_text>Exploratory Data Analysis  Reading Data<jupyter_code>df_train = pd.read_excel('Training_Data_Assessment.xlsx')
df_test = pd.read_excel('Data To Classify_Assessment.xlsx')
categories = pd.read_excel('Categories_Assessment.xlsx',header=None,names=['CategoryName'])
df_train.head()
df_train.describe()
df_test.head()
df_test.describe()
categories.head()
unique_categories = categories['CategoryName']<jupyter_output><empty_output><jupyter_text>**Observations:**1. Training data: 6031 unique ASIN in 6034 rows. Need to remove duplicates.2. Testing data: 57030 unique ASIN in 57030 rows. No duplicates found.3. No Null values observed  Removing duplicate ASINs<jupyter_code>df_train.drop_duplicates('ASIN',inplace=True)<jupyter_output><empty_output><jupyter_text>Training and Validation Split<jupyter_code>x_df_train, x_df_test, y_df_train, y_df_test = train_test_split(df_train[['Title','ImageUrl']],df_train[['CategoryName']],
train_size=.8, stratify=df_train['CategoryName'],random_state=30)<jupyter_output><empty_output><jupyter_text>Image Downloading **Aim*** To download training, validation and testing images from image url* The images are stored as        /train            /category1                train_image1.jpg                train_image2.jpg            /category2                train_image1.jpg                train_image2.jpg        /validation            /category1                validation_image1.jpg                validation_image2.jpg            /category2                validation_image1.jpg                validation_image2.jpg        /test            /New folder                test_image1.jpg                test_image2.jpg  Path variables<jupyter_code>current_path = os.getcwd()
train_path = os.path.join(current_path,'train')
validation_path = os.path.join(current_path,'validation')
test_path = os.path.join(current_path,'test')<jupyter_output><empty_output><jupyter_text>Function to download test and validation images<jupyter_code>def image_download(x_df,y_df,unique_categories,path,category_column='CategoryName',imageurl_column='ImageUrl'):
    for i in x_df.index:
        for j in unique_categories:            
            if y_df[category_column][i] == j:
                class_path = os.path.join(path,j)
                if not os.path.exists(class_path):
                    os.makedirs(class_path)
                filename = '%s.jpg'%i
                fullfilename = os.path.join(class_path,filename)
                if not os.path.exists(fullfilename):
                    try:
                        urllib.request.urlretrieve(x_df[imageurl_column][i],fullfilename)
                    except:
                        print('Did not download:')
                        print(i)
                        print(x_df[imageurl_column][i])
                        pass
    return<jupyter_output><empty_output><jupyter_text>Download Training Images<jupyter_code>image_download(x_df=x_df_train,y_df=y_df_train,unique_categories=unique_categories,path=train_path)<jupyter_output><empty_output><jupyter_text>Download Validation Images<jupyter_code>image_download(x_df=x_df_test,y_df=y_df_test,unique_categories=unique_categories,path=validation_path)<jupyter_output><empty_output><jupyter_text>Download Test Images<jupyter_code>for i in df_test.index:
    class_path = os.path.join(test_path,'New folder')
    if not os.path.exists(class_path):
        os.makedirs(class_path)
    filename = '%s.jpg'%i
    fullfilename = os.path.join(class_path,filename)
    if not os.path.exists(fullfilename):
        try:
            urllib.request.urlretrieve(df_test['ImageUrl'][i],fullfilename)
        except:
            print('Did not download:')
            print(i)
            print(df_test['ImageUrl'][i])
            pass<jupyter_output><empty_output><jupyter_text>Text Classification **Aim*** To build a pipeline for text features* To build classifiers for the text features **Steps in pipeline*** Convert to lowercase* Remove stop words* Convert a collection of text documents to a matrix of token counts* Transform a count matrix to a normalized tf or tf-idf representation. Tf means term-frequency while tf-idf means term-frequency times inverse document-frequency. This is a common term weighting scheme in information retrieval, that has also found good use in document classification. **Classifiers used*** Multinomial Naive Bayes* Linear SVM with SGD* Random Forests* Logistic Regression  Multinomial NB<jupyter_code>text_clf_multinomial_nb = Pipeline([('vect', CountVectorizer(stop_words='english',lowercase=True)),('tfidf', TfidfTransformer()),('clf', MultinomialNB())])
text_clf_multinomial_nb.fit(x_df_train['Title'], y_df_train['CategoryName'])
text_clf_multinomial_nb.score(x_df_test['Title'],y_df_test['CategoryName'])<jupyter_output><empty_output><jupyter_text>SGD Classifier<jupyter_code>text_clf_sgd = Pipeline([('vect', CountVectorizer(stop_words='english',lowercase=True)),('tfidf', TfidfTransformer()),('clf', SGDClassifier())])
text_clf_sgd.fit(x_df_train['Title'], y_df_train['CategoryName'])
text_clf_sgd.score(x_df_test['Title'],y_df_test['CategoryName'])<jupyter_output><empty_output><jupyter_text>Random Forest Classifier<jupyter_code>text_clf_rf = Pipeline([('vect', CountVectorizer(stop_words='english',lowercase=True)),('tfidf', TfidfTransformer()),('clf', RandomForestClassifier())])
text_clf_rf.fit(x_df_train['Title'], y_df_train['CategoryName'])
text_clf_rf.score(x_df_test['Title'],y_df_test['CategoryName'])<jupyter_output><empty_output><jupyter_text>Logistic Regression Classifier<jupyter_code>text_clf_logis = Pipeline([('vect', CountVectorizer(stop_words='english',lowercase=True)),('tfidf', TfidfTransformer()),('clf', LogisticRegression())])
text_clf_logis.fit(x_df_train['Title'], y_df_train['CategoryName'])
text_clf_logis.score(x_df_test['Title'],y_df_test['CategoryName'])<jupyter_output><empty_output><jupyter_text>**Note*** Due to time and computational constraints, I am not going be doing a grid search cross validation for the above models. * I believe that a test dataset accuracy of above 80% is sufficient enough as I am going to take the average of all classifiers.  Image Classification **Aim*** To train a classifier over a pre-trained VGG16 model  Define Variables<jupyter_code># dimensions of our images.
img_width, img_height = 150, 150

top_model_weights_path = 'bottleneck_fc_model1.h5'
train_data_dir = 'train'
validation_data_dir = 'validation'
epochs = 50
batch_size = 32

datagen = ImageDataGenerator(rescale=1. / 255)<jupyter_output><empty_output><jupyter_text>Building VGG16 network<jupyter_code>model = applications.VGG16(include_top=False, weights='imagenet')<jupyter_output><empty_output><jupyter_text>Running VGG16 is expensive, as working on CPU. I want to only do it only once. This prevents me from using data augmentation which can improve my model by preventing overfitting.  Generating Training Feature Set<jupyter_code>#train generator
train_generator = datagen.flow_from_directory(
    train_data_dir,
    target_size=(img_width, img_height),
    batch_size=batch_size,
    class_mode= 'categorical',
    shuffle=False)

#train variables
nb_train_samples = len(train_generator.filenames)
num_classes = len(train_generator.class_indices)
predict_size_train = int(math.ceil(nb_train_samples / batch_size))

#creaing training feature set
bottleneck_features_train = model.predict_generator(
    train_generator, predict_size_train)
np.save('bottleneck_features_train',bottleneck_features_train)

#train labels
train_labels = train_generator.classes
train_labels = to_categorical(train_labels,63)<jupyter_output>Found 4824 images belonging to 63 classes.<jupyter_text>Generating Validation Feature Set<jupyter_code>#validation generator
generator = datagen.flow_from_directory(
    validation_data_dir,
    target_size=(img_width, img_height),
    batch_size=batch_size,
    class_mode= 'categorical',
    shuffle=False)

#test variables
nb_validation_samples = len(generator.filenames)
predict_size_validation = int(math.ceil(nb_validation_samples / batch_size))

#creaing validation feature set
bottleneck_features_validation = model.predict_generator(
    generator, predict_size_validation)
np.save('bottleneck_features_validation',bottleneck_features_validation)

#test labels
validation_labels = generator.classes
validation_labels = to_categorical(validation_labels,63)<jupyter_output>Found 1207 images belonging to 63 classes.<jupyter_text>Loading Training and Validation feature set<jupyter_code>#loading training and validation feature set generated above
train_data = np.load('bottleneck_features_train.npy')
validation_data = np.load('bottleneck_features_validation.npy')<jupyter_output><empty_output><jupyter_text>Training top classifier model<jupyter_code>from keras import optimizers
rmsprop = optimizers.RMSprop(lr=0.001)
top_model = Sequential()
top_model.add(Flatten(input_shape=train_data.shape[1:]))
top_model.add(Dense(256, activation='relu'))
top_model.add(Dropout(0.6))
top_model.add(Dense(63, activation='softmax'))

top_model.compile(optimizer=rmsprop,loss='categorical_crossentropy', metrics=['accuracy'])

top_model.fit(train_data, train_labels,
        epochs=30,
        batch_size=batch_size,
        validation_data=(validation_data, validation_labels))
top_model.save_weights(top_model_weights_path)<jupyter_output>Train on 4824 samples, validate on 1207 samples
Epoch 1/30
4824/4824 [==============================] - 8s - loss: 3.9984 - acc: 0.0800 - val_loss: 3.3763 - val_acc: 0.2038
Epoch 2/30
4824/4824 [==============================] - 8s - loss: 3.3413 - acc: 0.2075 - val_loss: 2.7534 - val_acc: 0.3538
Epoch 3/30
4824/4824 [==============================] - 9s - loss: 2.9185 - acc: 0.2942 - val_loss: 2.4531 - val_acc: 0.4300
Epoch 4/30
4824/4824 [==============================] - 8s - loss: 2.6710 - acc: 0.3472 - val_loss: 2.3308 - val_acc: 0.4457
Epoch 5/30
4824/4824 [==============================] - 9s - loss: 2.4280 - acc: 0.3899 - val_loss: 2.2168 - val_acc: 0.4847
Epoch 6/30
4824/4824 [==============================] - 9s - loss: 2.2637 - acc: 0.4274 - val_loss: 2.1571 - val_acc: 0.4963
Epoch 7/30
4824/4824 [==============================] - 8s - loss: 2.1048 - acc: 0.4600 - val_loss: 2.0917 - val_acc: 0.4954
Epoch 8/30
4824/4824 [==============================] - 9s - loss: 1.9780 - a[...]<jupyter_text>Making Final Image Recognition Model<jupyter_code>model_vgg = applications.VGG16(weights='imagenet', include_top=False, input_shape=(150, 150, 3))

top_model = Sequential()
top_model.add(Flatten(input_shape=model_vgg.output_shape[1:]))
top_model.add(Dense(256, activation='relu'))
top_model.add(Dropout(0.6))
top_model.add(Dense(63, activation='softmax'))

top_model.load_weights('bottleneck_fc_model1.h5')

model = Model(inputs = model_vgg.input, outputs = top_model(model_vgg.output))<jupyter_output><empty_output><jupyter_text>Predict Category based on Test Images<jupyter_code>df_test.reset_index(inplace=True)
filename=[]
vgg16_predictions=[]
final_data_dir='test'
test_datagen = ImageDataGenerator(rescale=1./255)
test_generator = test_datagen.flow_from_directory(
        final_data_dir,
        target_size=(img_width, img_height),
        class_mode=None,
        batch_size=1)

filenames = test_generator.filenames
nb_samples = len(filenames)

predict = model.predict_generator(test_generator,steps = nb_samples)<jupyter_output>Found 2100 images belonging to 1 classes.<jupyter_text>Make image classification results into a data frame<jupyter_code>for f in filenames:
    filename.append(f)
for r in predict:
    vgg16_predictions.append(np.argmax(r))
ir_res = pd.DataFrame(
    {'filename': [f[11:-4] for f in filename],
     'category': vgg16_predictions,
    })
ir_res.to_csv('ir_res.csv')
len(ir_res)<jupyter_output><empty_output><jupyter_text>Final Product Classification  Getting category name from index<jupyter_code>x=pd.read_csv('ir_res.csv')
index=train_generator.class_indices
y=list(x['category'])
categories_p=[]
for i in y:
    for j in index:
        if i == index[j]:
            categories_p.append(j)<jupyter_output><empty_output><jupyter_text>Collecting all predictions from all classifiers<jupyter_code>df_test.set_index('ASIN',inplace=True)
a=text_clf_multinomial_nb.predict(df_test['Title'])
b=text_clf_sgd.predict(df_test['Title'])
c=text_clf_rf.predict(df_test['Title'])
d=text_clf_logis.predict(df_test['Title'])
ir_res_new = pd.DataFrame(
    {'ASIN': x['filename'],
     'CNN Category': categories_p,
    })
ir_res_new.set_index('ASIN',inplace=True)

predictions = df_test.join(ir_res_new)
predictions.drop(['Title','ImageUrl','BrandName'],1, inplace=True)
predictions['Multinomial NB Category'] = a
predictions['SVC Category'] = b
predictions['RF Category'] = c
predictions['LogisticR Category'] = d<jupyter_output><empty_output><jupyter_text>Take Average of all 5 classifiers<jupyter_code>final_pred =[]
for i in range(len(df_test)):
    lst = [a[i],b[i],c[i],d[i],predictions['CNN Category'][i]]
    final_pred.append(max(set(lst), key=lst.count))
df_test['Category'] = final_pred
df_test.to_csv('final_prediction.csv')
predictions.to_csv('predictions_allClassifiers.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12228260869565218, 'fraction_numerical': 0.027034559643255296, 'mean_word_length': 5.751175917215428, 'pattern_counts': {'":': 0, '<': 97, '<?xml version=': 0, '>': 97, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '25', 'dir': 'jupyter-structured-clean-dedup', 'id': '312107', 'license': 'MIT', 'n_tokens_mistral': 5013, 'n_tokens_neox': 4675, 'n_words': 897, 'path': 'projects/machine learning/e-commerce products/e-commerce products.ipynb', 'repo_name': 'rahulavadhoot/Data-Science-Portfolio', 'size': '14352'}