<jupyter_start><jupyter_text>Import the required libraries<jupyter_code>import time
import warnings
warnings.filterwarnings('ignore')

# For loading data and data manipulation
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
plt.style.use('fivethirtyeight')

# For cleaning the text
from nltk.corpus import stopwords
from nltk import RegexpTokenizer
from nltk.stem import WordNetLemmatizer

# For export the dataframe and load
import pickle<jupyter_output><empty_output><jupyter_text>Loading the datasets<jupyter_code>df_legacy = pd.read_csv('data/legacy_data.csv')
df_legacy.head()
df_sample = pd.read_csv('data/sample_output.csv')
df_sample.head()
df_inventory = pd.read_csv('data/inventory.csv')
df_inventory.head()<jupyter_output><empty_output><jupyter_text>Getting one with the data  Taking insights from Legacy data<jupyter_code>df_legacy.head()
# Dropping the Unnamed columns
df_legacy.drop(columns=['Unnamed: 0'], inplace=True)
df_legacy.head()
# Shape of the legacy data
df_legacy.shape
# Creating a plot between the 'OrderPrice' and 'Amount_Charged'
plt.figure(figsize=(12,7))
plt.plot(df_legacy['Amount_Charged'], c='g', label='Amount Charged')
plt.plot(df_legacy['OrderPrice'], c='r', label='Order Price')
plt.title('OrderPrice vs Amount_Charged')
plt.legend()
plt.xlabel('Order Index')
plt.ylabel('Price')
plt.show()
# Checking for the null data
df_legacy.isnull().sum()
# Fetching the null row
df_legacy[df_legacy.isnull().any(axis=1)]
# Drop the null value row
df_legacy.dropna(inplace=True)
# Check for duplicated
df_legacy[df_legacy.duplicated()]
# Drop duplicates
df_legacy.drop_duplicates(inplace=True)
df_legacy.shape
# Removing the outliers i.e. all the records which has 'OrderPrice less than 25 and more than 120'
df_legacy = df_legacy[df_legacy['OrderPrice'] < 120]
df_legacy = df_legacy[df_legacy['OrderPrice'] > 25]
# New shape of the legacy data
df_legacy.shape
# Plot between the 'OrderPrice' and 'Amount_Charged'
plt.figure(figsize=(12,7))
plt.plot(df_legacy['Amount_Charged'], c='g', label='Amount Charged')
plt.plot(df_legacy['OrderPrice'], c='r', label='Order Price')
plt.title('OrderPrice vs Amount_Charged')
plt.legend()
plt.xlabel('Order Index')
plt.ylabel('Price')
plt.show()
# Plot for the ratings
plt.figure(figsize=(12,7))
plt.plot(df_legacy['Rating'], label='Rating')
plt.title('Rating')
plt.legend()
plt.xlabel('Order Index')
plt.ylabel('Rating')
plt.show()
# Removing legacy data of products which has rating greater than 0 (behaves like outlier)
df_legacy = df_legacy[df_legacy['Rating'] == 0]
df_legacy.shape
# Reset the index after removing some rows
df_legacy.reset_index(inplace=True, drop=True)<jupyter_output><empty_output><jupyter_text>Taking insights from Sample data<jupyter_code>df_sample.head()
# Dropping the Unnamed columns
df_sample.drop(columns=['Unnamed: 0'], inplace=True)
df_sample.head()
# Shape of the sample data
df_sample.shape
# Check for the null value row
df_sample.isnull().sum()
# Check for duplicated
df_sample[df_sample.duplicated()]
# Drop duplicates
df_sample.drop_duplicates(inplace=True)
df_sample.shape
# Reset the index after removing some rows
df_sample.reset_index(inplace=True, drop=True)<jupyter_output><empty_output><jupyter_text>Taking insights from Inventory data<jupyter_code>df_inventory.head()
# Dropping the Unnamed columns
df_inventory.drop(columns=['Unnamed: 0'], inplace=True)
df_inventory.head()
# Shape of the inventory data
df_inventory.shape
# Check for the null value rows
df_inventory.isnull().sum()
# Check for duplicated
df_inventory[df_inventory.duplicated()]
# Check for duplicated product names
df_inventory[df_inventory['ProductName'].duplicated()]
# Drop duplicated product name records
df_inventory.drop([328], inplace=True)
df_inventory.shape
# Plot for number of products which has rating between 0 to 1, 1 to 2 and so on
plt.figure(figsize=(12,7))
plt.hist(df_inventory['Rating'], range(6))
plt.title('Product count of Rating i to i+1')
plt.xlabel('Rating')
plt.ylabel('Product count')
plt.show()
# Reset the index after removing some rows
df_inventory.reset_index(inplace=True, drop=True)<jupyter_output><empty_output><jupyter_text>Remove data from Legacy on the basis of gift product id which is not available in Inventory<jupyter_code># Taking all the product id's available in Inventory
product_ids = df_inventory['ProductId'].values
product_ids, len(product_ids)
# Helper function to know whether product id exist in inventory or not
def available_inventory(gift_box):
    ids = list(map(int, gift_box.split(',')))
    for id in ids:
        if id not in product_ids:
            return False
    return True
# Taking intersection from the product id available in GiftBox and store it availability status as a new column
df_legacy['Available_Inventory'] = df_legacy['GiftBox'].apply(available_inventory)
df_legacy.head()
df_legacy.shape
# All Product id in Legacy data are available in inventory
df_legacy = df_legacy[df_legacy['Available_Inventory'] == True]
df_legacy.shape
# Reset the index after removing some rows
df_legacy.reset_index(inplace=True, drop=True)<jupyter_output><empty_output><jupyter_text>Create a dataset for the content based filtering<jupyter_code>df_inventory.head()
# Adding a new column tag
df_inventory['Tag'] = df_inventory['ProductName'] + ' ' + df_inventory['ProductTag']
df_inventory.head()
# Creating object of tokenizer and lemmatizer
tokenizer = RegexpTokenizer(r"\w+")
lemmatizer = WordNetLemmatizer()
# Function to clean the Tag column in inventory
def clean(row):
    # convert into lower character
    row = row.lower()
    # split string into words
    words = tokenizer.tokenize(row)
    # remove stop words and apply lemmatization
    words = [lemmatizer.lemmatize(word.strip()) for word in words if word not in stopwords.words('english')]
    # convert word into string again and return
    return ' '.join(words)
%%time
# Applying clean tag function to transform the text
df_inventory['Tag'] = df_inventory['Tag'].apply(clean)
df_inventory.head()
# Taking required columns for recommendation into new dataframe
df_inventory_final = df_inventory[['ProductId', 'ProductName', 'ProductTag', 'Rating', 'Cost', 'Price', 'Tag']]
df_inventory_final.head()
# Shape of the final inventory data
df_inventory_final.shape<jupyter_output><empty_output><jupyter_text>Create a dataset for predictive modeling<jupyter_code>df_legacy.head()
# Create a new dataframe of legacy data
df_legacy_final = pd.DataFrame(columns=['Preferences', 'Category'])

for i in range(df_legacy.shape[0]):
    ids = list(map(int, df_legacy['GiftBox'][i].split(',')))
    preference = df_legacy['Preferences'][i]
    for id in ids:
        category = df_inventory[df_inventory['ProductId'] == id]['ProductTag'].values[0]
        df_legacy_final.loc[len(df_legacy_final.index)] = [preference, clean(category)]
        #categories = df_inventory[df_inventory['ProductId'] == id]['ProductTag'].values[0].split(',')
        #for category in categories:
        #    if category not in ['', '1', '2']:
        #        df_legacy_final.loc[len(df_legacy_final.index)] = [preference, category.lower().strip()]
# Shape of the final legacy data
df_legacy_final.shape
# Check for the duplicate column
df_legacy_final[df_legacy_final.duplicated()]
# Drop duplicated
df_legacy_final.drop_duplicates(inplace=True)
# Shape of the final legacy data
df_legacy_final.shape
# Reset the index after removing some rows
df_legacy_final.reset_index(inplace=True, drop=True)
# Unique Category in final legacy data
df_legacy_final['Category'].unique()
# Number of unique Category in final legacy data
df_legacy_final['Category'].nunique()
# Plot for the count of number of uniques 'Category' values
plt.figure(figsize=(12, 7))
plt.bar(df_legacy_final['Category'].value_counts().index, df_legacy_final['Category'].value_counts().values)
plt.title('Number of records in each category')
plt.xticks(rotation='vertical')
plt.xlabel('Category')
plt.ylabel('Number of records')
plt.plot()<jupyter_output><empty_output><jupyter_text>Remove the categories that has less than 30 records<jupyter_code># all the categories which has record more than 30
category_bool = df_legacy_final['Category'].value_counts() > 30
category_bool
# name of the categories
categories = category_bool[category_bool].index
categories
# update dataframe which has only categories that has record more than 30
df_legacy_final = df_legacy_final[df_legacy_final['Category'].isin(categories)]
df_legacy_final.head()
# Shape of the final legacy data
df_legacy_final.shape
# Reset the index after removing some rows
df_legacy_final.reset_index(inplace=True, drop=True)
# Unique Category in final legacy data
df_legacy_final['Category'].unique()
# Number of unique Category in final legacy data
df_legacy_final['Category'].nunique()
# Plot for the count of number of uniques 'Category' values
plt.figure(figsize=(12, 7))
plt.bar(df_legacy_final['Category'].value_counts().index, df_legacy_final['Category'].value_counts().values)
plt.title('Number of records in each category')
plt.xticks(rotation='vertical')
plt.xlabel('Category')
plt.ylabel('Number of records')
plt.plot()<jupyter_output><empty_output><jupyter_text>Export the data frame<jupyter_code>pickle.dump(df_inventory_final, open('eda_data/recommendation.pkl', 'wb'))
pickle.dump(df_legacy_final, open('eda_data/predict.pkl', 'wb'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11456228060844591, 'fraction_numerical': 0.005956813104988831, 'mean_word_length': 6.815461346633416, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '570885', 'license': 'MIT', 'n_tokens_mistral': 3174, 'n_tokens_neox': 3034, 'n_words': 799, 'path': 'data_analysis.ipynb', 'repo_name': 'yash0307jain/curated_gift_boxes', 'size': '9401'}