<jupyter_start><jupyter_text>CASE STUDY: LATE DELIVERY ROOT CAUSE  READING THE DATA The below code reads the data from excel file using python package pandas<jupyter_code>import pandas as pd

data=pd.read_excel(r"D:\files\5th Sem\MS4110-Introduction to Data Analytics\assignment\CASE STUDY_DATA_LATE DELIVERY ROOT CAUSE.xlsx")
data.head()
data.drop(['delivered_to_plan'],axis=0).to_excel('testing_X')
data.describe(include='all')
#we can see the number of unique categories in the categorical data
for i in data:
    print(i,data[i].dtype,sep='\t\t\t',end='\n')
#confirming the data type of the date time variables as datetime64[ns] which can
# be specially operated to find time intervals<jupyter_output>order_id			object
country			object
shipping_method			object
units_per_order			int64
facility			object
product_category			object
on_sale			object
transit_days			int64
datetime_ordered			datetime64[ns]
datetime_sourced			datetime64[ns]
datetime_product_ready			datetime64[ns]
datetime_planned			datetime64[ns]
deadline_source			float64
deadline_make			int64
deadline_deliver			int64
delivered_to_plan			object<jupyter_text>CREATING NEW COLUMNS THAT RECORD THE TIME INTERVAL BETWEEN DATES We create new columns to record the time intervals between the carious dates and the time ordered as this is easier to work with and captures the necessary information from this data<jupyter_code>#cell pending deletion;was used to confirm the units of the three columns


from datetime import timedelta
# for i,j in enumerate((data.datetime_sourced-data.datetime_ordered)):
#     print(j.seconds/60,data.deadline_source[i])
print(((data.datetime_sourced-data.datetime_ordered).map(lambda x: x.seconds/(60))<data.deadline_source).values.sum(),
      (((data.datetime_product_ready-data.datetime_ordered).map(lambda x: int(x.seconds/(3600))))<data.deadline_make).values.sum(),
      (((data.datetime_product_ready-data.datetime_ordered).map(lambda x: int(x.days/(1))))<data.deadline_deliver).values.sum())
print(((data.datetime_sourced-data.datetime_ordered).map(lambda x: x.seconds/(60))>data.deadline_source).values.sum(),
      (((data.datetime_product_ready-data.datetime_ordered).map(lambda x: int(x.seconds/(3600))))>data.deadline_make).values.sum(),
      (((data.datetime_product_ready-data.datetime_ordered).map(lambda x: int(x.days/(1))))>data.deadline_deliver).values.sum())
#converting the given dates to time intervals which ccan be worked with. check the lambda
# function for conversion to seconds minutes or days

data['time_sourced']=(data.datetime_sourced-data.datetime_ordered).map(lambda x:int(x.seconds/(60)))
data['time_make']=(data.datetime_product_ready-data.datetime_ordered).map(lambda x: int(x.seconds/(3600)))
data['time_ready']=(data.datetime_product_ready-data.datetime_ordered).map(lambda x: int(x.days/(1)))<jupyter_output><empty_output><jupyter_text>IMPUTING NULL VALUES<jupyter_code>#checking for null values in columns

for i in data:
    print(i,data[i].isnull().values.any(),sep='\t\t\t')
    
#surveying the null values. Below the dataset we can see 535 null values

data[data.isnull().values]
#pending deletion. to check the plot of the misiing na values. Can keep/ expand for aesthetics

import matplotlib.pyplot as plt
from tqdm import tqdm_notebook as tqdm
import seaborn as sns
%matplotlib inline

plt.hist(data.deadline_source.dropna(),bins=100)
plt.show()
plt.hist(data.time_sourced.dropna(),bins=100)
plt.show()
#creating a new variable data_corr with ehich we can work on our dataset 
# and drop the colums irrelevant to the problem

data_corr=data;
data_corr=data_corr.drop(['order_id','delivered_to_plan','datetime_ordered','datetime_sourced','datetime_product_ready','datetime_planned'],axis=1)
data_corr.head()
#creating a dictionary so that we can map the categorical values. can create the maps manually as well.

col=['country','shipping_method','facility','product_category','on_sale']#categorical colums
tmp=[];
for i,j in enumerate(col):
    a=data_corr[j].unique()
    tmp.append(dict(zip(a,list(range(0,len(a))))))
    print(tmp[i])
    data_corr[j]=data_corr[j].map(tmp[i])
data_corr.head()
#pushin the column with nan values:deadline_source to the front for easy matrix manipulation whic 
# will be required later

a=list(data_corr.columns[0:7])
b=list(data_corr.columns[8:13])
c=[data_corr.columns[7]]
print(c+a+b)
data_corr=data_corr[c+a+b]
data_corr.head()<jupyter_output><empty_output><jupyter_text>![Formula1.png](attachment:Formula1.png)![Formula2.png](attachment:Formula2.png)<jupyter_code>x_b=data_corr[data_corr.isnull().values].iloc[:,1:];#data without the null values
x_b.shape
import numpy as np 
mew=np.mean(data_corr.dropna(),axis=0)#mean
sigma=np.cov(data_corr.dropna(),rowvar=0)#covariance matrix

mew_a=mew[0].reshape(1,1)#mean of deadline_source
mew_b=mew[1:].values.reshape(12,1)#remaining means

np.set_printoptions(suppress=True,precision=3)
# print(mew.iloc[7])

#see above image
sigma_aa=sigma[0,0].reshape(1,1)
sigma_bb=sigma[1:,1:].reshape(12,12)
sigma_ab=sigma[0,1:].reshape(1,12)
sigma_ba=sigma[1:,0].reshape(12,1)

#mew a|b is the imputed value(src:deovrath mehendale)
#tile will duplicate the mew matrix for 535 rows so as to avoid cubersome for loops;
#use numpy for matrix operations since python does not treat lists as matrices
a= np.tile(mew_a,(535,1))
b=np.matmul(sigma_ab,np.linalg.inv(sigma_bb))
c=np.subtract(x_b,np.tile(mew_b.T,(535,1))).T#reason given above for using tile
d=np.matmul(b,c).T
e=np.add(a,d)
x_a=e
# mew_a_giv_b=(np.tile(mew_a,(535,1))
#              +np.matmul(np.matmul(sigma_ab*np.linalg.inv(sigma_bb)),
#              ((x_b-np.tile(mew_b.T,(535,1)).T))))


#below code pending deletion but very helpful in finding errors for matrix multiplication
print(sigma_aa.shape,sigma_ab.shape,sigma_ba.shape,sigma_bb.shape,sep='\n')
print(np.matmul(sigma_ab,np.linalg.inv(sigma_bb)).shape)
print(mew_b.shape,mew_a.shape)
print(x_a.shape)
#replacing data corr nan values
data_corr.iloc[list(data_corr[data_corr.isnull().values].index),[0]]=x_a
print(data_corr.isnull().values.any())#double check
data_corr.head()<jupyter_output>False<jupyter_text>GETTING RID OF COLLINEARITY USING VIF(VARIANCE INFLATION FACTOR)<jupyter_code>import seaborn as sns

#collinearity heatmap
sns.heatmap(data_corr.corr(),annot=True,fmt=".2f",cmap="coolwarm")<jupyter_output><empty_output><jupyter_text>from statsmodels.stats.outliers_influence import variance_inflation_factorvif = pd.DataFrame()vif["VIF Factor"] = [variance_inflation_factor(data_corr.values.astype('int'), i) for i in range(data_corr.shape[1])]vif["features"] = data_corr.columnsvif.round(1)<jupyter_code>def calculate_vif_(X, thresh=5.0):
    variables = list(range(X.shape[1]))
    dropped = True
    while dropped:
        dropped = False
        vif = [variance_inflation_factor(X.iloc[:, variables].values.astype(int), ix)
               for ix in range(X.iloc[:, variables].shape[1])]

        maxloc = vif.index(max(vif))
        if max(vif) > thresh:
            print('dropping \'' + X.iloc[:, variables].columns[maxloc] +
                  '\' at index: ' + str(maxloc))
            del variables[maxloc]
            dropped = True

    print('Remaining variables:')
    print(X.columns[variables])
    return X.iloc[:, variables],X.columns[variables]<jupyter_output><empty_output><jupyter_text>X,rem_cols=calculate_vif_(data_corr,4)the second value is the threshold above which vif the columns will be droppedX.head()<jupyter_code>from  sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA

def PCAconv(data,n=13):
    tmp=StandardScaler().fit_transform(data.values)
    pca = PCA(n_components=n)
    tmp.shape
    # np.isnan(np.min(tmp))
    principalComponents = pca.fit_transform(tmp)
    
    X = pd.DataFrame(data = principalComponents)
    X.shape
    return(X)
X=PCAconv(data_corr,11)
y=data.delivered_to_plan;
plt.hist(y)
y=y.map({'PASS':1,'FAIL':0})
print(y.unique())
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)
print(X_train.shape, X_test.shape,y_train.shape, y_test.shape)
print(X_test.head())
X_test.to_excel('Testing_X.xlsx')
print(y_train.head())
X_train.head()<jupyter_output>51812    1
37618    1
35827    0
9492     0
31343    1
Name: delivered_to_plan, dtype: int64<jupyter_text>DOWNSAMPLING THE DATA<jupyter_code>X_train.join(y_train).head()
tmp=X_train.join(y_train)
b=tmp[tmp.delivered_to_plan==0]#.sample(n=3000,random_state=4)
a=tmp[tmp.delivered_to_plan==1].sample(n=int(b.shape[0]*(0.6/0.4)),random_state=4)
c=a.append(b)
X_train=c[c.columns[0:(c.shape[1]-1)]]
y_train=c[c.columns[c.shape[1]-1]]
plt.hist(y_train)
X_train.describe()
# count=0
# for i in tqdm(a+b):
# #     print(i)
#     try:
#         _=X_train.iloc[i,:]
#     except:
#         count=count+1
#         continue
# print((count))
def plot_roc_curve(fpr, tpr, label=None):
    """
    The ROC curve, modified from 
    Hands-On Machine learning with Scikit-Learn and TensorFlow; p.91
    """
    plt.figure(figsize=(8,8))
    plt.title('ROC Curve')
    plt.plot(fpr, tpr, linewidth=2, label=label)
    plt.plot([0, 1], [0, 1], 'k--')
    plt.axis([-0.005, 1, 0, 1.005])
    plt.xticks(np.arange(0,1, 0.05), rotation=90)
    plt.xlabel("False Positive Rate")
    plt.ylabel("True Positive Rate (Recall)")
    plt.legend(loc='best')
def adjusted_classes(y_scores, t):
    """
    This function adjusts class predictions based on the prediction threshold (t).
    Will only work for binary classification problems.
    """
    return [1 if y >= t else 0 for y in y_scores]
def precision_recall_threshold(p, r, thresholds, t=0.5):
    """
    plots the precision recall curve and shows the current value for each
    by identifying the classifier's threshold (t).
    """
    
    # generate new class predictions based on the adjusted_classes
    # function above and view the resulting confusion matrix.
    y_pred_adj = adjusted_classes(y_pred_, t)
    print(pd.DataFrame(confusion_matrix(y_test, y_pred_adj),
                       columns=['pred_neg', 'pred_pos'], 
                       index=['neg', 'pos']))
    
    # plot the curve
    plt.figure(figsize=(8,8))
    plt.title("Precision and Recall curve ^ = current threshold")
    plt.step(r, p, color='b', alpha=0.2,
             where='post')
    plt.fill_between(r, p, step='post', alpha=0.2,
                     color='b')
    plt.ylim([0.5, 1.01]);
    plt.xlim([0.5, 1.01]);
    plt.xlabel('Recall');
    plt.ylabel('Precision');
    # plot the current threshold on the line
    close_default_clf = np.argmin(np.abs(thresholds - t))
    plt.plot(r[close_default_clf], p[close_default_clf], '^', c='k',
             markersize=15)
from sklearn.svm import SVC,LinearSVC
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_val_score
from sklearn.metrics import confusion_matrix
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.metrics import precision_recall_curve

classifiers = [
#     KNeighborsClassifier(3),
#     SVC(kernel="linear", C=0.025),
#     SVC(gamma=2, C=1),
#     GaussianProcessClassifier(1.0 * RBF(1.0)),
    DecisionTreeClassifier(max_depth=5),
    RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
#     MLPClassifier(alpha=1),
#     AdaBoostClassifier(),
#     GaussianNB(),
    QuadraticDiscriminantAnalysis(),
    LogisticRegression(random_state=0, solver='lbfgs',
                         multi_class='multinomial',verbose=1).fit(X_train, y_train)]
names = ["Decision Tree", "Random Forest", "QDA",'Logistic Regression']

for name, classifier in zip(names, classifiers):
    classifier.fit(X_train, y_train)
#     scores = cross_val_score(classifier, X, y, cv=5,verbose=1)
    y_pred = classifier.predict(X_test)
    print(name,'acc sc',accuracy_score(y_test, y_pred))

    scores = cross_val_score(classifier, X, y, cv=2,verbose=1)
    print(name,'cross val',scores)
    if hasattr(classifier, "decision_function"):
        y_pred_ = classifier.decision_function(X_test.values)
        p, r, thresholds = precision_recall_curve(y_test, y_pred_)
        precision_recall_threshold(p, r, thresholds, 0.30)
    else:
        y_pred_ = classifier.predict_proba(X_test.values)
        fpr,tpr,auc_scores=roc_curve(y_test,y_pred_[:,1])
        plot_roc_curve(fpr,tpr)
    
    
    
    
    

# classifier_1 =LinearSVC(dual=True,verbose=1)
# classifier_1.fit(X_train,y_train)
# y_pred = classifier_1.predict(X_test)
# print(accuracy_score(y_test, y_pred))

# # classifier = LogisticRegression(random_state=0, solver='liblinear',
# #                          multi_class='ovr',verbose=1).fit(X_train, y_train)
# classifier = LogisticRegression(random_state=0, solver='lbfgs',
#                          multi_class='multinomial',verbose=1).fit(X_train, y_train)
# y_pred = classifier.predict(X_test)
# print(accuracy_score(y_test, y_pred))

# scores = cross_val_score(classifier, X, y, cv=5,verbose=1)
# print(scores)
from sklearn.metrics import confusion_matrix
tn, fp, fn, tp = confusion_matrix(y_test, y_pred).ravel()
print(tn, fp, fn, tp)
true_positive_rate = tp/(tp+fn)
false_positive_rate = fp/(tn+fp)
precision = tp/(tp+fp)
print(true_positive_rate)
print(false_positive_rate)
print(precision)

from sklearn.metrics import roc_curve


y_pred_ = classifier.predict_proba(X_test.values)
# print(y_pred_)
fpr,tpr,auc_scores=roc_curve(y_test,y_pred_[:,1])
# print(fpr,tpr,auc_scores)
plot_roc_curve(fpr,tpr)
# plt.savefig(r'C:\Users\Aashish\Desktop\test.png')
from sklearn.externals import joblib
joblib.dump(classifier,'LogiRegr.joblib')
def testing(filename,filename2):
    data=pd.read_excel(filename)
    data['time_sourced']=(data.datetime_sourced-data.datetime_ordered).map(lambda x:int(x.seconds/(60)))
    data['time_make']=(data.datetime_product_ready-data.datetime_ordered).map(lambda x: int(x.seconds/(3600)))
    data['time_ready']=(data.datetime_product_ready-data.datetime_ordered).map(lambda x: int(x.days/(1)))
    data_corr=data;
    data_corr=data_corr.drop(['order_id','delivered_to_plan','datetime_ordered','datetime_sourced','datetime_product_ready','datetime_planned'],axis=1)
    col=['country','shipping_method','facility','product_category','on_sale']#categorical colums
    tmp=[];
    for i,j in enumerate(col):
        a=data_corr[j].unique()
        tmp.append(dict(zip(a,list(range(0,len(a))))))
        print(tmp[i])
        data_corr[j]=data_corr[j].map(tmp[i])
    X=PCAconv(data_corr,11)
    
    
    
    classifier = joblib.load(filename2)
    
    y_pred = classifier.predict(X)
#     print(accuracy_score(y_test, y_pred))
    sub=pandas.Dataframe(data=y_pred)
    sub.to_csv('Predicted.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13492274906766116, 'fraction_numerical': 0.019179541822056473, 'mean_word_length': 5.172215371968763, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '303175', 'license': 'MIT', 'n_tokens_mistral': 5662, 'n_tokens_neox': 5353, 'n_words': 991, 'path': 'E_Slot_Assign.ipynb', 'repo_name': 'asmaknikar/E_Slot_Assignment', 'size': '15016'}