<jupyter_start><jupyter_text>To define baseline model with basic data preprocessing.<jupyter_code>cust_demo_data_expt = cust_demo_data.copy()
cust_demo_data_expt['marital_status'].fillna('Unspecified',inplace=True)
cust_demo_data_expt['no_of_children'].fillna(0,inplace=True)
cust_demo_data_expt['age_range'].replace(['18-25','26-35','36-45','46-55','56-70','70+'],[18,26,36,46,56,70],inplace=True)
cust_demo_data_expt['family_size'].replace('5+',5,inplace=True)
cust_demo_data_expt['no_of_children'].replace('3+',3,inplace=True)
cust_tran_data_expt = cust_tran_data.copy()
cust_tran_data_expt = pd.merge(cust_tran_data_expt,coupon_data,how='inner',on='item_id')
cust_tran_data_expt.drop('date',axis=1,inplace=True)
for column in ['quantity','coupon_discount','other_discount','selling_price']:
    tran_summation(column)

cust_tran_data_expt.drop_duplicates(subset=['customer_id','item_id','coupon_id'], keep='first', inplace=True)
train_data_merge = pd.merge(train_data,cust_tran_data_expt,how='inner',on=['customer_id','coupon_id'])
train_data_merge = pd.merge(train_data_merge,cust_demo_data_expt,how='left',on='customer_id')
train_data_merge = pd.merge(train_data_merge,item_data,how='left',on='item_id')
train_data_merge.drop('marital_status',axis=1,inplace=True)
train_data_merge.fillna({'age_range':0,'rented':0,'family_size':0,'no_of_children':0,'income_bracket':0},inplace=True)
train_data_merge['family_size'].astype('int8')
train_data_merge['no_of_children'].astype('int8')
train_data_merge = pd.get_dummies(train_data_merge, columns=['brand_type','category'], drop_first=False)
X = train_data_merge.drop('redemption_status', axis=1)
y = train_data_merge['redemption_status']

X_train,X_test,y_train,y_test = train_test_split(X,y,train_size=0.7,random_state=7)
# defining the model
classifier = LogisticRegression(solver='lbfgs',max_iter=10000)

# fitting the model
classifier.fit(X_train,y_train)

# predicting test result with model
y_pred = classifier.predict(X_test)

# Creating Classification report for Logistic Regression Baseline model

print ("Classification Report for Baseline Logistic Regression")
print(classification_report(y_test,y_pred))

report = pd.DataFrame(classification_report(y_test,y_pred,output_dict=True)).transpose()

write_file(file_write_cnt,'No','No','Yes',len(X.columns),list(X.columns),'Logistic Regresssion',report['precision'][1],report['recall'][1],report['support']['accuracy'],'Baseline Model')
file_write_cnt = file_write_cnt + 1
# defining the model
classifier = RandomForestClassifier(n_estimators=100)

# fitting the model
classifier.fit(X_train,y_train)

# predicting test result with model
y_pred = classifier.predict(X_test)

# Creating Classification report for RandomForest Classifier Baseline model

print ("Classification Report for Baseline RandomForest Classifier")
print(classification_report(y_test,y_pred))

report = pd.DataFrame(classification_report(y_test,y_pred,output_dict=True)).transpose()

write_file(file_write_cnt,'No','No','Yes',len(X.columns),X.columns,'Random Forest Classifier',report['precision'][1],report['recall'][1],report['support']['accuracy'],'Baseline Model')
file_write_cnt += 1<jupyter_output><empty_output><jupyter_text>One Hot Encoding<jupyter_code>del train_data_merge
train_data_merge = pd.merge(train_data,cust_tran_data_expt,how='inner',on=['customer_id','coupon_id'])
train_data_merge = pd.merge(train_data_merge,cust_demo_data,how='left',on='customer_id')
train_data_merge = pd.merge(train_data_merge,item_data,how='left',on='item_id')
train_data_merge['no_of_children'].fillna(0,inplace=True)
train_data_merge.fillna({'marital_status':'Unspecified','rented':'Unspecified','family_size':'Unspecified','age_range':'Unspecified'},inplace=True)
train_data_merge['income_bracket'].fillna(train_data_merge['income_bracket'].mean(),inplace=True)
train_data_merge.drop(['id'],axis=1,inplace=True)
train_data_merge['no_of_children'].replace('3+',3,inplace=True)
train_data_merge['no_of_children'].astype('int')
train_data_merge = pd.get_dummies(train_data_merge, columns=['age_range','marital_status','rented','family_size','brand_type','category'], drop_first=False)
X = train_data_merge.drop('redemption_status', axis=1)
y = train_data_merge['redemption_status']

X_train,X_test,y_train,y_test = train_test_split(X,y,train_size=0.7,random_state=7)
# defining the model
classifier = RandomForestClassifier(n_estimators=100)

# fitting the model
classifier.fit(X_train,y_train)

# predicting test result with model
y_pred = classifier.predict(X_test)

# Creating Classification report for RandomForest Classifier Baseline model

print ("Classification Report for RandomForest Classifier")
print(classification_report(y_test,y_pred))

report = pd.DataFrame(classification_report(y_test,y_pred,output_dict=True)).transpose()

write_file(file_write_cnt,'No','No','Yes',len(X.columns),X.columns,'Random Forest Classfier',report['precision'][1],report['recall'][1],report['support']['accuracy'],'with OHE')
file_write_cnt += 1<jupyter_output><empty_output><jupyter_text>Label Encoding<jupyter_code>del train_data_merge
train_data_merge = pd.merge(train_data,cust_tran_data_expt,how='inner',on=['customer_id','coupon_id'])
train_data_merge = pd.merge(train_data_merge,cust_demo_data,how='left',on='customer_id')
train_data_merge = pd.merge(train_data_merge,item_data,how='left',on='item_id')
train_data_merge['no_of_children'].fillna(0,inplace=True)
train_data_merge.fillna({'marital_status':'Unspecified','rented':'Unspecified','family_size':'Unspecified','age_range':'Unspecified'},inplace=True)
train_data_merge['income_bracket'].fillna(train_data_merge['income_bracket'].mean(),inplace=True)
train_data_merge['no_of_children'].replace('3+',3,inplace=True)
train_data_merge['no_of_children'].astype('int')
train_data_merge.drop(['id'],axis=1,inplace=True)
for column in ['marital_status','rented','family_size','age_range','brand_type','category']:
    label_encode(column)
X = train_data_merge.drop('redemption_status', axis=1)
y = train_data_merge['redemption_status']

X_train,X_test,y_train,y_test = train_test_split(X,y,train_size=0.7,random_state=7)
# defining the model
classifier = RandomForestClassifier(n_estimators=100)

# fitting the model
classifier.fit(X_train,y_train)

# predicting test result with model
y_pred = classifier.predict(X_test)

# Creating Classification report for RandomForest Classifier Baseline model

print ("Classification Report for RandomForest Classifier")
print(classification_report(y_test,y_pred))

report = pd.DataFrame(classification_report(y_test,y_pred,output_dict=True)).transpose()

write_file(file_write_cnt,'No','No','Yes',len(X.columns),X.columns,'Random Forest Classifier',report['precision'][1],report['recall'][1],report['support']['accuracy'],'with Label Encoding')
file_write_cnt += 1<jupyter_output><empty_output><jupyter_text>Feature Engineering with Treatment of Campaign Date, Transaction Date and using Coupon Redemption percentage as a value to convert categorical columns to integers<jupyter_code>del train_data_merge
del cust_tran_data_expt
campaign_data_expt = campaign_data.copy()
campaign_data_expt['start_date_q'] = campaign_data_expt['start_date'].map(lambda x: date_q(x,'/'))
campaign_data_expt['end_date_q'] = campaign_data_expt['end_date'].map(lambda x: date_q(x,'/'))
campaign_data_expt.drop(['start_date','end_date'],axis=1,inplace=True)
cust_tran_data_expt = cust_tran_data.copy()
cust_tran_data_expt = pd.merge(cust_tran_data_expt,coupon_data,how='inner',on='item_id')
cust_tran_data_expt['tran_date_q'] = cust_tran_data_expt['date'].map(lambda x: date_q(x,'-'))
cust_tran_data_expt.drop('date',axis=1,inplace=True)

for column in ['quantity','coupon_discount','other_discount','selling_price']:
    tran_summation_1(column)

cust_tran_data_expt.drop_duplicates(subset=['customer_id','item_id','coupon_id','tran_date_q'], keep='first', inplace=True)
train_data_merge = pd.merge(train_data,cust_tran_data_expt,how='inner',on=['customer_id','coupon_id'])
train_data_merge = pd.merge(train_data_merge,cust_demo_data,how='left',on='customer_id')
train_data_merge = pd.merge(train_data_merge,item_data,how='left',on='item_id')
train_data_merge = pd.merge(train_data_merge,campaign_data_expt,how='left',on='campaign_id')
train_data_merge['no_of_children'].fillna(0,inplace=True)
train_data_merge.fillna({'marital_status':'Unspecified','rented':'Unspecified','family_size':'Unspecified','age_range':'Unspecified','income_bracket':'Unspecified'},inplace=True)
train_data_merge.drop('id',axis=1,inplace=True)
train_data_merge['no_of_children'].replace('3+',3,inplace=True)
for column in ['customer_id','coupon_id','item_id','campaign_id','no_of_children','marital_status','rented','family_size','age_range','income_bracket','start_date_q','end_date_q','tran_date_q','brand','category']:
    cat_percent(column)
train_data_merge = pd.get_dummies(train_data_merge, columns=['campaign_type','brand_type'], drop_first=False)
X = train_data_merge.drop('redemption_status', axis=1)
y = train_data_merge['redemption_status']
feature_sel = [5,10,15,23]

rforc = RandomForestClassifier(n_estimators=100)

for i in feature_sel:
    rfe = RFE(rforc, i)
    rfe.fit(X, y)

# Selecting columns

    sel_cols = []
    for a, b, c in zip(rfe.support_, rfe.ranking_, X.columns):
        if b == 1:
            sel_cols.append(c)
    print ('Number of features selected are ::',i)
    print ('Columns Selected are ::',sel_cols)

# Creating new DataFrame with selected columns only as X

    X_sel = X[sel_cols]

# Split data in to train and test

    X_sel_train, X_sel_test, y_sel_train, y_sel_test = train_test_split(X_sel, y, train_size=0.7, random_state=7)
    
# Fit and Predict the model using selected number of features    
    grid={"n_estimators":[100]}
    rforc_cv = GridSearchCV(rforc,grid,cv=10)
    rforc_cv.fit(X_sel_train, y_sel_train)
    rforc_pred = rforc_cv.predict(X_sel_test)

# Classification Report    
    
    print(classification_report(y_sel_test,rforc_pred))
    
    report = pd.DataFrame(classification_report(y_sel_test,rforc_pred,output_dict=True)).transpose()

    write_file(file_write_cnt,'No','No','Yes',len(X_sel.columns),X_sel.columns,'Random Forest Classifier',report['precision'][1],report['recall'][1],report['support']['accuracy'],'Treating Date and Label Encoding with RFE')
    file_write_cnt += 1<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14756460755115766, 'fraction_numerical': 0.011912767797098664, 'mean_word_length': 12.09433962264151, 'pattern_counts': {'":': 1, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '22088', 'license': 'MIT', 'n_tokens_mistral': 3851, 'n_tokens_neox': 3730, 'n_words': 437, 'path': 'Hackathon_ML Sample Problems/Amex Dataset/ML on Amex Dataset Final.ipynb', 'repo_name': 'girishvankudre/hackathon_ml_sample', 'size': '10409'}