<jupyter_start><jupyter_text>Analysis Data Portfolioid (string) - offer id offer_type (string) - type of offer ie BOGO, discount, informational difficulty (int) - minimum required spend to complete an offer reward (int) - reward given for completing an offer duration (int) - time for offer to be open, in days channels (list of strings) """<jupyter_code>portfolio<jupyter_output><empty_output><jupyter_text>Transcript<jupyter_code>transcript.sample(10)
transcript.event.value_counts()
transcript.isna().sum()<jupyter_output><empty_output><jupyter_text>Profiles<jupyter_code>profile.sample(10)
profile.isna().sum()
sns.distplot(profile.age,kde=False) 
profile[profile['age']>110]
profile_df = profile.copy()
profile_df.dropna(inplace=True)
profile_df.reset_index(drop=True, inplace=True)
sns.distplot(profile_df.income)
gender_count = profile_df.gender.value_counts()
genders = profile_df.gender.unique()
sns.barplot(gender_count,y=genders)
ages_df = profile_df
ages_df.loc[(profile_df['age'] >10) & (profile_df['age'] <=19 ), 'Age'] = "10's"
ages_df.loc[(profile_df['age'] > 20) & (profile_df['age'] <= 29), 'Age'] = "20's"
ages_df.loc[(profile_df['age'] > 30) & (profile_df['age'] <= 39), 'Age'] = "30's"
ages_df.loc[(profile_df['age'] > 40) & (profile_df['age'] <= 49), 'Age'] = "40's"
ages_df.loc[(profile_df['age'] > 50) & (profile_df['age'] <= 59), 'Age'] = "50's"
ages_df.loc[(profile_df['age'] > 60) & (profile_df['age'] <= 69), 'Age'] = "60's"
ages_df.loc[(profile_df['age'] > 70) & (profile_df['age'] <= 79), 'Age'] = "70's"
ages_df.loc[(profile_df['age'] > 80) & (profile_df['age'] <= 89), 'Age'] = "80's"
ages_df.loc[(profile_df['age'] > 90) & (profile_df['age'] <= 99), 'Age'] = "90's"
ages_df.loc[profile_df['age'] > 100, 'Age'] = "100's"
female_df = ages_df[ages_df['gender'] == 'F']
male_df = ages_df[ages_df['gender'] == 'M']
other_df = ages_df[ages_df['gender'] == 'O']
fig,ax = plt.subplots(figsize=(15,5),
                    nrows=1,
                    ncols=3)
plt.sca(ax[0])
male_age = male_df.sort_values(by='age')
sns.barplot(x=male_age.Age,y=male_df.income)
plt.ylim(0,100000)
plt.title('Male income')

plt.sca(ax[1])
female_age = female_df.sort_values(by='age')
sns.barplot(x=female_age.Age,y=female_df.income)
plt.ylim(0,100000)
plt.title('Female income')

plt.sca(ax[2])
other_age = other_df.sort_values(by='age')
plt.ylim(0,100000)
sns.barplot(x=other_age.Age,y=other_df.income)
plt.title('Other income')
from datetime import date

actual_date = date.today()
actual_date = pd.to_datetime(actual_date)
profile_df['became_member_on'] = pd.to_datetime(profile_df['became_member_on'], format='%Y%m%d')
profile_df.became_member_on = (actual_date - profile_df['became_member_on'])/np.timedelta64(1,'D')
profile_df.rename(columns={'became_member_on': 'membership_days'}, inplace=True)
member = profile_df.groupby(['membership_days'])['id'].count()
plt.figure(figsize=(16,4))
plt.xlabel('Membership Days')
plt.ylabel('Number of people')
plt.plot(member)
#Encoding gender
from sklearn.preprocessing import MultiLabelBinarizer

encoder = MultiLabelBinarizer()
profile_encoded = encoder.fit(profile_df['gender'])
gender_df = pd.DataFrame(profile_encoded.transform(profile_df['gender']),columns=profile_encoded.classes_)
profile_df = profile_df.drop(columns=['gender'],axis=1)
profile_df = pd.concat([profile_df,gender_df],axis=1)
profile_df
portfolio_df = portfolio.copy()
transcript_df = transcript.copy()
transcript_df.event.value_counts()
transcript_df.sample(10)
event_df = transcript_df
event_df['event'] = event_df.event.str.replace(' ','_')
response_df = event_df.copy()
trans_encod =pd.get_dummies(event_df['event'])
event_df = pd.concat([event_df,trans_encod],axis=1)
event_df = event_df.drop(columns=['event'],axis=1)
event_df = event_df.set_index('person')
profile_df = profile_df.set_index('id')
profile_df.info()
#profile_df = profile_df.reset_index()
profile_df = profile_df.rename(columns ={'id': 'person'})
profile_df
merged_df = profile_df.merge(event_df, left_index=True, right_index=True)
female_df = merged_df[merged_df['F'] == 1]
male_df = merged_df[merged_df['M'] == 1]
other_df = merged_df[merged_df['O'] == 1]
def plotOffer(df,df_name):
    receivedOffer = df.offer_received.sum()
    viewedOffer = df.offer_viewed.sum()
    completedOffer = df.offer_completed.sum()
    #d = 'offer_received':[receivedOffer],'offer_viewed':[viewedOffer],'offer_completed':[completedOffer]}
    diff_r_v = ((receivedOffer-viewedOffer)/receivedOffer) * 100
    diff_v_c = ((viewedOffer-completedOffer)/viewedOffer) * 100
    diff_r_c = ((receivedOffer-completedOffer)/receivedOffer) * 100
    
    print(df_name + ' diffrence Received and Viewed: -{:.2f}%'.format(diff_r_v))
    print(df_name + ' diffrence Viewed and Completed: -{:.2f}%'.format(diff_v_c))
    print(df_name + ' diffrence Received and Completed: -{:.2f}%'.format(diff_r_c))
    print('\n')
    return [receivedOffer,viewedOffer,completedOffer]
fig,ax = plt.subplots(figsize=(15,5),
                    nrows=1,
                    ncols=3)
offers = ['offer_received','offer_viewed','offer_completed']
plt.sca(ax[0])
sns.barplot(x=offers,y=plotOffer(male_df , 'Male'))
plt.ylim(0,50000)
plt.title('Male offers')

plt.sca(ax[1])
sns.barplot(x=offers,y=plotOffer(female_df,'Female'))
plt.ylim(0,50000)
plt.title('Female offers')

plt.sca(ax[2])
sns.barplot(x=offers,y=plotOffer(other_df,'Others'))
plt.ylim(0,3000)
plt.title('Other offers')
completed_df = response_df[response_df['event'] == 'offer_completed'].copy()
completed_df = pd.concat([completed_df, completed_df['value'].apply(pd.Series)], axis=1)
completed_df.drop(['value','reward','time'],inplace=True, axis=1)
completed_df.drop_duplicates(inplace=True)
completed_df
viewed_df = response_df[response_df['event'] == 'offer_viewed'].copy()
viewed_df = pd.concat([viewed_df, viewed_df['value'].apply(pd.Series)], axis=1)
viewed_df.drop(['value','time'],inplace=True, axis=1)
viewed_df.rename(columns={'offer id': 'offer_id'},inplace=True)
viewed_df.drop_duplicates(inplace=True)
completed_and_viewed = viewed_df.merge(completed_df,how='left',on=['person','offer_id'])
completed_and_viewed.rename(columns={'event_x': 'offer_viewed', 'event_y': 'offer_completed'},inplace=True)
completed_and_viewed['offer_viewed']= completed_and_viewed.offer_viewed.apply(lambda x: 1 if x == 'offer_viewed' else 0)
completed_and_viewed['offer_completed']= completed_and_viewed.offer_completed.apply(lambda x: 1 if x == 'offer_completed' else 0)
completed_and_viewed.info()
def return_response(df):
    if (df.offer_completed == 1) & (df.offer_viewed == 1):
        response = 1
    else:
        response = 0
    return response
completed_and_viewed['response'] = completed_and_viewed.apply(return_response,axis=1)
completed_and_viewed.drop(['offer_viewed','offer_completed'],axis=1,inplace=True)
completed_and_viewed.shape
profile_df.reset_index(inplace=True)
profile_df = profile_df.rename(columns={'id':'person'})
final_df = completed_and_viewed.merge(profile_df,how='left',on='person')
final_df.dropna(inplace=True)
final_df
final_df = final_df[(final_df.offer_id != '3f207df678b143eea3cee63160fa8bed') & (final_df.offer_id != '5a8bc65990b245e5a138643cd4eb9837')]
offer_dummies =pd.get_dummies(portfolio_df['offer_type'])
portfolio_df = pd.concat([portfolio_df,offer_dummies],axis=1)
portfolio_df = portfolio_df.drop(columns=['offer_type'],axis=1)
"Encoding channel featurees"
from sklearn.preprocessing import MultiLabelBinarizer

encoder = MultiLabelBinarizer()

cat_encoded = encoder.fit(portfolio['channels'])
channels_df = pd.DataFrame(cat_encoded.transform(portfolio['channels']),columns=cat_encoded.classes_)
portfolio_df = portfolio_df.drop(columns=['channels'],axis=1)
portfolio_df = portfolio_df.rename(columns={'id':'offer_id'})
portfolio_df = pd.concat([portfolio_df,channels_df],axis=1)
portfolio_df
portfolio_df = portfolio_df.drop(['reward','difficulty','duration','informational'], axis=1)
portfolio_df
final_df = final_df.merge(portfolio_df,how='left', on='offer_id')
final_df['offer_id'] = final_df['offer_id'].map({'f19421c1d4aa40978ebb69ca19b0e20d':'offer_1',
                          '5a8bc65990b245e5a138643cd4eb9837':'offer_2',
                          '4d5c57ea9a6940dd891ad53e9dbe8da0':'offer_3',
                          '2906b810c7d4411798c6938adc9daaa5':'offer_4',
                          '2298d6c36e964ae4a3e7e9706d1fb8c2':'offer_5',
                          'ae264e3637204a6fb9bb56bc8210ddfd':'offer_6',
                          '9b98b8c7a33c4b65b9aebfe6a799e6d9':'offer_7',
                          'fafdcd668e3743c1bb461111dcafc2a4':'offer_8',
                          '3f207df678b143eea3cee63160fa8bed':'offer_9',
                          '0b1e1539f2cc45b7b9fa7c272da2e1d7':'offer_10'})
final_df['offer_id'].unique()
offer_dummies =pd.get_dummies(final_df['offer_id'])
final_df = pd.concat([final_df,offer_dummies],axis=1)
final_df.drop(['offer_id','person','Age'], axis=1, inplace=True)
final_df
from sklearn.preprocessing import MinMaxScaler, StandardScaler

scaler = StandardScaler()
norm = MinMaxScaler()

income_scal = norm.fit_transform(final_df[['income']])
member_scal = norm.fit_transform(final_df[['membership_days']])
age = norm.fit_transform(final_df[['age']])
final_df['income'] = income_scal
final_df['membership_days'] = member_scal
final_df['age'] = age
final_df.drop('email',inplace=True,axis=1)
corr = final_df.corr()
corr.style.background_gradient(cmap='coolwarm')
final_df.to_csv('dataset.csv',index=False)
final_df_x = final_df.iloc[:,1:]
final_df_y = final_df.iloc[:,0]
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2
bestfeatures = SelectKBest(score_func=chi2, k=10)
fit = bestfeatures.fit(final_df_x,final_df_y)
dfscores = pd.DataFrame(fit.scores_)
dfcolumns = pd.DataFrame(final_df_x.columns)
featureScores = pd.concat([dfcolumns,dfscores],axis=1)
featureScores.columns = ['Specs','Score']
print(featureScores.nlargest(10,'Score'))
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB


X_train, X_test, y_train, y_test = train_test_split(final_df_x,final_df_y, test_size=0.1, random_state=0)
clf = GaussianNB()
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
from sklearn.metrics import classification_report
print(classification_report(y_test,y_pred))
final_df.columns
from sklearn.metrics import confusion_matrix,roc_auc_score
import matplotlib.pyplot as plt


cm = confusion_matrix(y_test,y_pred)
img = plt.matshow(cm, cmap=plt.cm.autumn)
plt.colorbar(img, fraction=0.045)
for x in range(cm.shape[0]):
    for y in range(cm.shape[1]):
        plt.text(x,y, "%0.2f" %cm[x,y], size=12, color='black', ha='center', va='center')
plt.show()
roc_auc_score(y_test,y_pred)
from sklearn.linear_model import LogisticRegression

log_clf = LogisticRegression()
log_clf.fit(X_train, y_train)
y_pred = log_clf.predict(X_test)
cm = confusion_matrix(y_test,y_pred)
img = plt.matshow(cm, cmap=plt.cm.autumn)
plt.colorbar(img, fraction=0.045)
for x in range(cm.shape[0]):
    for y in range(cm.shape[1]):
        plt.text(x,y, "%0.2f" %cm[x,y], size=12, color='black', ha='center', va='center')
plt.show()
print(classification_report(y_test,y_pred))
roc_auc_score(y_test,y_pred)
import xgboost as xgb
model = xgb.XGBClassifier(silent=False, 
                      scale_pos_weight=1,
                      learning_rate=0.01,  
                      colsample_bytree = 0.4,
                      subsample = 0.8,
                      objective='binary:logistic', 
                      n_estimators=1000, 
                      reg_alpha = 0.3,
                      max_depth=10, 
                      gamma=10)
#model = xgb.XGBClassifier(**xgb_params)
model.fit(X_train,y_train)
y_pred = model.predict(X_test)
predictions = [round(value) for value in y_pred]
print(classification_report(y_test,y_pred))
cm = confusion_matrix(y_test,y_pred)
img = plt.matshow(cm, cmap=plt.cm.autumn)
plt.colorbar(img, fraction=0.045)
for x in range(cm.shape[0]):
    for y in range(cm.shape[1]):
        plt.text(x,y, "%0.2f" %cm[x,y], size=12, color='black', ha='center', va='center')
plt.show()
roc_auc_score(y_test,y_pred)
X_train.shape
#!pip install hyperopt
#from hyperopt import hp, fmin, tpe, STATUS_OK
import torch
import torch.nn as nn
import torch.nn.functional as F


class BinaryClassifier(nn.Module):

    def __init__(self, input_features, hidden_dim, output_dim):
        super(BinaryClassifier, self).__init__()
        
        self.fc1 = nn.Linear(input_features,hidden_dim)
        self.fc2 = nn.Linear(hidden_dim,16)
        self.fc3 = nn.Linear(16,output_dim)
        #self.drop = nn.Dropout(0.5)
        self.sig = nn.Sigmoid()
    
    def forward(self, x):
        out = F.relu(self.fc1(x))
        out = F.relu(self.fc2(out))
        #out = self.drop(out)
        out = self.fc3(out)
        return self.sig(out)
        
def train(model, train_loader, epochs, criterion, optimizer, device):
    for epoch in range(1, epochs + 1):
        model.train()
        total_loss = 0
        
        for batch in train_loader:
            
            batch_x, batch_y = batch
            batch_x = batch_x.to(device)
            batch_y = batch_y.to(device)
            optimizer.zero_grad()
            # get predictions from model
            y_pred = model(batch_x)           
            # perform backprop
            loss = criterion(y_pred, batch_y)
            loss.backward()
            optimizer.step()           
            total_loss += loss.data.item()

        print("Epoch: {}, Loss: {}".format(epoch, total_loss / len(train_loader)))
X_train = X_train.astype(np.float32)
X_test = X_test.astype(np.float32)
y_train = y_train.astype(np.float32)
y_test = y_test.astype(np.float32)
X_train = X_train.to_numpy()
y_train = y_train.to_numpy()
X_test = X_test.to_numpy()
y_test = y_test.to_numpy()
from torch.utils.data import DataLoader, Dataset
import torch.optim as optim
import torch.utils.data

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = BinaryClassifier(19, 24, 1).to(device)

print(device)
optimizer = optim.Adam(model.parameters(), lr=0.001)
criterion = nn.BCELoss()


features_train = torch.from_numpy(X_train)
targets_train = torch.from_numpy(y_train)

features_Test = torch.from_numpy(X_test)
targets_Test = torch.from_numpy(y_test)

batch_size = 16
#n_iters = 10000
#num_epochs = n_iters / (len(features_train) / batch_size)
num_epochs = 50


train = torch.utils.data.TensorDataset(features_train,targets_train)
test = torch.utils.data.TensorDataset(features_Test,targets_Test)

train_loader = DataLoader(train, batch_size = batch_size, shuffle = False)
test_loader = DataLoader(test, batch_size = batch_size, shuffle = False)


for epoch in range(1, num_epochs + 1):
        
        model.train()
        total_loss = 0
        
        for i, batch in enumerate(train_loader):
            
            batch_x, batch_y = batch
            batch_x = batch_x.to(device)
            batch_y = batch_y.to(device)
            optimizer.zero_grad()
            # get predictions from model
            y_pred = model(batch_x)           
            # perform backprop
            loss = criterion(y_pred, batch_y)
            loss.backward()
            optimizer.step()           
            total_loss += loss.data.item()

        print("Epoch: {}, Loss: {}".format(epoch, total_loss / len(train_loader)))
def predict_fn(input_data, model):
    print('Predicting class labels for the input data...')

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    
    # Process input_data so that it is ready to be sent to our model.
    data = torch.from_numpy(input_data.astype('float32'))
    data = data.to(device)

    # Put the model into evaluation mode
    model.eval()

    # Compute the result of applying the model to the input data
    # The variable `out_label` should be a rounded value, either 1 or 0
    out = model(data)
    out_np = out.cpu().detach().numpy()
    out_label = out_np.round()
    print(out_label)
    return out_label
predictions = predict_fn(X_test,model)
print(classification_report(y_test,predictions))
cm = confusion_matrix(y_test,predictions)
img = plt.matshow(cm, cmap=plt.cm.autumn)
plt.colorbar(img, fraction=0.045)
for x in range(cm.shape[0]):
    for y in range(cm.shape[1]):
        plt.text(x,y, "%0.2f" %cm[x,y], size=12, color='black', ha='center', va='center')
plt.show()
roc_auc_score(y_test,predictions)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1464490782713811, 'fraction_numerical': 0.03161075853732245, 'mean_word_length': 4.797477224947442, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '358545', 'license': 'MIT', 'n_tokens_mistral': 6540, 'n_tokens_neox': 6021, 'n_words': 956, 'path': 'Analysis_data.ipynb', 'repo_name': 'KWeselski/Starbucks_Capstone', 'size': '16545'}