<jupyter_start><jupyter_text>Many columns have missing values. Below I use different imputation methods to replacethe missing values<jupyter_code>#totalvisits has 137 missing values
#use fillna() method to replace missing values with mean value
df['totalvisits'].fillna(df['totalvisits'].mean(), inplace = True)
df['page_views_per_visit'].fillna(df['page_views_per_visit'].mean(), inplace = True)
df['asymmetrique_activity_score'].fillna(df['asymmetrique_activity_score'].mean(), inplace = True)
df['asymmetrique_profile_score'].fillna(df['asymmetrique_profile_score'].mean(), inplace = True)

#get one list with categorical and one list with numerical features
#'prospect_id' is not included in the model building
categorical = [#'prospect_id',
 'lead_origin',
 'lead_source',
 'do_not_email',
 'do_not_call',
 'last_activity',
 'country',
 'specialization',
 'how_did_you_hear_about_x_education',
 'what_is_your_current_occupation',
 'what_matters_most_to_you_in_choosing_a_course',
 'search',
 'magazine',
 'newspaper_article',
 'x_education_forums',
 'newspaper',
 'digital_advertisement',
 'through_recommendations',
 'receive_more_updates_about_our_courses',
 'tags',
 'lead_quality',
 'update_me_on_supply_chain_content',
 'get_updates_on_dm_content',
 'lead_profile',
 'city',
 'asymmetrique_activity_index',
 'asymmetrique_profile_index',
 'i_agree_to_pay_the_amount_through_cheque',
 'a_free_copy_of_mastering_the_interview',
 'last_notable_activity']
numerical = ['lead_number',
 'totalvisits',
 'total_time_spent_on_website',
 'page_views_per_visit',
 'asymmetrique_activity_score',
 'asymmetrique_profile_score']
#check if any of the numerical variables still have NaN values
n = 0
for n in numerical:
    print(df[n].isnull().sum())
    n =+ 1
#for 'lead_source', use impuation method 'most common class'
df['lead_source'].fillna(df['lead_source'].value_counts().index[0], inplace=True)
#for 'last_activity', use impution method 'most common class'
df['last_activity'].fillna(df['last_activity'].value_counts().index[0], inplace=True)
#for 'country', we use imputation method 'unknown' class
df['country'].fillna("unknown", inplace=True)
df['specialization'].fillna("unknown", inplace=True)
df['how_did_you_hear_about_x_education'].fillna("unknown", inplace=True)
df['what_is_your_current_occupation'].fillna("unknown", inplace=True)
df['what_matters_most_to_you_in_choosing_a_course'].fillna("unknown", inplace=True)
df['lead_quality'].fillna("unknown", inplace=True)
df['tags'].fillna("unknown", inplace=True)
df['lead_profile'].fillna("unknown", inplace=True)
df['city'].fillna("unknown", inplace=True)
df['asymmetrique_activity_index'].fillna("unknown", inplace=True)
df['asymmetrique_profile_index'].fillna("unknown", inplace=True)
#checking if there are still any NaN values
df.isnull().sum()
#splitting the dataset in different subsets
from sklearn.model_selection import train_test_split
#shuffling the data of df and splitting it into 2 sets
#df_train_full (80%), df_test(20%)
#random_state guarantees that the data is always shuffled in the same way
df_train_full, df_test = train_test_split(df, test_size=0.2, random_state=1)
#take df_train_full and split it into train and val
df_train, df_val = train_test_split(df_train_full, test_size=0.33, random_state=11)
#save target value in matrix array
y_train = df_train.converted.values
y_val = df_val.converted.values
#delete target variable from training and validation set
del df_train['converted']
del df_val['converted']
#check distribution of target variable
#two thirds did not convert
df_train_full.converted.value_counts()
#get conversion rate
#it seems to be an inbalanced dataset
global_mean = df_train_full.converted.mean()
round(global_mean, 5)<jupyter_output><empty_output><jupyter_text>Mutual Information (MI)* tells us how much info we learn about one variable if we know the value of other variables* we use it to measure mutual dependency between 2 variables* higher mutual info means higher dependency* MI only works for categorical but not for numerical variables<jupyter_code>#get mutual info of all categorical variables
#... most useful, ... least useful variable
from sklearn.metrics import mutual_info_score

#this function calculates mutual information
def calculate_mi(series):
    return mutual_info_score(series, df_train_full.converted)

df_mi = df_train_full[categorical].apply(calculate_mi)
df_mi = df_mi.sort_values(ascending=False).to_frame(name='MI')
df_mi
#calculate correlation coefficient between numerical features and target variable
df_train_full[numerical].corrwith(df_train_full.converted)
def prepare_X(df_inp):
    #convert training set to dictionary
    train_dict = df_inp[categorical + numerical].to_dict(orient='rows')
    from sklearn.feature_extraction import DictVectorizer
    dv = DictVectorizer(sparse=False)
    dv.fit(train_dict)
    #use 'transform' method to convert dictionaries to matrix
    X = dv.transform(train_dict)
    return X
#train logistic regression model
from sklearn.linear_model import LogisticRegression
#train model by calling 'fit' method
model = LogisticRegression(solver='liblinear', random_state = 1)
model.fit(prepare_X(df_train), y_train)
def prepare_val(df_inp):
    #convert training set to dictionary
    train_dict = df_inp[categorical + numerical].to_dict(orient='rows')
    from sklearn.feature_extraction import DictVectorizer
    dv_new = DictVectorizer(sparse=False)
    dv_new.fit(train_dict)
    #use 'transform' method to convert dictionaries to matrix
    X = dv_new.transform(train_dict)
    return X
#use model to predict target variable
y_pred = model.predict_proba(prepare_val(df_val))[:, 1]
y_pred >= 0.5
converted = y_pred >= 0.5
print('Accuracy', (y_val == converted).mean())<jupyter_output><empty_output><jupyter_text>The above cell throws an error. Reason: the number of features in the training set and the validation set is different, after the one-hot encoding was conducted. The outputs of the cells below also give that result (205 != 188)<jupyter_code>train_dict = df_train[categorical + numerical].to_dict(orient='rows')
from sklearn.feature_extraction import DictVectorizer
dv = DictVectorizer(sparse=False)
dv.fit(train_dict)
#use 'transform' method to convert dictionaries to matrix
X = dv.transform(train_dict)
X.shape
#get a list with all the feature names (column names) from training set after the one-hot-encoding
list_train = dv.get_feature_names()
train_dict = df_val[categorical + numerical].to_dict(orient='rows')
from sklearn.feature_extraction import DictVectorizer
dv_new = DictVectorizer(sparse=False)
dv_new.fit(train_dict)
#use 'transform' method to convert dictionaries to matrix
X = dv_new.transform(train_dict)
X.shape
#get a list with all the feature names (column names) from validation set after the one-hot-encoding
list_val = dv_new.get_feature_names()
#get those items, that only exist in one of the lists
diff_list = list(set(list_train) - set(list_val))
#calculate the difference of the length between the 2 lists
len_diff_list = len(list_train) - len(list_val)
len_diff_list
#below is a example that helps to understand how to add columns to a df, where the values are 0
data = {'Name': ['Jai', 'Princi', 'Gaurav', 'Anuj'], 
        'Height': [5.1, 6.2, 5.1, 5.2], 
        'Qualification': ['Msc', 'MA', 'Msc', 'Msc']} 
df = pd.DataFrame(data)
df
for n in diff_list:
    df[n] = 0
df
def prepare_X(df_inp):
    train_dummies = pd.get_dummies(df_inp[categorical + numerical])
    #use 'values' method to convert df to matrix
    X = train_dummies.values
    return X
#train logistic regression model
from sklearn.linear_model import LogisticRegression
#train model by calling 'fit' method
model = LogisticRegression(solver='liblinear', random_state = 1)
model.fit(prepare_X(df_train), y_train)
def prepare_val(df_inp):
    train_dummies = pd.get_dummies(df_inp[categorical + numerical])
    #add missing features by adding columns, where the values are 0
    for n in range(len_diff_list):
        train_dummies[n] = 0
    #use 'values' method to convert df to matrix
    X = train_dummies.values
    return X
#use model to predict target variable
y_pred = model.predict_proba(prepare_val(df_val))[:, 1]
y_pred >= 0.5
converted = y_pred >= 0.5
print('Accuracy', (y_val == converted).mean())<jupyter_output>Accuracy 0.7245901639344262<jupyter_text>Useful links that help to understand how to deal with datasets of different sizes* https://stackoverflow.com/questions/44266677/machine-learning-test-set-with-fewer-features-than-the-train-set* https://stackoverflow.com/questions/45422000/add-n-zeros-to-the-end-of-an-array<jupyter_code>from sklearn.metrics import roc_curve

fpr, tpr, thresholds = roc_curve(y_val, y_pred)

plt.figure(figsize=(5, 5))
plt.plot(fpr, tpr)
plt.plot([0,1],[0,1])
plt.xlabel('False Positive Rate', color='white')
plt.ylabel('True Positive Rate', color='white')
plt.title('ROC Curve', color='white')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10783985725437716, 'fraction_numerical': 0.010259841641574662, 'mean_word_length': 6.664957264957265, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 20, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '428632', 'license': 'MIT', 'n_tokens_mistral': 3027, 'n_tokens_neox': 2901, 'n_words': 777, 'path': '200830-leads.ipynb', 'repo_name': 'MartinTschendel/leads', 'size': '8967'}