<jupyter_start><jupyter_text>Missing Values<jupyter_code># NaN Counting
nan_count = donors.isnull().sum().sort_values(ascending = False)
nan_count = nan_count[nan_count != 0]
nan_count/len(donors)
#Drop RFM columns from nan_counts
RDATE = nan_count[nan_count.index.str.contains("RDATE_")].index
RAMNT = nan_count[nan_count.index.str.contains("RAMNT_")].index
ADATE = nan_count[nan_count.index.str.contains("ADATE_")].index
nan_count = nan_count.drop(labels = RDATE)
nan_count = nan_count.drop(labels = RAMNT)
nan_count = nan_count.drop(labels = ADATE)
nan_count/len(donors)
#Delete Columns with more than 50% of Nan
columns_to_delete = ['NUMCHLD','MBCOLECT','PUBDOITY', 'PUBGARDN', 'PUBHLTH', 'MAGMALE','PUBNEWFN', 
                     'PUBPHOTO', 'PUBOPP', 'MAGFEM', 'MAGFAML', 'MBBOOKS','MBGARDEN', 
                     'MBCRAFT', 'PUBCULIN']
donors = donors.drop(columns = columns_to_delete)
nan_count = nan_count.drop(labels = columns_to_delete)
nan_count/len(donors)<jupyter_output><empty_output><jupyter_text>Feature Eng.<jupyter_code>#Last promotion date
lastpromotiondate = datetime.strptime('2017-06-01', '%Y-%m-%d')
lastpromotiondate.year
#Create Variable Age
donors['DOB'] = donors['DOB'].map(lambda x: datetime.strptime(x, '%Y-%m-%d') if pd.notnull(x) else x)
donors['AGE'] = lastpromotiondate.year - donors['DOB'].dt.year
#Create Function to count how many days to date.
def howmanydays(df, column, recent_year):
    df[column] = df[column].fillna('2017-06-01')
    df[column] = df[column].astype(str)
    df[column] = df[column].map(lambda x: datetime.strptime(x, '%Y-%m-%d') if pd.notnull(x) else x)
    df[column+'_days'] = (recent_year.
                       year - df[column].
                       dt.year)*12 + (recent_year.
                                      month - df[column].
                                      dt.month)
#Create Variable date_days for the date features.
for column in ['MAXADATE','MINRDATE','MAXRDATE','FISTDATE','NEXTDATE']:
    howmanydays(donors, column, lastpromotiondate)
#Split Variable DOMAIN wealth and type of neighbor.
donors['DOMAIN_LVL'] = donors['DOMAIN'].str.split('', expand = True)[1]
donors['DOMAIN_ECO'] = donors['DOMAIN'].str.split('', expand = True)[2]
#Create Variables for RFM_Promotion

rdate = donors[['RDATE_3','RDATE_4','RDATE_5','RDATE_6','RDATE_7','RDATE_8','RDATE_9','RDATE_10','RDATE_11','RDATE_12','RDATE_13','RDATE_14','RDATE_15','RDATE_16','RDATE_17','RDATE_18','RDATE_19','RDATE_20','RDATE_21','RDATE_22','RDATE_23','RDATE_24']]
rdate.columns = ['16NK','16TK','16SK','16LL','16G1','16GK','16CC','16WL','16X1','16XK','15FS','15NK','15TK','15LL','15G1','15GK','15CC','15WL','15X1','15XK','14FS','14NK']
rdate = rdate #  Date the gift was received for 16NK

ramnt = donors[['RAMNT_3','RAMNT_4','RAMNT_5','RAMNT_6','RAMNT_7','RAMNT_8','RAMNT_9','RAMNT_10','RAMNT_11','RAMNT_12','RAMNT_13','RAMNT_14','RAMNT_15','RAMNT_16','RAMNT_17','RAMNT_18','RAMNT_19','RAMNT_20','RAMNT_21','RAMNT_22','RAMNT_23','RAMNT_24']]
ramnt.columns = ['16NK','16TK','16SK','16LL','16G1','16GK','16CC','16WL','16X1','16XK','15FS','15NK','15TK','15LL','15G1','15GK','15CC','15WL','15X1','15XK','14FS','14NK']
ramnt = ramnt  #Dollar amount of the gift for 16NK
#Take the last date that the donor sent a gift with the first non null value from rows.
lastgiftdate = []
for i in range(0,len(rdate)):
    if rdate.iloc[i].isnull().sum() == 22:
        lastgiftdate.append(0)
    else:
        lastgiftdate.append(rdate.iloc[i][rdate.iloc[i].notnull()][0])
#Add Columns
rdate['lastgiftdate'] = lastgiftdate
#Transforming to Datetime
rdate.loc[:,'lastgiftdate'] = rdate.loc[:,'lastgiftdate'].map(lambda x: datetime.strptime(x, '%Y-%m-%d')
                                                              if not isinstance(x, int) else 0)
#Recency: Last promotion date in months
lastpromotiondate = datetime.strptime('2017-06-01', '%Y-%m-%d')
Recency = []
for i in rdate['lastgiftdate']:
    if i == 0:
        Recency.append(None)
    else:
        Recency.append((lastpromotiondate.year - i.year)*12 + (lastpromotiondate.month - i.month))
# Monetary sum of all gifts from donors/rows
Monetary = ramnt.sum(axis=1)
#Frequency count of how many times the donor gifted.
Frequency = ramnt.count(axis=1)
#Create Columns from RFM
donors['Recency'] = Recency
donors['Frequency'] = Frequency
donors['Monetary'] = Monetary<jupyter_output><empty_output><jupyter_text>Delete rows/donors that didn't gifted with the promotion.<jupyter_code>delete_non_donors = donors[donors['Recency'].isnull()].index
donors.drop(index = delete_non_donors, inplace = True)
#Create score Classification for RFM_Promotion
donors['Recency_score'] = pd.qcut(donors['Recency'].rank(method='first'), q=5 , labels= (5,4,3,2,1))
donors['Recency_score'] = donors['Recency_score'].astype(int)
donors['Frequency_score'] = pd.qcut(donors['Frequency'].rank(method='first'), q=5 , labels= range(1,6))
donors['Frequency_score'] = donors['Frequency_score'].astype(int)
donors['Monetary_score'] = pd.qcut(donors['Monetary'].rank(method='first'), q=5 , labels= range(1,6))
donors['Monetary_score'] = donors['Monetary_score'].astype(int)
#Create RFM from Lifetime Donors

donors['Frequency_LTD'] = donors['NGIFTALL']
donors['Monetary_LTD'] = donors['AVGGIFT']
#Transform LASTDATE to create Recency
donors['LASTDATE'] = donors['LASTDATE'].map(lambda x: datetime.strptime(x, '%Y-%m-%d'))
#Create Recency from life time donors
lastpromotiondate = datetime.strptime('2017-06-01', '%Y-%m-%d')
lastpromotiondate.year
#Date associated with the most recent gift
donors['Recency_LTD'] = (lastpromotiondate.year - donors['LASTDATE']
                     .dt.year)*12 + (lastpromotiondate.month - donors['LASTDATE']
                                                                              .dt.month)
#Create star Classification for RFM
donors['Recency_LTD_score'] = pd.qcut(donors['Recency_LTD'].rank(method='first'), q=5 , labels= (5,4,3,2,1))
donors['Recency_LTD_score'] = donors['Recency_LTD_score'].astype(int)
donors['Frequency_LTD_score'] = pd.qcut(donors['Frequency_LTD'].rank(method='first'), q=5 , labels= range(1,6))
donors['Frequency_LTD_score'] = donors['Frequency_LTD_score'].astype(int)
donors['Monetary_LTD_score'] = pd.qcut(donors['Monetary_LTD'].rank(method='first'), q=5 , labels= range(1,6))
donors['Monetary_LTD_score'] = donors['Monetary_LTD_score'].astype(int)
#Create Variables sum and count for presents
def sum_by_column(df, new_col_name, list_of_cols_to_sum):
    df[new_col_name] = df[list_of_cols_to_sum].astype(float).sum(axis=1)
    return(df)

def count_by_column(df, new_col_name, list_of_cols_to_sum):
    df[new_col_name] = df[list_of_cols_to_sum].astype(float).count(axis=1)
    return(df)
GIFT = ramnt.copy()

# Aggregating by types and years
sum_by_column(GIFT, "16-labels", ['16LL','16WL'])
sum_by_column(GIFT, "15-labels", ['15LL','15WL'])
sum_by_column(GIFT, "16-calendars_with_stickers", ['16CC'])
sum_by_column(GIFT, "15-calendars_with_stickers", ['15CC'])
sum_by_column(GIFT, "15-blank_folding_cards_with_labels", ['15FS'])
sum_by_column(GIFT, "14-blank_folding_cards_with_labels", ['14FS'])
sum_by_column(GIFT, "16-blank_cards_with_labels", ['16NK','16SK'])
sum_by_column(GIFT, "15-blank_cards_with_labels", ['15NK'])
sum_by_column(GIFT, "14-blank_cards_with_labels", ['14NK'])
sum_by_column(GIFT, "16-thank_you_printed_with_labels", ['16TK'])
sum_by_column(GIFT, "15-thank_you_printed_with_labels", ['15TK'])
sum_by_column(GIFT, "16-general_greeting_cards_with_labels", ['16GK'])
sum_by_column(GIFT, "15-general_greeting_cards_with_labels", ['15GK'])
sum_by_column(GIFT, "16-christmas_cards_with_labels", ['16XK'])
sum_by_column(GIFT, "15-christmas_cards_with_labels", ['15XK'])
sum_by_column(GIFT, "16-notepad_with_labels", ['16X1','16G1'])
sum_by_column(GIFT, "15-notepad_with_labels", ['15X1','15G1'])

# Now summing by type of gift
sum_by_column(GIFT, "Just labels", ['16-labels','15-labels'])
sum_by_column(GIFT, "Calendars with stickers", ['16-calendars_with_stickers','15-calendars_with_stickers'])
sum_by_column(GIFT, "Blank folding cards with labels", ['15-blank_folding_cards_with_labels','14-blank_folding_cards_with_labels'])
sum_by_column(GIFT, "Blank cards with labels", ['16-blank_cards_with_labels','15-blank_cards_with_labels', '14-blank_cards_with_labels'])
sum_by_column(GIFT, "Thank you cards with labels", ['16-thank_you_printed_with_labels', '15-thank_you_printed_with_labels','16-general_greeting_cards_with_labels','15-general_greeting_cards_with_labels'])
sum_by_column(GIFT, "Christmas Cards with labels", ['16-christmas_cards_with_labels', '15-christmas_cards_with_labels'])
sum_by_column(GIFT, "General Greeting Cards with Labels", ['16-general_greeting_cards_with_labels','15-general_greeting_cards_with_labels'])
sum_by_column(GIFT, "Notepad with labels", ['16-notepad_with_labels', '15-notepad_with_labels'])

GIFT_sum = GIFT.iloc[:,-8:]
GIFT = ramnt.copy()

# Aggregating by types and years
count_by_column(GIFT, "16-labels", ['16LL','16WL'])
count_by_column(GIFT, "15-labels", ['15LL','15WL'])
count_by_column(GIFT, "16-calendars_with_stickers", ['16CC'])
count_by_column(GIFT, "15-calendars_with_stickers", ['15CC'])
count_by_column(GIFT, "15-blank_folding_cards_with_labels", ['15FS'])
count_by_column(GIFT, "14-blank_folding_cards_with_labels", ['14FS'])
count_by_column(GIFT, "16-blank_cards_with_labels", ['16NK','16SK'])
count_by_column(GIFT, "15-blank_cards_with_labels", ['15NK'])
count_by_column(GIFT, "14-blank_cards_with_labels", ['14NK'])
count_by_column(GIFT, "16-thank_you_printed_with_labels", ['16TK'])
count_by_column(GIFT, "15-thank_you_printed_with_labels", ['15TK'])
count_by_column(GIFT, "16-general_greeting_cards_with_labels", ['16GK'])
count_by_column(GIFT, "15-general_greeting_cards_with_labels", ['15GK'])
count_by_column(GIFT, "16-christmas_cards_with_labels", ['16XK'])
count_by_column(GIFT, "15-christmas_cards_with_labels", ['15XK'])
count_by_column(GIFT, "16-notepad_with_labels", ['16X1','16G1'])
count_by_column(GIFT, "15-notepad_with_labels", ['15X1','15G1'])

# Now summing by type of gift
sum_by_column(GIFT, "Just labels_count", ['16-labels','15-labels'])
sum_by_column(GIFT, "Calendars with stickers_count", ['16-calendars_with_stickers','15-calendars_with_stickers'])
sum_by_column(GIFT, "Blank folding cards with labels_count", ['15-blank_folding_cards_with_labels','14-blank_folding_cards_with_labels'])
sum_by_column(GIFT, "Blank cards with labels_count", ['16-blank_cards_with_labels','15-blank_cards_with_labels', '14-blank_cards_with_labels'])
sum_by_column(GIFT, "Thank you cards with labels_count", ['16-thank_you_printed_with_labels', '15-thank_you_printed_with_labels','16-general_greeting_cards_with_labels','15-general_greeting_cards_with_labels'])
sum_by_column(GIFT, "Christmas Cards with labels_count", ['16-christmas_cards_with_labels', '15-christmas_cards_with_labels'])
sum_by_column(GIFT, "General Greeting Cards with Labels_count", ['16-general_greeting_cards_with_labels','15-general_greeting_cards_with_labels'])
sum_by_column(GIFT, "Notepad with labelss_count", ['16-notepad_with_labels', '15-notepad_with_labels'])

GIFT_count = GIFT.iloc[:,-8:]
donors = pd.concat([donors, GIFT_sum, GIFT_count], axis = 1)
#Delete non donors Ids after gift table.
donors.drop(index = delete_non_donors, inplace = True)
#Fill Nan From GEOCODE2
donors['GEOCODE2'].fillna(' ', inplace = True)<jupyter_output><empty_output><jupyter_text>Feature Selection  Separate the variables Financial information- INCOME                      HOUSEHOLD INCOME -> **KNN IMPUTER**- WEALTH1                     Wealth Rating -> **KNN IMPUTER**    - WEALTH2                     Wealth Rating -> **KNN IMPUTER** Geographic information- STATE                       State abbreviation (a nominal/symbolic field) -> **DELETE**- ZIP                         Zipcode (a nominal/symbolic field) -> **DELETE**- MAILCODE                    Mail Code ->- PVASTATE                    EPVA State or PVA State -> - DOMAIN                      DOMAIN/Cluster code. A nominal or symbolic field. -> **'DISCRETIZING'**- GEOCODE                     Geo Cluster Code indicating the level geography at which ->  Donation History- OSOURCE                     Origin Source -> **DELETE**- MDMAUD                      The Major Donor Matrix code -> **DELETE**- MAJOR                       Major Donor Flag ->- PEPSTRFL                    Indicates PEP Star RFA Status -> - HIT                         MOR Flag HIT (Mail Order Response) ->  ID information- ODATEDW                     Origin Date. Date of donors first gift -> **'DELETE'**- TCODE                       Donor title code -> **'DELETE'**- DOB                         Date of birth (YYMM, Year/Month format.) -> **DELETE**- CHILD03                     Presence of Children age 0-3 -> - CHILD07                     Presence of Childern age 4-7 ->- CHILD12                     Presence of Childern age 8-12 ->- CHILD18                     Presence of Childern age 13-18 -> - NUMCHLD                     NUMBER OF CHILDREN -> **DELETE**- GENDER                      Gender -> - DATASRCE                    Source of Overlay Data -> **'DELETE'**- MALEMILI                    % Males active in the Military -> - MALEVET                     % Males Veterans -> - VIETVETS                    % Vietnam Vets -> - WWIIVETS                    % WWII Vets -> - LOCALGOV                    % Employed by Local Gov -> - STATEGOV                    % Employed by State Gov ->- FEDGOV                      % Employed by Fed Gov -> - COLLECT1                    COLLECTABLE (Y/N) ->- VETERANS                    VETERANS (Y/N) -> - BIBLE                       BIBLE READING (Y/N) -> - CATLG                       SHOP BY CATALOG (Y/N) ->- HOMEE                       WORK FROM HOME (Y/N) -> - PETS                        HOUSEHOLD PETS (Y/N) -> - CDPLAY                      CD PLAYER OWNERS (Y/N) -> - STEREO                      STEREO/RECORDS/TAPES/CD (Y/N) -> - PCOWNERS                    HOME PC OWNERS/USERS ->- PHOTO                       PHOTOGRAPHY (Y/N) -> - CRAFTS                      CRAFTS (Y/N) -> - FISHER                      FISHING (Y/N) ->- GARDENIN                    GARDENING (Y/N) -> - BOATS                       POWER BOATING (Y/N) ->- WALKER                      WALK FOR HEALTH (Y/N) -> - KIDSTUFF                    BUYS CHILDRENS PRODUCTS (Y/N) -> - CARDS                       STATIONARY/CARDS BUYER (Y/N) -> - PLATES                      PLATE COLLECTOR (Y/N) -> - LIFESRC                     LIFE STYLE DATA SOURCE/ SAME AS DATASRCE -> **'DELETE'**- HOMEOWNR                    Home Owner Flag ->  Data permission for ID information- NOEXCH                      Do Not Exchange Flag (For list rental) -> - RECINHSE                    In House File Flag -> - RECP3                       P3 File Flag -> - RECPGVG                     Planned Giving File Flag ->- RECSWEEP                    Sweepstakes file fla g ->  Unknown - SOLP3                       SOLICIT LIMITATION CODE P3 ->- SOLIH                       SOLICITATION LIMIT CODE IN HOUSE ->  N. information everything is ok. RFM information- ADATE_K **DELETE**- RFA_K **DELETE**- RDATE_K **DELETE**- RAMNT_K **DELETE**- FISTDATE **DELETE**- NEXTDATE **DELETE**<jupyter_code># Delete RFM information
RFA_columns_delete = donors.columns[donors.columns.str.contains('RFA_')]
RAMNT_columns_delete = donors.columns[donors.columns.str.contains('RAMNT_')]
RDATE_columns_delete = donors.columns[donors.columns.str.contains('RDATE_')]
ADATA_columns_delete = donors.columns[donors.columns.str.contains('ADATE_')]
MDMAUD_columns_delete = donors.columns[donors.columns.str.contains('MDMAUD_')]

donors = donors.drop(columns = RFA_columns_delete, errors = 'ignore')
donors = donors.drop(columns = RAMNT_columns_delete, errors = 'ignore')
donors = donors.drop(columns = RDATE_columns_delete, errors = 'ignore')
donors = donors.drop(columns = ADATA_columns_delete, errors = 'ignore')
donors = donors.drop(columns = MDMAUD_columns_delete, errors = 'ignore')
donors = donors.drop(columns = ['FISTDATE','NEXTDATE','MAXADATE','MINRDATE','MAXRDATE','FISTDATE','NEXTDATE','LASTDATE'], 
                     errors = 'ignore')
# Delete Geographic information
Geo_delete_columns = ['STATE', 'ZIP']
donors = donors.drop(columns = Geo_delete_columns, errors = 'ignore')
# Delete Donation History
Don_delete_columns = ['OSOURCE', 'MDMAUD']
donors = donors.drop(columns = Don_delete_columns, errors = 'ignore')
# Delete ID Information
ID_delete_columns = ['TCODE','ODATEDW', 'DOB', 'NUMCHLD', 'DATASRCE', 'LIFESRC']
donors = donors.drop(columns = ID_delete_columns, errors = 'ignore')
# NaN Counting
nan_count = donors.isnull().sum().sort_values(ascending = False)
nan_count = nan_count[nan_count != 0]
nan_count/len(donors)<jupyter_output><empty_output><jupyter_text>OHE<jupyter_code>nonmetric_features = donors.dtypes[(donors.dtypes == 'object')].index.to_list()
len(nonmetric_features)
nonmetric_features
train_data_standard_ohe = donors.copy()
ohc = OneHotEncoder(sparse=False, drop = 'if_binary')
ohc_feat = ohc.fit_transform(train_data_standard_ohe[nonmetric_features])
ohc_feat_names = ohc.get_feature_names()
ohc_df = pd.DataFrame(ohc_feat, index=train_data_standard_ohe[nonmetric_features].index, columns=ohc_feat_names)  # Why the index=df_ohc.index?
ohc_df

metric_features = donors.columns.to_list()
for i in nonmetric_features:
    metric_features.remove(i)
donors = pd.concat([donors[metric_features],
                            ohc_df],
                           axis = 1)
donors.head()<jupyter_output><empty_output><jupyter_text>Detecting OutLiers<jupyter_code>from sklearn.neighbors import LocalOutlierFactor
nan_columns = donors.isnull().sum()[(donors.isnull().sum() > 0)].index.to_list()
non_nan = donors.columns.to_list()
for i in nan_columns:
    non_nan.remove(i)
# model specification
model1 = LocalOutlierFactor(n_neighbors = 8)
# model fitting
y_pred = model1.fit_predict(donors[non_nan])
# filter outlier index
outlier_index = (y_pred == -1) # negative values are outliers and positives inliers
# filter outlier values
outlier_values = donors[non_nan].iloc[outlier_index]
#Delete Outliers to fill NaN
donors.drop(index = outlier_values.index, inplace = True)<jupyter_output><empty_output><jupyter_text>Fill Na Values<jupyter_code># NaN Counting
nan_count = donors.isnull().sum().sort_values(ascending = False)
nan_count = nan_count[nan_count != 0]
nan_count/len(donors)
#Check correlation
income_cor = donors.corr()
age_fill = income_cor['AGE'][(income_cor['AGE'].
                              sort_values()>0.15) | (income_cor['AGE'].
                                                     sort_values()<-0.15)].index.to_list()
income_fill = income_cor['INCOME'][(income_cor['INCOME'].
                                    sort_values()>0.3) | (income_cor['INCOME'].
                                                          sort_values()<-0.3)].index.to_list()
income_cor['INCOME'][(income_cor['INCOME'].
                                    sort_values()>0.3) | (income_cor['INCOME'].
                                                          sort_values()<-0.3)]
imputer = KNNImputer(n_neighbors=8, weights="uniform")
fill_na_income = imputer.fit_transform(donors[income_fill])
imputer = KNNImputer(n_neighbors=8, weights="uniform")
fill_na_age = imputer.fit_transform(donors[age_fill])
donors[income_fill] = fill_na_income
donors[age_fill] = fill_na_age
donors['INCOME'] = round(donors['INCOME'],0)
donors['WEALTH1'] = round(donors['WEALTH1'],0)
donors['WEALTH2'] = round(donors['WEALTH2'],0)
donors['AGE'] = round(donors['AGE'],0)
# NaN Counting
nan_count = donors.isnull().sum().sort_values(ascending = False)
nan_count = nan_count[nan_count != 0]
nan_count/len(donors)
DMA_na = donors[donors['DMA'].isnull()].index.to_list()
drop_na = DMA_na
donors.drop(index = drop_na, inplace = True)
nan_count = donors.isnull().sum().sort_values(ascending = False)
nan_count = nan_count[nan_count != 0]
nan_count/len(donors)
#TIMELAG is a real NaN
donors['TIMELAG'].fillna(0, inplace = True)<jupyter_output><empty_output><jupyter_text>Outliars check<jupyter_code># model specification
model1 = LocalOutlierFactor(n_neighbors = 8)
# model fitting
y_pred = model1.fit_predict(donors)
# filter outlier index
outlier_index = (y_pred == -1) # negative values are outliers and positives inliers
# filter outlier values
outlier_values = donors.iloc[outlier_index]
#Delete Outliers to fill NaN
donors.drop(index = outlier_values.index, inplace = True)
len(donors)<jupyter_output><empty_output><jupyter_text>Feature Selection<jupyter_code>donors_corr = donors.corr().loc[:,['Recency_score','Frequency_score', 'Monetary_score']]
donors_corr['Total'] = donors_corr.loc[:,['Recency_score','Frequency_score', 'Monetary_score']].pow(2).sum(axis = 1)
high_corr = donors_corr.sort_values(by = 'Total').tail(40).index.to_list()
high_corr<jupyter_output><empty_output><jupyter_text>Scale Data<jupyter_code>deleted_features = ['Recency_LTD', 'Frequency', 'Recency', 'NEXTDATE_days', 'Frequency_LTD', 'Monetary', 'Monetary_LTD', 'FISTDATE_days']
for i in deleted_features:
    metric_features.remove(i)
donors_scaled = donors.copy()
# Use MinMaxScaler to scale the data
scaler = StandardScaler()
scaled_feat = scaler.fit_transform(donors_scaled[metric_features])
scaled_feat
donors_scaled[metric_features] = scaled_feat
donors_scaled.head()<jupyter_output><empty_output><jupyter_text>PCA<jupyter_code># NaN Counting
nan_count = donors_scaled.isnull().sum().sort_values(ascending = False)
nan_count = nan_count[nan_count != 0]
nan_count/len(donors_scaled)
# Use PCA to reduce dimensionality of data
pca = PCA()
pca_feat = pca.fit_transform(donors_scaled[metric_features])
pca_feat  # What is this output?
# Output PCA table
pd.DataFrame(
    {"Eigenvalue": pca.explained_variance_,
     "Difference": np.insert(np.diff(pca.explained_variance_), 0, 0),
     "Proportion": pca.explained_variance_ratio_,
     "Cumulative": np.cumsum(pca.explained_variance_ratio_)},
    index=range(1, pca.n_components_ + 1)
).head(20)
# figure and axes
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5))

# draw plots
ax1.plot(pca.explained_variance_, marker=".", markersize=12)
ax2.plot(pca.explained_variance_ratio_, marker=".", markersize=12, label="Proportion")
ax2.plot(np.cumsum(pca.explained_variance_ratio_), marker=".", markersize=12, linestyle="--", label="Cumulative")

# customizations
ax2.legend()
ax1.set_title("Scree Plot", fontsize=14)
ax2.set_title("Variance Explained", fontsize=14)
ax1.set_ylabel("Eigenvalue")
ax2.set_ylabel("Proportion")
ax1.set_xlabel("Components")
ax2.set_xlabel("Components")
ax1.set_xticks(range(0, pca.n_components_, 2))
ax1.set_xticklabels(range(1, pca.n_components_ + 1, 2))
ax2.set_xticks(range(0, pca.n_components_, 2))
ax2.set_xticklabels(range(1, pca.n_components_ + 1, 2))

plt.show()
donors_scaled
# Perform PCA again with the number of principal components you want to retain
pca = PCA(n_components=10)
pca_feat = pca.fit_transform(donors_scaled[metric_features])
pca_feat_names = [f"PC{i}" for i in range(pca.n_components_)]
df_pca = donors_scaled.copy()
pca_df = pd.DataFrame(pca_feat, index=df_pca.index, columns=pca_feat_names)  # remember index=df_pca.index
pca_df
# Reassigning df to contain pca variables
donors_scaled = pd.concat([donors_scaled, pca_df], axis=1)
donors_scaled.head()
donors_scaled.to_csv(os.path.join("donors_scaled.csv"), index=True)<jupyter_output><empty_output><jupyter_text>Clusters Analysis<jupyter_code># Splitting feature names into groups
non_metric_features = donors_scaled.columns[donors_scaled.columns.str.startswith('x')]
pc_features = donors_scaled.columns[donors_scaled.columns.str.startswith('PC')]
metric_features = donors_scaled.columns[~donors_scaled.columns.str.startswith('x') & ~donors_scaled.columns.str.startswith('PC')]<jupyter_output><empty_output><jupyter_text>RFM Cluster<jupyter_code>kmeans = KMeans(init='k-means++', n_clusters=8, n_init=15, max_iter=300, random_state=1)
donors_rfm = donors.loc[:, ['Recency_score','Frequency_score', 'Monetary_score']]
kmeans.fit(donors_rfm)
donors_rfm['Cluster_RFM'] = kmeans.labels_
donors['Cluster_RFM'] = kmeans.labels_
donors_scaled['Cluster_RFM'] = kmeans.labels_
X = donors_rfm.loc[:, ['Recency_score','Frequency_score', 'Monetary_score']]
kelbow_visualizer(KMeans(init='k-means++', n_init=15, max_iter=300, random_state=1), X, k=(3,12),
                 timing = False)
X = donors_rfm.loc[:, ['Recency_score','Frequency_score', 'Monetary_score']]
model = KMeans(init='k-means++', n_init=15, max_iter=300, random_state=1)
visualizer = KElbowVisualizer(
    model, k=(4,12), metric='silhouette', timing = False
)

visualizer.fit(X)        # Fit the data to the visualizer
visualizer.show()        # Finalize and render the figure
RFM_table = donors_rfm.groupby(['Cluster_RFM']).agg(['mean'])
RFM_table['Total'] = RFM_table.loc[:,['Recency_score','Frequency_score', 'Monetary_score']].sum(axis = 1)
RFM_table = round(RFM_table.sort_values(by = 'Total',ascending = False),2)
RFM_table
bins = [1,7,6,5,4,2,0,3]
labels = ['Best Customers', 'Almost lost Customers', 'Loyal Customers','New Customers','Almost lost cheap Customers','Lost Customers','New cheap Customers','Lost Cheap Customers']
donors_rfm['Customer_RFM'] = donors_rfm['Cluster_RFM'].map(dict(zip(bins, labels)))

RFM_table = donors_rfm.groupby(['Customer_RFM']).agg(['mean'])
RFM_table['Total'] = RFM_table.loc[:,['Recency_score','Frequency_score', 'Monetary_score']].sum(axis = 1)
RFM_table = round(RFM_table.sort_values(by = 'Total',ascending = False),2)
RFM_table
sns.heatmap(RFM_table.loc[:,['Recency_score','Frequency_score', 'Monetary_score']], annot=True, linewidths=.5)
plt.figure(figsize=(20, 5))
sns.histplot(donors_rfm, x="Customer_RFM")<jupyter_output><empty_output><jupyter_text>Behavior analysis<jupyter_code>donors['Customer_RFM'] = donors['Cluster_RFM'].map(dict(zip(bins, labels)))
promotion_features = donors[metric_features].loc[:,'Just labels': 'Notepad with labels'].columns.to_list()
promotion_features.append('Customer_RFM')
Promotion_behave = donors[promotion_features].groupby(['Customer_RFM']).mean()
round(Promotion_behave,2)
sns.heatmap(round(Promotion_behave,2), annot=True, linewidths=.5)
donors['Customer_RFM'] = donors['Cluster_RFM'].map(dict(zip(bins, labels)))
promotion_features_count = donors[metric_features].loc[:,'Just labels_count':'Notepad with labelss_count'].columns.to_list()
promotion_features_count.append('Customer_RFM')
Promotion_behave_count = donors[promotion_features_count].groupby(['Customer_RFM']).sum()
round(Promotion_behave_count,2)
sns.heatmap(round(Promotion_behave_count,2), annot=True, linewidths=.5)
age_features = donors[metric_features].loc[:,'AGE'].name
age_features.append('Customer_RFM')
age_behavior = donors[age_features].groupby(['Customer_RFM']).mean()
round(id_behavior,2)
id_features = donors[metric_features].loc[:,'INCOME':'WEALTH2'].columns.to_list()
id_features.append('Customer_RFM')
id_behavior = donors[id_features].groupby(['Customer_RFM']).mean()
round(id_behavior,2)
cat_features = donors[non_metric_features].columns.to_list()
cat_features.append('Customer_RFM')
cat_behavior = donors[cat_features].groupby(['Customer_RFM']).mean()
round(cat_behavior,2)
donors_scaled['Customer_RFM'] = donors_scaled['Cluster_RFM'].map(dict(zip(bins, labels)))
pca_features = donors_scaled[pc_features].columns.to_list()
pca_features.append('Customer_RFM')
pca_behavior = donors_scaled[pca_features].groupby(['Customer_RFM']).mean()
round(pca_behavior,2)<jupyter_output><empty_output><jupyter_text>PCA Cluster<jupyter_code>kmeans = KMeans(init='k-means++', n_clusters=8, n_init=15, max_iter=300, random_state=1)
donors_pca = donors_scaled[pc_features]
kmeans.fit(donors_pca)
donors_rfm['Cluster_PCA'] = kmeans.labels_
RFM_table = donors_rfm.groupby(['Cluster_PCA']).agg(['mean'])
RFM_table['Total'] = RFM_table.loc[:,['Recency_score','Frequency_score', 'Monetary_score']].sum(axis = 1)
RFM_table = round(RFM_table.sort_values(by = 'Total',ascending = False),2)
RFM_table<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14799943145476513, 'fraction_numerical': 0.0240921043280506, 'mean_word_length': 4.885194479297365, 'pattern_counts': {'":': 4, '<': 64, '<?xml version=': 0, '>': 123, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '190382', 'license': 'MIT', 'n_tokens_mistral': 10600, 'n_tokens_neox': 9829, 'n_words': 1671, 'path': 'Cleaning_ALL.ipynb', 'repo_name': 'm20200268/DM-PROJECT-PVA-2020', 'size': '28142'}