<jupyter_start><jupyter_text><jupyter_code># Load the data from Kaggle
!pip install kaggle

# Upgrade the version of Seaborn
!pip install -U seaborn

# Install category_encoders
!pip install category_encoders
# Mount the drive to download the data from Kaggle
from google.colab import drive
drive.mount('/content/drive')
%env KAGGLE_CONFIG_DIR=/content/drive/My Drive

!kaggle competitions download -c ds1-predictive-modeling-challenge
# Extract the csv files
!unzip train_features.csv.zip 
!unzip train_labels.csv.zip 
!unzip test_features.csv.zip
# Generic imports
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
# Loading the independent features as X and
# dependent variable as y
nan_values_list = ['Not Known', 'Unknown', 'None', 'Not known', 'not known', 
                   '-', 'unknown', 'Unknown Installer', '##', 'none']

train_features_df = pd.read_csv('train_features.csv', na_values=nan_values_list)
train_labels_df = pd.read_csv('train_labels.csv')

"""# Finding other possible NaN values 
null_cols = []

for col in train_features_df.columns:
  null_count = train_features_df[col].isnull().sum()
  null_cols.append(col)
  print(f'------------------------------------------------------------------')
  print(f'Column name {col}:')
  print(f'\tType {train_features_df[col].dtype}')
  print(f'\tNull count {null_count}')
  unique_values = train_features_df[col].unique()
  print(f'\tValue count {len(unique_values)}')
  #for value in unique_values:
    #print(f'\t\t{value}')
  print(f'------------------------------------------------------------------\n')"""

for col in train_features_df.columns:
  null_count = train_features_df[col].isnull().sum()
  unique_values = train_features_df[col].unique()
  print(f'{col},{train_features_df[col].dtype},{null_count},{len(unique_values)}')
data_cardinality_df = pd.read_csv('./data cardinality.csv')

data_cardinality_df.head()
sns.set(style="whitegrid")

# Initialize the matplotlib figure
f, ax = plt.subplots(figsize=(6, 10))


# Plot the Cardinality
data_cardinality_obj_df = data_cardinality_df[data_cardinality_df['Type'] == 'object']
sns.set_color_codes("muted")
sns.barplot(x="Cardinality", y="Feature", 
            data=data_cardinality_obj_df[data_cardinality_obj_df['Cardinality'] > 200],
            label="Cardinality", color="b")

# Add a legend and informative axis label
ax.legend(ncol=2, loc="lower right", frameon=True)
ax.set(ylabel="Categorical Features",
       xlabel="Cardinality > 200")
f.suptitle('Categorical coding of high cardinal features is an overhead', fontsize=16)
sns.despine(left=True, bottom=True)
sns.set(style="whitegrid")

# Initialize the matplotlib figure
f, ax = plt.subplots(figsize=(6, 10))


# Plot the Null
sns.set_color_codes("muted")
sns.barplot(x="Null Count", y="Feature", 
            data=data_cardinality_df[data_cardinality_df['Null Count'] > 0],
            label="NaN Count", color="b")

# Add a legend and informative axis label
ax.legend(ncol=2, loc="lower right", frameon=True)
ax.set(ylabel="Features",
       xlabel="NaN Count")
f.suptitle('Both numerical and categorical features have NaN', fontsize=16)
sns.despine(left=True, bottom=True)
data_cardinality_obj_df = data_cardinality_df[data_cardinality_df['Type'] == 'object']
data_cardinality_obj_df
# Test Code to be removed later
"""train_features_df = train_features_df.sample(1000, random_state=42)
train_labels_df = train_labels_df.sample(1000, random_state=42)
train_labels_df.status_group.value_counts()"""
train_features_df.head()
def atleast(row, value_count_series, count=5):
  # Identify items who have funded atleast 5 pumps
  if str(row) == "nan":
    return np.nan
  
  value_count = value_count_series.get(row)
  
  if value_count < count:
    return 0
  else:
    return 1

def character_grouping(row):
  # Reduce the dimension based on 1st character else return *
  if str(row) == "nan":
    return np.nan
  
  if row[0].isalpha():
    return row[0].lower()
  else:
    return "*"
  
def classify_lga(row):
  # Classify lga into Rural, Urban and others
  if str(row) == "nan":
    return np.nan
  
  if row.lower().find('rural'):
    return "rural"
  elif row.lower().find('urban'):
    return "urban"
  else:
    return "other"
  
def prefix_grouping(row, prefix_count=3):
  # Reduce the dimension based on 1st character else return *
  if str(row) == "nan":
    return np.nan
  
  if prefix_count > len(row):
    return "#"
  
  if row[0:prefix_count].isalpha():
    return row[0:prefix_count].lower()
  else:
    return "*"
  
def map_ward_construction_year():
  # Map ward to construction year
  
  # Here train_features_df shall be used as reference for
  # both trainging and test data set.
  df = train_features_df.copy()
  
  ward_construction_year_dict = {}
  ward_list = df['ward'].unique()
 
  # top ward's construction year shall be used incase there is no
  # matching construction year for individual ward.  
  top_ward = df['ward'].describe().top
  top_ward_construction_year =  \
    int(df[df['ward'] == top_ward]['construction_year'].median())

  for ward in ward_list:
    ward_construction_year = \
      int(df[df['ward'] == ward]['construction_year'].median())
    
    if ward not in ward_construction_year_dict:\
      
      if ward_construction_year == 0:
        ward_construction_year_dict[ward] = top_ward_construction_year
      else:
        ward_construction_year_dict[ward] = ward_construction_year
  
  return ward_construction_year_dict


def compute_construction_year(row, ward_construction_year_dict):
  # compute the consturction year if it is 0  
  ward = row['ward']
  construction_year = row['construction_year']  
  
  if construction_year == 0:
    return ward_construction_year_dict[ward]
  else:
    return construction_year


def compute_age(row):
  # compute the consturction age
  date_recorded = row['date_recorded']
  year_recorded = int(date_recorded.split('-')[0])
  
  construction_year = row['construction_year']
  
  return (year_recorded - construction_year)


def compute_year_recorded(row):
  # split year from date_recorded
  return int(row.split('-')[0])

def compute_month_recorded(row):
  # split year from date_recorded
  return int(row.split('-')[1])
def feature_engineering(df):
  # Create a column to indicate funder with atleast 5 pumps maintained.
  value_count_funder = df.funder.value_counts()
  df['funder_aleast_5'] = df['funder'].apply(atleast, 
                                            args=(value_count_funder,))
  
  # Create a column to indicate installer with atleast 5 pumps maintained.
  value_count_installer = df.installer.value_counts()
  df['installer_aleast_5'] = df['installer'].apply(atleast, 
                                            args=(value_count_installer,))
  
  # Apply mean for missing values of latitude and longitude
  mean_longitude = df['longitude'].mean()
  df['longitude'] = df['longitude'].apply(lambda x: mean_longitude if round(x, 2) == 0 else x)
  mean_latitude = df['latitude'].mean()
  df['latitude'] = df['latitude'].apply(lambda x: mean_latitude if round(x, 2) == 0 else x)
  
  # Grouping wpt_name, subvillage based on 1st alphabet
  df['wpt_name_engineered'] = df['wpt_name'].apply(character_grouping)
  df['subvillage_engineered'] = df['subvillage'].apply(character_grouping)
  
  # Classify lga based on Rural, Urban and others
  df['lga_engineered'] = df['lga'].apply(classify_lga)
  
  # Grouping ward, scheme_name based on 1st alphabet
  df['ward_engineered'] = df['ward'].apply(character_grouping)
  df['scheme_name_engineered'] = df['scheme_name'].apply(character_grouping)
  
  # Grouping based on prefix
  df['funder_prefix_grouping'] = df['funder'].apply(prefix_grouping)
  df['installer_prefix_grouping'] = df['installer'].apply(prefix_grouping)
  df['wpt_name_prefix_grouping'] = df['wpt_name'].apply(prefix_grouping)
  df['subvillage_prefix_grouping'] = df['subvillage'].apply(prefix_grouping)
  df['lga_prefix_grouping'] = df['lga'].apply(prefix_grouping)
  df['ward_prefix_grouping'] = df['ward'].apply(prefix_grouping)
  df['scheme_name_prefix_grouping'] = df['scheme_name'].apply(prefix_grouping)
  
  # Compute missing construction year
  ward_construction_year_dict = map_ward_construction_year()
  df['construction_year'] = df.apply(compute_construction_year, axis=1, 
                                     args=(ward_construction_year_dict,))
  
  # Compute age of well
  df['age'] = df.apply(compute_age, axis=1)
  
  # Fetch Year and Month of date recorded
  df['year_recorder'] = df['date_recorded'].apply(compute_year_recorded)
  df['month_recorder'] = df['date_recorded'].apply(compute_month_recorded)
  
feature_engineering(train_features_df)
train_features_df[train_features_df['age'] < 0]
pd.set_option('display.max_columns', None)
test_df = train_features_df.sample(5, random_state=42)
test_df
  test_df['age'] = test_df.apply(compute_age, axis=1)
test_df.loc[:,['date_recorded','construction_year','age']]
data = {'accuracy_score': [0.542334455667789, 0.708013468013468, 
                           0.7770370370370371, 0.7924354657687991,
                           0.793625140291807, 0.793625140291807,
                           0.793625140291807, 0.793625140291807,
                           0.793625140291807],
        'iteration': ['Majority Class Estimate', 'Feature Engineering with LR',
                      'Feature Engineering with XGB','Hyper-Parameter Tuning 1',
                      'Hyper-Parameter Tuning 2', 'Hyper-Parameter Tuning 3',
                      'Hyper-Parameter Tuning 4', 'Hyper-Parameter Tuning 5',
                      'Hyper-Parameter Tuning 6']}

accuracy_iteration_df = pd.DataFrame.from_dict(data)
accuracy_iteration_df['accuracy_score'] = accuracy_iteration_df['accuracy_score'] * 100
accuracy_iteration_df
sns.set(style="whitegrid")

# Initialize the matplotlib figure
f, ax = plt.subplots(figsize=(6, 10))


# Plot the Null
sns.set_color_codes("muted")

ax = sns.barplot(x='accuracy_score', y='iteration',
                data=accuracy_iteration_df, color="b")

ax.set(ylabel="Iterations",
       xlabel="Accuracy Score")
f.suptitle('As the model performance increases, it becomes exponentially difficult to tune.', fontsize=16)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13252309188138064, 'fraction_numerical': 0.021973748176956732, 'mean_word_length': 4.107249255213506, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '593036', 'license': 'MIT', 'n_tokens_mistral': 3571, 'n_tokens_neox': 3335, 'n_words': 723, 'path': 'Feature_Engineering.ipynb', 'repo_name': 'ShreyasJothish/taarifa_water_pumps', 'size': '10285'}