<jupyter_start><jupyter_text>Kaggle Assignment 4  Previous Day's CodeBasically unchanged aside from reducing the number of dropped duplicate columns to 'payment_type', 'quantity_group', and 'extraction_type_group'. Of the "nearly-duplicate" columns, 'extraction_type_group' was the only one, when removed, that improved my val accuracy.Also added the 'year_difference' column.<jupyter_code>import sys

DATA_PATH = 'https://raw.githubusercontent.com/LambdaSchool/DS-Unit-2-Kaggle-Challenge/master/data/'
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

train = pd.merge(pd.read_csv(DATA_PATH+'waterpumps/train_features.csv'), 
                 pd.read_csv(DATA_PATH+'waterpumps/train_labels.csv'))
test = pd.read_csv(DATA_PATH+'waterpumps/test_features.csv')
sample_submission = pd.read_csv(DATA_PATH+'waterpumps/sample_submission.csv')

train.shape, test.shape
#Checked the average elevation of Tanzania, and it's 1200, so I 
#really can ignore the zeroes here. But also the lowest elevation is 0,
#so I'm a little confused about the negative numbers
train['gps_height'].value_counts()
# Numeric Columns to clean
numeric_to_clean = ['longitude','latitude','construction_year', 'gps_height']
# Checking for duplicat columns
duplicates1 = ['extraction_type','extraction_type_group','extraction_type_class']
duplicates2 = ['payment','payment_type']
duplicates3 = ['quantity_group','quantity']
duplicates4 = ['source','source_type']
duplicates5 = ['waterpoint_type','waterpoint_type_group']
train.head()
#Checking the duplicates to decide which to keep
duplicate_lists = [duplicates1, duplicates2, duplicates3, duplicates4,
                   duplicates5]

for duplicate in duplicate_lists:
  print(train[duplicate].describe())
  print("")
#my_train['region'].value_counts().index
Mwanza = train[train['region'] == 'Mwanza']
Mwanza['longitude'] = Mwanza['longitude'].replace(0,np.nan)
Mwanza['latitude'] = Mwanza['latitude'].replace(-2e-08, np.nan)
np.mean(Mwanza['latitude'])
np.mean(Mwanza['longitude'])
Shinyanga = train[train['region'] == 'Shinyanga']
Shinyanga['longitude'] = Shinyanga['longitude'].replace(0,np.nan)
Shinyanga['latitude'] = Shinyanga['latitude'].replace(-2e-08, np.nan)
np.mean(Shinyanga['longitude'])
np.mean(Shinyanga['latitude'])
#Looking at the above lists, I'll remove the duplicate columns
# and nearly duplicate columns that have fewer unique variables

duplicates_to_drop = ['extraction_type_group','extraction_type_class',
                    'payment_type','quantity_group', 'source_type',
                    'waterpoint_type_group']
from sklearn.model_selection import train_test_split

my_train, my_val = train_test_split(train, random_state=333)
my_train[my_train['longitude']==0]['region'].value_counts()
my_train[my_train['region']== 'Mwanza']
import numpy as np
def wrangle(X):
  
  # To prevent copy warnings
  X = X.copy()

  # Latitude is strange in that it doesn't have any 0s, but it does have these
  # near-0 values
  X['latitude'] = X['latitude'].replace(-2e-08, 0)

  # there are numeric columns with 0s that should be nana
  nans_as_zeros = ['latitude','longitude', 'construction_year',
                   'gps_height', 'population']
  for column in nans_as_zeros:
    X[column] = X[column].replace(0, np.nan)
    #I like this code Ryan had to make a new column for if its missing data
    X[column+'_MISSING'] = X[column].isnull()    
  
  # X['longitude'] = X.apply(
  #   lambda row: np.mean(Mwanza['longitude']) if np.isnan(row['longitude']) and row['region'] == 'Mwanza' else row['longitude'],
  #   axis=1)
  # X['longitude'] = X.apply(
  #   lambda row: np.mean(Shinyanga['longitude']) if np.isnan(row['longitude']) and row['region'] == 'Shinyanga' else row['longitude'],
  #   axis=1)


  # X['latitude'] = X.apply(
  #   lambda row: np.mean(Mwanza['latitude']) if np.isnan(row['latitude']) and row['region'] == 'Mwanza' else row['latitude'],
  #   axis=1)
  # X['latitude'] = X.apply(
  #   lambda row: np.mean(Shinyanga['latitude']) if np.isnan(row['latitude']) and row['region'] == 'Shinyanga' else row['latitude'],
  #   axis=1)
  
  #Date recorded is treated as an int. Extracting y/m/d
  X['date_recorded'] = pd.to_datetime(X['date_recorded'])
  X['year_recorded'] = X['date_recorded'].dt.year
  X['month_recorded'] = X['date_recorded'].dt.month
  X['day_recorded'] = X['date_recorded'].dt.day
  X = X.drop(columns='date_recorded')
    
  # Engineer feature: how many years from construction_year to date_recorded
  X['years_difference'] = X['year_recorded'] - X['construction_year']    

  #Removing duplicate or near-duplicate columns
  X = X.drop(columns=['payment_type','quantity_group', 'extraction_type_group'])

  #Can be used for each train and validation
  return X
%%time
my_train = wrangle(my_train)
my_val = wrangle(my_val)
test = wrangle(test)
my_train['longitude'].value_counts()
my_train[my_train['id']== 6091]
# # Copied from previous assignment. 
# # Unecessary because high cardinality features are fine

# # # Selecting target

# target = 'status_group'

# #Removing the target and useless id columns
# train_columns = my_train.drop(columns=[target,'id'])

# # separating numeric columns to readd after
# numeric_columns = train_columns.select_dtypes(include='number').columns.tolist()

# #Getting a list of cardinality for categorical features to exclude the large
# cardinality = train_columns.select_dtypes(exclude='number').nunique()

# #Excluding features with a cardinality over 50
# categorical_columns = cardinality[cardinality <50].index.tolist()

# #combining lists to get the features I will use for my model
# features = numeric_columns + categorical_columns
# We can use high cardinality features, so no need to remove them
target = 'status_group'

features = my_train.drop(columns=[target,'id']).columns
#Assigning variables

X_train = my_train[features]
y_train = my_train[target]

X_val = my_val[features]
y_val = my_val[target]

X_test = test[features]<jupyter_output><empty_output><jupyter_text>Assignment 4 code Went with the column transformer to use ordinal encoding ONLY for the high cardinality columns, one hot encoding otherwise.<jupyter_code>from sklearn.compose import ColumnTransformer
import category_encoders as ce
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import make_pipeline
from sklearn.pipeline import Pipeline

#Getting a list of cardinality for categorical features to include in ordinal encoder
train_columns = my_train.drop(columns=[target,'id'])
cardinality = train_columns.select_dtypes(exclude='number').nunique()
high_cardinality = cardinality[cardinality >49].index.tolist()

# Making the high cardinality pipeline
high_cardinality_pipeline = make_pipeline(
    ce.OrdinalEncoder(),
    SimpleImputer(strategy='median')
)

# Making the non-high cardinality pipeline
non_high_cardinality = train_columns.drop(columns=high_cardinality).columns

non_high_cardinality_pipeline = make_pipeline(
    ce.OneHotEncoder(),
    SimpleImputer(strategy='median')
)


#Associating the transformers with the list of features
preprocessor = ColumnTransformer(
    transformers=[
        ('high_card',high_cardinality_pipeline, high_cardinality),
        ('low_card',non_high_cardinality_pipeline, non_high_cardinality)])


my_pipeline = make_pipeline(
    preprocessor,
    RandomForestClassifier(n_estimators=100, random_state=333, n_jobs=-1,
                           max_depth=20)
)
my_pipeline.fit(X_train,y_train)
my_pipeline.score(X_val,y_val)
from scipy.stats import randint, uniform
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV<jupyter_output><empty_output><jupyter_text>Set up the RandomizedSearchCV to<jupyter_code>%%time

param_distributions = {
    'randomforestclassifier__n_estimators': randint(100, 200),
    'randomforestclassifier__max_depth': [17, 18,19,20,21,22, None],
    'randomforestclassifier__min_samples_leaf': [1,2,3,4,5]
}

search = RandomizedSearchCV(
    my_pipeline,
    param_distributions=param_distributions,
    n_iter=10,
    cv=5,
    scoring='accuracy',
    verbose=10,
    return_train_score=False,
    n_jobs=-1
)
search.fit(X_train,y_train)
search.best_score_
search.best_params_
search.best_estimator_<jupyter_output><empty_output><jupyter_text>Submission Code<jupyter_code>pipeline = search.best_estimator_
pred_y_test = pipeline.predict(X_test)
cv8_submission = test.copy()
cv8_submission['status_group'] = pred_y_test
cv8_submission = cv8_submission.filter(['id','status_group'])
cv8_submission
cv8_submission.to_csv('cv8_submission.csv', index=False)<jupyter_output><empty_output><jupyter_text>The code below used to combine the my_train and my_val data to use in the final fitting with the randomCV optomized model<jupyter_code>my_train.shape,my_val.shape
# Combining all training data to use with the best model the randomcv found
final_train = pd.concat([my_train,my_val], ignore_index=True)
final_train.shape
final_train.head()
pipeline.fit(final_train[features],final_train[target])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11808604038630377, 'fraction_numerical': 0.010645302897278315, 'mean_word_length': 5.967125382262997, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '637210', 'license': 'MIT', 'n_tokens_mistral': 3076, 'n_tokens_neox': 2939, 'n_words': 678, 'path': 'module4/assignment_kaggle_challenge_4.ipynb', 'repo_name': 'Pdugovich/DS-Unit-2-Kaggle-Challenge', 'size': '9112'}