<jupyter_start><jupyter_text>Lambda School Data Science*Unit 2, Sprint 2, Module 4*---<jupyter_code>%%capture
import sys

# If you're on Colab:
if 'google.colab' in sys.modules:
    DATA_PATH = 'https://raw.githubusercontent.com/LambdaSchool/DS-Unit-2-Kaggle-Challenge/master/data/'
    !pip install category_encoders==2.*

# If you're working locally:
else:
    DATA_PATH = 'data/'
import pandas as pd
from sklearn.model_selection import train_test_split

# Merge train_features.csv & train_labels.csv
train = pd.merge(pd.read_csv(DATA_PATH+'train_features.csv'), 
                 pd.read_csv(DATA_PATH+'train_labels.csv'))

# Read test_features.csv & sample_submission.csv
test = pd.read_csv(DATA_PATH+'test_features.csv')
sample_submission = pd.read_csv(DATA_PATH+'sample_submission.csv')

train, val = train_test_split(train, train_size=0.80, test_size=0.20, 
                              stratify=train['status_group'], random_state=42)

train.shape, val.shape, test.shape
test.columns
#get the mean baseline
train['status_group'].value_counts(normalize=True)
# train['permit'].isnull().sum()
import numpy as np

def wrangle(X):
    """Wrangle train, validate, and test sets in the same way"""
    
    # Prevent SettingWithCopyWarning
    X = X.copy()
    
    # About 3% of the time, latitude has small values near zero,
    # outside Tanzania, so we'll treat these values like zero.
    X['latitude'] = X['latitude'].replace(-2e-08, 0)
    
    # When columns have zeros and shouldn't, they are like null values.
    # So we will replace the zeros with nulls, and impute missing values later.
    cols_with_zeros = ['longitude', 'latitude']
    for col in cols_with_zeros:
        X[col] = X[col].replace(0, np.nan)
            
      
    # return the wrangled dataframe
    return X


train = wrangle(train)
val = wrangle(val)
test = wrangle(test)
# The status_group column is the target
target = 'status_group'

# Set features to drop
todrop = [target, 'id', 'recorded_by', 
         'extraction_type_group','extraction_type_class', 
         'payment', 'source_type', 'quantity_group',
         'source_class']

#readded 'waterpoint_type_group',  'region_code',

# Get a dataframe with all train columns except todrop features
train_features = train.drop(columns= todrop)

# Get a list of the numeric features
numeric_features = train_features.select_dtypes(include='number').columns.tolist()

# Get a series with the cardinality of the nonnumeric features
cardinality = train_features.select_dtypes(exclude='number').nunique()

# Get a list of all categorical features with cardinality <= 50
categorical_features = cardinality[cardinality <= 50].index.tolist()

# Combine the lists 
features = numeric_features + categorical_features
print(features)
len(features)
X_train = train[features]
y_train = train[target]
X_test = test[features]
X_val = val[features]
y_val = val[target]
# y_test = test[target]
#check output is as desired (cleaned dataset)
X_test.head()
#build a pipeline for Random Forest Classifier
import category_encoders as ce
from sklearn.feature_selection import f_regression, SelectKBest
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score, RandomizedSearchCV
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from scipy.stats import randint, uniform

RFC_pipeline = make_pipeline(
    ce.OrdinalEncoder(),
    SimpleImputer(),
    StandardScaler(),
    RandomForestClassifier(random_state=42)
)

p_dist = {
    'simpleimputer__strategy': ['mean', 'median'],
    'randomforestclassifier__n_estimators': randint(50, 500), 
    'randomforestclassifier__max_depth': [5, 10, 15, 20, None], 
    'randomforestclassifier__max_features': uniform(0, 1),
}

RFC_search = RandomizedSearchCV(
    RFC_pipeline,
    param_distributions=p_dist,
    n_iter=10,
    cv=4,
    scoring='accuracy',
    verbose=10,
    return_train_score=True,
    n_jobs=-1
)

RFC_search.fit(X_train, y_train);
print('Best hyperparameters', RFC_search.best_params_)
print('Accuracy', RFC_search.best_score_)
pipeline = RFC_search.best_estimator_
k = 4
scores = cross_val_score(pipeline, X_train, y_train, cv=k, 
                         scoring='accuracy')
print(f'Accuracy for {k} folds:', scores)
print ('Validation Accuracy', pipeline.score(X_val, y_val))
# Plot a confusion matrix
from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
import seaborn as sns

unique_labels(y_val)
def plot_confusion_matrix(y_true, y_pred):
    labels = unique_labels(y_pred)
    columns = [f'Predicted {label}' for label in labels]
    index = [f'Actual {label}' for label in labels]
    table = pd.DataFrame(confusion_matrix(y_true, y_pred),
                        columns=columns, index=index)
    return sns.heatmap(table, annot=True, fmt='d', cmap = 'viridis')
y_pred = pipeline.predict(X_val)
plot_confusion_matrix(y_val, y_pred)
# make a pipeline for a Decision Tree Classifier model
# to compare accuracy results (different features than earlier work)
from sklearn.tree import DecisionTreeClassifier

DTC_pipeline = make_pipeline(
    ce.OrdinalEncoder(),
    SimpleImputer(),
    StandardScaler(),
    DecisionTreeClassifier(random_state=42)
)

p_dist = {'decisiontreeclassifier__max_depth': [5, 10, 15, 20, None], 
          'decisiontreeclassifier__max_features': uniform(0, 1),
         }

DTC_search = RandomizedSearchCV(
    DTC_pipeline,
    param_distributions=p_dist,
    n_iter=10,
    cv=4,
    scoring='accuracy',
    verbose=10,
    return_train_score=True,
    n_jobs=-1,
    
)

DTC_search.fit(X_train, y_train)
print(DTC_search.best_estimator_)
pipeline2 = DTC_search.best_estimator_
print ('Validation Accuracy', pipeline2.score(X_val, y_val))
# test_3 = pipeline.predict(X_test)
# submissionR = sample_submission.copy()
# submissionR['status_group'] = test_3
# submissionR.to_csv('Kaggle_water_Eric_Ramon_3.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10392189309945392, 'fraction_numerical': 0.01125268906172431, 'mean_word_length': 4.6644798500468605, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '179170', 'license': 'MIT', 'n_tokens_mistral': 2050, 'n_tokens_neox': 1963, 'n_words': 475, 'path': 'module4/Eric_Ramon_Kaggle_Water_Final.ipynb', 'repo_name': 'nephylum/DS-Unit-2-Kaggle-Challenge', 'size': '6043'}