<jupyter_start><jupyter_text><jupyter_code># Installs
%%capture
!pip install --upgrade category_encoders plotly
# Imports
import os, sys

os.chdir('/content')
!git init .
!git remote add origin https://github.com/LambdaSchool/DS-Unit-2-Regression-Classification.git
!git pull origin master
    
os.chdir('module4')
# Disable warning
import warnings
warnings.filterwarnings(action='ignore', category=FutureWarning, module='numpy')<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>import pandas as pd

train_features = pd.read_csv('../data/tanzania/train_features.csv')
train_labels = pd.read_csv('../data/tanzania/train_labels.csv')
test_features = pd.read_csv('../data/tanzania/test_features.csv')
sample_submission = pd.read_csv('../data/tanzania/sample_submission.csv')

assert train_features.shape == (59400, 40)
assert train_labels.shape == (59400, 2)
assert test_features.shape == (14358, 40)
assert sample_submission.shape == (14358, 2)<jupyter_output><empty_output><jupyter_text>Train/Validate/Test Split<jupyter_code># Load initial train features and labels
from sklearn.model_selection import train_test_split
X_train = train_features
y_train = train_labels['status_group']

X_train.shape, y_train.shape
# Split the initial train features and labels 80% into new train and new validation
X_train, X_val, y_train, y_val = train_test_split(
  X_train, y_train, train_size = 0.80, test_size = 0.20,
  stratify = y_train, random_state=42
)

X_train.shape, X_val.shape, y_train.shape, y_val.shape
# Check values of new train labels
y_train.value_counts(normalize=True)
# Check values of new validation labels
y_val.value_counts(normalize=True)<jupyter_output><empty_output><jupyter_text>One-Hot Encoding - Quantity<jupyter_code># Check values of quantity feature
X_train['quantity'].value_counts(normalize=True)
# Recombine X_train and y_train, for exploratory data analysis
train = X_train.copy()
train['status_group'] = y_train

train.groupby('quantity')['status_group'].value_counts(normalize=True)
# Plot the values, dry shows a strong relationship to functional
import matplotlib.pyplot as plt
import seaborn as sns

train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='quantity', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Water Quantity')<jupyter_output><empty_output><jupyter_text>One-Hot Encoding - Waterpoint Type<jupyter_code>X_train['waterpoint_type'].value_counts(normalize=True)
# Recombine X_train and y_train, for exploratory data analysis
train = X_train.copy()
train['status_group'] = y_train
train.groupby('waterpoint_type')['status_group'].value_counts(normalize=True)
train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='waterpoint_type', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Waterpoint Type')<jupyter_output><empty_output><jupyter_text>One-Hot Encoding - Extraction Type<jupyter_code>X_train['extraction_type'].value_counts(normalize=True)
train = X_train.copy()
train['status_group'] = y_train
train.groupby('extraction_type')['status_group'].value_counts(normalize=True)
train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='extraction_type', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Extraction Type')<jupyter_output><empty_output><jupyter_text>Bin and One-Hot Encoding - Installer<jupyter_code>X_train['installer'] = X_train['installer'].str.lower()
X_val['installer'] = X_val['installer'].str.lower()

X_train['installer'] = X_train['installer'].str.lower()
X_val['installer'] = X_val['installer'].str.lower()

X_train['installer'] = X_train['installer'].str.replace('danid', 'danida')
X_val['installer'] = X_val['installer'].str.replace('danid', 'danida')

X_train['installer'] = X_train['installer'].str.replace('disti', 'district council')
X_val['installer'] = X_val['installer'].str.replace('disti', 'district council')

X_train['installer'] = X_train['installer'].str.replace('commu', 'community')
X_val['installer'] = X_val['installer'].str.replace('commu', 'community')

X_train['installer'] = X_train['installer'].str.replace('central government', 'government')
X_val['installer'] = X_val['installer'].str.replace('central government', 'government')

X_train['installer'] = X_train['installer'].str.replace('kkkt _ konde and dwe', 'kkkt')
X_val['installer'] = X_val['installer'].str.replace('kkkt _ konde and dwe', 'kkkt')

X_train['installer'].value_counts(normalize=True)
top10 = X_train['installer'].value_counts()[:5].index
X_train.loc[~X_train['installer'].isin(top10), 'installer'] = 'Other'
X_val.loc[~X_val['installer'].isin(top10), 'installer'] = 'Other'

train = X_train.copy()
train['status_group'] = y_train

train.groupby('installer')['status_group'].value_counts(normalize=True)
train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='installer', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Installer')<jupyter_output><empty_output><jupyter_text>New Feature - Pump Age<jupyter_code>X_train['pump_age'] = 2013 - X_train['construction_year']
X_train.loc[X_train['pump_age'] == 2013, 'pump_age'] = 0
X_val['pump_age'] = 2013 - X_val['construction_year']
X_val.loc[X_val['pump_age'] == 2013, 'pump_age'] = 0

X_train.loc[X_train['pump_age'] == 0, 'pump_age'] = 10
X_val.loc[X_val['pump_age'] == 0, 'pump_age'] = 10

train = X_train.copy()
train['status_group'] = y_train

train.groupby('pump_age')['status_group'].value_counts(normalize=True)
train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='pump_age', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Pump Age')<jupyter_output>/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  """Entry point for launching an IPython kernel.
/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:543: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  self.obj[item] = s
/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http[...]<jupyter_text>Bin and One-Hot Encoding - Funder<jupyter_code>X_train['funder'] = X_train['funder'].str.lower()
X_val['funder'] = X_val['funder'].str.lower()

X_train['funder'] = X_train['funder'].str[:3]
X_val['funder'] = X_val['funder'].str[:3]

X_train['funder'].value_counts(normalize=True)
top10 = X_train['funder'].value_counts()[:20].index
X_train.loc[~X_train['funder'].isin(top10), 'funder'] = 'Other'
X_val.loc[~X_val['funder'].isin(top10), 'funder'] = 'Other'

train = X_train.copy()
train['status_group'] = y_train

train.groupby('funder')['status_group'].value_counts(normalize=True)

train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='funder', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Funder')<jupyter_output>/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  """Entry point for launching an IPython kernel.
/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  
/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/p[...]<jupyter_text>One-Hot Encoding - Water Quality<jupyter_code>X_train['water_quality'].value_counts(normalize=True)
train = X_train.copy()
train['status_group'] = y_train
train.groupby('water_quality')['status_group'].value_counts(normalize=True)
train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='water_quality', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Water Quality')<jupyter_output><empty_output><jupyter_text>One-Hot Encoding - Basin<jupyter_code>X_train['basin'].value_counts(normalize=True)
train = X_train.copy()
train['status_group'] = y_train
train.groupby('basin')['status_group'].value_counts(normalize=True)
train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='basin', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Basin')<jupyter_output><empty_output><jupyter_text>One-Hot Encoding - Region<jupyter_code>X_train['region'].value_counts(normalize=True)
train = X_train.copy()
train['status_group'] = y_train
train.groupby('region')['status_group'].value_counts(normalize=True)
train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='region', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Region')<jupyter_output><empty_output><jupyter_text>Use Mean for GPS Height Missing Values<jupyter_code>X_train.loc[X_train['gps_height'] == 0, 'gps_height'] = X_train['gps_height'].mean()
X_val.loc[X_val['gps_height'] == 0, 'gps_height'] = X_val['gps_height'].mean()

train = X_train.copy()
train['status_group'] = y_train

train.groupby('gps_height')['status_group'].value_counts(normalize=True)
train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']];

#sns.catplot(x='amount_tsh', y='functional', data=train, kind='bar', color='grey')
#plt.title('% of Waterpumps Functional by Pump Age')<jupyter_output>/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:543: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  self.obj[item] = s<jupyter_text>One-Hot Encoding - Payment<jupyter_code>X_train['payment'].value_counts(normalize=True)
train = X_train.copy()
train['status_group'] = y_train
train.groupby('payment')['status_group'].value_counts(normalize=True)
train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='payment', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Payment')<jupyter_output><empty_output><jupyter_text>One-Hot Encoded - Source<jupyter_code>X_train['source'].value_counts(normalize=True)
train = X_train.copy()
train['status_group'] = y_train
train.groupby('source')['status_group'].value_counts(normalize=True)
train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='source', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Source')<jupyter_output><empty_output><jupyter_text>Bin and One-Hot Encoded - LGA<jupyter_code>X_train['lga'].value_counts(normalize=True)
top10 = X_train['lga'].value_counts()[:10].index
X_train.loc[~X_train['lga'].isin(top10), 'lga'] = 'Other'
X_val.loc[~X_val['lga'].isin(top10), 'lga'] = 'Other'

train = X_train.copy()
train['status_group'] = y_train

train.groupby('lga')['status_group'].value_counts(normalize=True)

train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='lga', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by LGA')<jupyter_output>/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:543: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  self.obj[item] = s<jupyter_text>Bin and One-Hot Encoded - Ward<jupyter_code>X_train['ward'].value_counts(normalize=True)
top10 = X_train['ward'].value_counts()[:20].index
X_train.loc[~X_train['ward'].isin(top10), 'ward'] = 'Other'
X_val.loc[~X_val['ward'].isin(top10), 'ward'] = 'Other'

train = X_train.copy()
train['status_group'] = y_train

train.groupby('ward')['status_group'].value_counts(normalize=True)

train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='ward', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Ward')<jupyter_output>/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:543: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  self.obj[item] = s<jupyter_text>One-Hot Encode - Scheme Management<jupyter_code>X_train['scheme_management'].value_counts(normalize=True)
train = X_train.copy()
train['status_group'] = y_train
train.groupby('scheme_management')['status_group'].value_counts(normalize=True)
train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='scheme_management', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Scheme Management')<jupyter_output><empty_output><jupyter_text>One-Hot Encode - Management<jupyter_code>X_train['management'].value_counts(normalize=True)
train = X_train.copy()
train['status_group'] = y_train
train.groupby('management')['status_group'].value_counts(normalize=True)
train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='management', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Management')<jupyter_output><empty_output><jupyter_text>Create a Region/District Feature<jupyter_code>X_train['region_code'].value_counts(normalize=True)
train = X_train.copy()
train['status_group'] = y_train
train.groupby('region_code')['status_group'].value_counts(normalize=True)
train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='region_code', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Region Code')
X_train['region_district'] = X_train['region_code'].astype(str) + X_train['district_code'].astype(str)
X_val['region_district'] = X_val['region_code'].astype(str) + X_val['district_code'].astype(str)

train = X_train.copy()
train['status_group'] = y_train

train.groupby('region_district')['status_group'].value_counts(normalize=True)
train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='region_district', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Region/District')<jupyter_output>/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  """Entry point for launching an IPython kernel.
/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy<jupyter_text>One-Hot Encode - Subvillage<jupyter_code>X_train['subvillage'].value_counts(normalize=True)
top10 = X_train['subvillage'].value_counts()[:10].index
X_train.loc[~X_train['subvillage'].isin(top10), 'subvillage'] = 'Other'
X_val.loc[~X_val['subvillage'].isin(top10), 'subvillage'] = 'Other'

train = X_train.copy()
train['status_group'] = y_train

train.groupby('subvillage')['status_group'].value_counts(normalize=True)

train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='subvillage', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Subvillage')<jupyter_output>/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:543: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  self.obj[item] = s<jupyter_text>One-Hot Encoding - Water Quality<jupyter_code>X_train['water_quality'].value_counts(normalize=True)
train = X_train.copy()
train['status_group'] = y_train
train.groupby('water_quality')['status_group'].value_counts(normalize=True)
train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']]

sns.catplot(x='water_quality', y='functional', data=train, kind='bar', color='grey')
plt.title('% of Waterpumps Functional by Quality')<jupyter_output><empty_output><jupyter_text>Lat/Long Cleanup<jupyter_code>#test['region'].value_counts()
average_lat = X_train.groupby('region').latitude.mean().reset_index()
average_long = X_train.groupby('region').longitude.mean().reset_index()

shinyanga_lat = average_lat.loc[average_lat['region'] == 'Shinyanga', 'latitude']
shinyanga_long = average_long.loc[average_lat['region'] == 'Shinyanga', 'longitude']

X_train.loc[(X_train['region'] == 'Shinyanga') & (X_train['latitude'] > -1), ['latitude']] = shinyanga_lat[17]
X_train.loc[(X_train['region'] == 'Shinyanga') & (X_train['longitude'] == 0), ['longitude']] = shinyanga_long[17]

mwanza_lat = average_lat.loc[average_lat['region'] == 'Mwanza', 'latitude']
mwanza_long = average_long.loc[average_lat['region'] == 'Mwanza', 'longitude']

X_train.loc[(X_train['region'] == 'Mwanza') & (X_train['latitude'] > -1), ['latitude']] = mwanza_lat[13]
X_train.loc[(X_train['region'] == 'Mwanza') & (X_train['longitude'] == 0) , ['longitude']] = mwanza_long[13]<jupyter_output>/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:543: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  self.obj[item] = s<jupyter_text>Impute Amount TSH<jupyter_code>def tsh_calc(tsh, source, base, waterpoint):
  if tsh == 0:
    if (source, base, waterpoint) in tsh_dict:
      new_tsh = tsh_dict[source, base, waterpoint]
      return new_tsh
  else:
    return tsh
  
  return tsh
temp = X_train[X_train['amount_tsh'] != 0].groupby(['source_class',
                                                    'basin',
                                                    'waterpoint_type_group'])['amount_tsh'].mean()

tsh_dict = dict(temp)
X_train['amount_tsh'] = X_train.apply(lambda x: tsh_calc(x['amount_tsh'], x['source_class'], x['basin'], x['waterpoint_type_group']), axis=1)
#X_train
#X_train.loc[X_train['amount_tsh'] == 0, 'amount_tsh'] = X_train['amount_tsh'].median()
#X_val.loc[X_val['amount_tsh'] == 0, 'amount_tsh'] = X_val['amount_tsh'].median()

train = X_train.copy()
train['status_group'] = y_train

train.groupby('amount_tsh')['status_group'].value_counts(normalize=True)
train['functional']= (train['status_group'] == 'functional').astype(int)
train[['status_group', 'functional']];
#X_train.loc[X_train['public_meeting'].isnull(), 'public_meeting'] = False
#X_val.loc[X_val['public_meeting'].isnull(), 'public_meeting'] = False

#train = X_train.copy()
#train['status_group'] = y_train

#train.groupby('public_meeting')['status_group'].value_counts(normalize=True)
#train['functional']= (train['status_group'] == 'functional').astype(int)
#train[['status_group', 'functional']];

#sns.catplot(x='public_meeting', y='functional', data=train, kind='bar', color='grey')
#plt.title('% of Waterpumps Functional by Region/District')<jupyter_output><empty_output><jupyter_text>Run the Logistic Regression<jupyter_code>import sklearn
sklearn.__version__

# Import the class
from sklearn.linear_model import LogisticRegressionCV

# Import package and scaler
import category_encoders as ce
from sklearn.preprocessing import StandardScaler
# use quantity feature and the numerical features but drop id
categorical_features = ['quantity', 'waterpoint_type', 'extraction_type', 'installer',
                         'basin', 'region', 'payment', 'source', 'lga', 'public_meeting',
                         'scheme_management', 'permit', 'management', 'region_district',
                         'subvillage', 'funder', 'water_quality', 'ward']
                                             
# 
numeric_features = X_train.select_dtypes('number').columns.drop('id').drop('num_private').tolist()
features = categorical_features + numeric_features

# make subsets using the quantity feature all numeric features except id
X_train_subset = X_train[features]
X_val_subset = X_val[features]

# Do the encoding
encoder = ce.OneHotEncoder(use_cat_names=True)
X_train_encoded = encoder.fit_transform(X_train_subset)
X_val_encoded = encoder.transform(X_val_subset)

# Use the scaler
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train_encoded)
X_val_scaled = scaler.transform(X_val_encoded)

# Fit the model and check the accuracy
model = LogisticRegressionCV(n_jobs = -1)
model.fit(X_train_scaled, y_train)

print('Validation Accuracy', model.score(X_val_scaled, y_val));<jupyter_output>/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.
  "this warning.", FutureWarning)
/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.
  warnings.warn(CV_WARNING, FutureWarning)<jupyter_text>Run RandomForestClassifier<jupyter_code>from sklearn.ensemble import RandomForestClassifier

model = RandomForestClassifier(n_estimators=1000, 
                               random_state=42,
                               max_features = 'auto',
                               n_jobs=-1,
                               verbose = 1)

model.fit(X_train_scaled, y_train)

print('Validation Accuracy', model.score(X_val_scaled, y_val));


test_features['pump_age'] = 2013 - test_features['construction_year']
test_features.loc[test_features['pump_age'] == 2013, 'pump_age'] = 0

test_features['region_district'] = test_features['region_code'].astype(str) + test_features['district_code'].astype(str)

test_features.drop(columns=['num_private'])

X_test_subset = test_features[features]
X_test_encoded = encoder.transform(X_test_subset)
X_test_scaled = scaler.transform(X_test_encoded)
assert all(X_test_encoded.columns == X_train_encoded.columns)

y_pred = model.predict(X_test_scaled)
submission = sample_submission.copy()
submission['status_group'] = y_pred
submission.to_csv('/content/submission-01.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15906096538507605, 'fraction_numerical': 0.008464087515470915, 'mean_word_length': 7.2584899439498844, 'pattern_counts': {'":': 0, '<': 96, '<?xml version=': 0, '>': 98, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '26', 'dir': 'jupyter-structured-clean-dedup', 'id': '7522', 'license': 'MIT', 'n_tokens_mistral': 9045, 'n_tokens_neox': 8660, 'n_words': 1524, 'path': 'module4/assignment_regression_classification_4e.ipynb', 'repo_name': 'JimKing100/DS-Unit-2-Regression-Classification', 'size': '25047'}