<jupyter_start><jupyter_text><jupyter_code>%%time 

# Important library for many geopython libraries
!apt install gdal-bin python-gdal python3-gdal 
# Install rtree - Geopandas requirment
!apt install python3-rtree 
# Install Geopandas
!pip install git+git://github.com/geopandas/geopandas.git
# Install descartes - Geopandas requirment
!pip install descartes 
# Install Folium for Geographic data visualization
!pip install folium
# Install plotlyExpress
!pip install plotly_express
!pip install shapely
from shapely.geometry.polygon import Polygon
from shapely.geometry.multipolygon import MultiPolygon
!pip install PyShp
import pandas as pd
import geopandas as gdp 
import matplotlib.pyplot as plt
import seaborn as sns 
import numpy as np
from sklearn.model_selection import train_test_split
  !pip install category_encoders==2.*
  !pip install pandas-profiling==2.*
import sys

# If you're on Colab:
if 'google.colab' in sys.modules:
    DATA_PATH = 'https://raw.githubusercontent.com/LambdaSchool/DS-Unit-2-Kaggle-Challenge/master/data/'
train = pd.merge(pd.read_csv('/content/final_waterpumps.csv'), 
                 pd.read_csv(DATA_PATH+'waterpumps/train_labels.csv'))

test = pd.read_csv('/content/test_final.csv')

sample_submission = pd.read_csv(DATA_PATH+'waterpumps/sample_submission.csv')
train['date_recorded'] = train['date_recorded'].astype('datetime64[ns]')
train['date_recorded'] = train['date_recorded'].dt.year
train['age'] = train['date_recorded']-train['construction_year']
train['age']
train['age'] = train['age'].replace([2011],0)
train['age']
test['date_recorded'] = test['date_recorded'].astype('datetime64[ns]')
test['date_recorded'] = test['date_recorded'].dt.year
test['age'] = test['date_recorded']-test['construction_year']

test['age'] = test['age'].replace([2011],0)
test['age']
train, val = train_test_split(train, train_size=0.90, test_size=0.10, 
                              stratify=train['status_group'], random_state=42)
train.shape, val.shape, test.shape
train['status_group'].value_counts(normalize=True)
#trained = train.drop(['extraction_type', 'id', 'index_right', 'quantity_group', 'num_private', 'basin', 'region', 'AREA', 'PERIMETER', 'public_meeting', 'recorded_by', 'scheme_management', 'permit', 'extraction_type_group', 'extraction_type_class', 'management_group', 'payment', 'payment_type', 'quality_group', 'source_type', 'source_class', 'MAIN_TYPE', 'date_recorded', 'waterpoint_type_group', 'status_group'], axis=1)
train.head()
trained = train.drop(['id', 'num_private', 'extraction_type_class', 'management_group',
       'region_code', 'recorded_by', 'geometry', 'date_recorded', 'permit', 'basin',
       'scheme_management', 'payment_type', 'water_quality', 'construction_year', 'SOIL_TYPE',
       'extraction_type_group', 'public_meeting', 'AREA', 'installer', 'PERIMETER',
       'management', 'MAIN_TYPE', 'quality_group', 'quantity_group', 'waterpoint_type_group', 
       'source_type', 'source_class', 'index_right', 'status_group'], axis=1)
trained.columns
numeric_features = trained.select_dtypes(include='number').columns.tolist()

# Get a series with the cardinality of the nonnumeric features
cardinality = trained.select_dtypes(exclude='number').nunique()

# Get a list of all categorical features with cardinality <= 10000
categorical_features = cardinality[cardinality <= 100000].index.tolist()

# Combine the lists 

features = numeric_features + categorical_features
print(features)
target = train['status_group']
X_train = trained[features]
y_train = train['status_group']
X_val = val[features]
y_val = val['status_group']
X_test = test[features]
import category_encoders as ce
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler

pipeline = make_pipeline(
    ce.OrdinalEncoder(),
    SimpleImputer(strategy='mean'), 
    StandardScaler(), 
    RandomForestClassifier(n_estimators=310, 
                               bootstrap = True,
                               max_features = 'sqrt')
)


# Fit on train
pipeline.fit(X_train, y_train)

# Score on val
print('Validation Accuracy', pipeline.score(X_val, y_val))

# Predict on test
y_pred = pipeline.predict(X_test)
Submission = pd.DataFrame(y_pred, columns = ['status_group'])
Submission
sample = pd.read_csv('/content/sample_submission.csv')
sample
Submission['id'] = sample['id']
Submission = Submission[['id', 'status_group']]
Submission
Submission.to_csv('GeoTest27.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1340453356582389, 'fraction_numerical': 0.00980819529206626, 'mean_word_length': 6.08179012345679, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '481250', 'license': 'MIT', 'n_tokens_mistral': 1529, 'n_tokens_neox': 1461, 'n_words': 307, 'path': 'TanzaniaGEOTEST_RanForrModel.ipynb', 'repo_name': 'kvinne-anc/Tanzania-Soil-Topography-Waterpumps-Project', 'size': '4588'}