<jupyter_start><jupyter_text>Notebook 3b - Breakdown Soil TypeIn this notebook, we test out breaking down the soil-type features using domain knowledge and their descriptions<jupyter_code># Global variables for testing changes to this notebook quickly
RANDOM_SEED = 0
NUM_FOLDS = 12
import numpy as np
import pandas as pd
import time
import pyarrow
import gc

# Model evaluation
from functools import partial
from sklearn.base import clone
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedKFold, train_test_split
from sklearn.metrics import accuracy_score, recall_score
from sklearn.inspection import partial_dependence, permutation_importance
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import AdaBoostClassifier, BaggingClassifier, ExtraTreesClassifier, RandomForestClassifier

# Plotting
import matplotlib
import seaborn as sns
from matplotlib import pyplot as plt

# Hide warnings
import warnings
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>Prepare Data<jupyter_code># Encode soil type
def categorical_encoding(input_df):
    data = input_df.copy()
    data['Soil_Type'] = 0
    soil_features = list()
    for i in range(1,41):
        data['Soil_Type'] += i*data[f'Soil_Type{i}']
        soil_features.append(f'Soil_Type{i}')
    nonsoil_features = [x for x in data.columns if x not in soil_features]
    return data[nonsoil_features]
%%time

# Load original data
original = categorical_encoding(pd.read_feather('../data/original.feather'))

# Label Encode
old_encoder = LabelEncoder()
original["Cover_Type"] = old_encoder.fit_transform(original["Cover_Type"])
y_train = original['Cover_Type'].iloc[:15119]
y_test = original['Cover_Type'].iloc[15119:]

# Get feature columns
features = [x for x in original.columns if x not in ['Id','Cover_Type']]

# Data structures for summary scores
bagging_scores = list()
extratrees_scores = list()
adaboost_scores = list()
random_scores = list()<jupyter_output>Wall time: 273 ms<jupyter_text>Scoring Function<jupyter_code>def train_original(sklearn_model, processing = None):
    
    # Original Training/Test Split
    X_temp = original[features].iloc[:15119]
    X_test = original[features].iloc[15119:]
    y_temp = original['Cover_Type'].iloc[:15119]
    y_test = original['Cover_Type'].iloc[15119:]
    
    # Feature Engineering
    if processing:
        X_temp = processing(X_temp)
        X_test = processing(X_test)
        
    # Store the out-of-fold predictions
    test_preds = np.zeros((X_test.shape[0],7))
    oof_preds = np.zeros((X_temp.shape[0],))
    scores, times = np.zeros(NUM_FOLDS), np.zeros(NUM_FOLDS)
    
    # Stratified k-fold cross-validation
    skf = StratifiedKFold(n_splits = NUM_FOLDS, shuffle = True, random_state = RANDOM_SEED)
    for fold, (train_idx, valid_idx) in enumerate(skf.split(X_temp,y_temp)):
       
        # Training and Validation Sets
        X_train, X_valid = X_temp.iloc[train_idx], X_temp.iloc[valid_idx]
        y_train, y_valid = y_temp.iloc[train_idx], y_temp.iloc[valid_idx]
        
        # Create model
        start = time.time()
        model = clone(sklearn_model)
        model.fit(X_train, y_train)

        # validation and test predictions
        valid_preds = np.ravel(model.predict(X_valid))
        oof_preds[valid_idx] = valid_preds
        test_preds += model.predict_proba(X_test)
        
        # Save scores and times
        scores[fold] = accuracy_score(y_valid, valid_preds)
        end = time.time()
        times[fold] = end-start
        time.sleep(0.5)
    
    test_preds = np.argmax(test_preds, axis = 1)
    test_score = accuracy_score(y_test, test_preds)
    print('\n'+model.__class__.__name__)
    print("Train Accuracy:", round(scores.mean(), 5))
    print('Test Accuracy:', round(test_score, 5))
    print(f'Training Time: {round(times.sum(), 2)}s')
    
    return scores.mean(), oof_preds, test_score<jupyter_output><empty_output><jupyter_text>ModelsWe use the following 4 models from the scikit-learn library:1. AdaBoost 2. ExtraTrees3. Bagging4. Random Forest<jupyter_code># AdaBoost Classifier
adaboost = AdaBoostClassifier(
    base_estimator = DecisionTreeClassifier(
        splitter = 'random',
        random_state = RANDOM_SEED,
    ),
    random_state = RANDOM_SEED,
)

# ExtraTrees Classifier
extratrees = ExtraTreesClassifier(
    n_jobs = -1,
    random_state = RANDOM_SEED,
    max_features = None,
)

# Bagging Classifier
bagging = BaggingClassifier(
    base_estimator = DecisionTreeClassifier(
        splitter = 'random',
        random_state = RANDOM_SEED,
    ),
    n_jobs = -1,
    random_state = RANDOM_SEED
)

# Random Forest Classifier
randomforest = RandomForestClassifier(
    n_jobs = -1,
    random_state = RANDOM_SEED,
)<jupyter_output><empty_output><jupyter_text>Baselines<jupyter_code># AdaBoost
cv_score, oof_preds, test_score = train_original(adaboost)

adaboost_scores.append((
    'Baseline', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))

# ExtraTrees
cv_score, oof_preds, test_score = train_original(extratrees)

extratrees_scores.append((
    'Baseline', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))

# Bagging
cv_score, oof_preds, test_score = train_original(bagging)

bagging_scores.append((
    'Baseline', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))

# Random Forest
cv_score, oof_preds, test_score = train_original(randomforest)

random_scores.append((
    'Baseline', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))<jupyter_output>AdaBoostClassifier
Train Accuracy: 0.78596
Test Accuracy: 0.76229
Training Time: 3.79s

ExtraTreesClassifier
Train Accuracy: 0.88538
Test Accuracy: 0.78206
Training Time: 36.41s

BaggingClassifier
Train Accuracy: 0.85006
Test Accuracy: 0.75982
Training Time: 20.23s

RandomForestClassifier
Train Accuracy: 0.86454
Test Accuracy: 0.74822
Training Time: 37.07s<jupyter_text>Soil Type FeaturesWe test the following features based off of the soil type descriptions from the original data:1. Climatic Zone (Ordinal)2. Geologic Zone (Nominal)3. Surface Cover (Ordinal)4. Rock Size (Ordinal)  Climatic Zone (Ordinal)We create a feature based on the climatic zone of the soil, which has a natural ordering:1. lower montane dry2. lower montane3. montane dry4. montane5. montane dry and montane6. montane and subalpine7. subalpine8. alpineHowever, the ordering of the soil type labels roughly follows the ordering of their respectively climatic zones, so there's a chance this feature won't be particularly informative.<jupyter_code>def climatic_zone_original(input_df):
    code = {
        1:2702,2:2703,3:2704,4:2705,5:2706,6:2717,7:3501,8:3502,9:4201,
        10:4703,11:4704,12:4744,13:4758,14:5101,15:5151,16:6101,17:6102,
        18:6731,19:7101,20:7102,21:7103,22:7201,23:7202,24:7700,25:7701,
        26:7702,27:7709,28:7710,29:7745,30:7746,31:7755,32:7756,33:7757,
        34:7790,35:8703,36:8707,37:8708,38:8771,39:8772,40:8776
    }
    df = input_df.copy()
    df['Climatic_Zone'] = input_df['Soil_Type'].apply(lambda x: int(str(code[x])[0]))
    return df
# AdaBoost
cv_score, oof_preds, test_score = train_original(adaboost, climatic_zone_original)

adaboost_scores.append((
    'Climatic_Zone', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))

# Extra Trees
cv_score, oof_preds, test_score = train_original(extratrees, climatic_zone_original)

extratrees_scores.append((
    'Climatic_Zone', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))

# Bagging
cv_score, oof_preds, test_score = train_original(bagging, climatic_zone_original)

bagging_scores.append((
    'Climatic_Zone', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))

# Random Forest
cv_score, oof_preds, test_score = train_original(randomforest, climatic_zone_original)

random_scores.append((
    'Climatic_Zone', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))<jupyter_output>AdaBoostClassifier
Train Accuracy: 0.79536
Test Accuracy: 0.75743
Training Time: 3.5s

ExtraTreesClassifier
Train Accuracy: 0.88511
Test Accuracy: 0.78361
Training Time: 38.33s

BaggingClassifier
Train Accuracy: 0.85396
Test Accuracy: 0.75876
Training Time: 19.07s

RandomForestClassifier
Train Accuracy: 0.8656
Test Accuracy: 0.75154
Training Time: 38.48s<jupyter_text>Geologic Zones (Nominal)This is another feature which is based on the soil type codes, but is not ordered like climatic zone.1. alluvium2. glacial3. shale4. sandstone5. mixed sedimentary6. unspecified in the USFS ELU Survey7. igneous and metamorphic8. volcanic<jupyter_code>def geologic_zone_original(input_df):
    code = {
        1:2702,2:2703,3:2704,4:2705,5:2706,6:2717,7:3501,8:3502,9:4201,
        10:4703,11:4704,12:4744,13:4758,14:5101,15:5151,16:6101,17:6102,
        18:6731,19:7101,20:7102,21:7103,22:7201,23:7202,24:7700,25:7701,
        26:7702,27:7709,28:7710,29:7745,30:7746,31:7755,32:7756,33:7757,
        34:7790,35:8703,36:8707,37:8708,38:8771,39:8772,40:8776
    }
    df = input_df.copy()
    df['Geologic_Zone'] = input_df['Soil_Type'].apply(lambda x: int(str(code[x])[1]))
    return df
# AdaBoost
cv_score, oof_preds, test_score = train_original(adaboost, geologic_zone_original)

adaboost_scores.append((
    'Geologic_Zone', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))

# Extra Trees
cv_score, oof_preds, test_score = train_original(extratrees, geologic_zone_original)

extratrees_scores.append((
    'Geologic_Zone', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))

# Bagging
cv_score, oof_preds, test_score = train_original(bagging, geologic_zone_original)

bagging_scores.append((
    'Geologic_Zone', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))

# Random Forest
cv_score, oof_preds, test_score = train_original(randomforest, geologic_zone_original)

random_scores.append((
    'Geologic_Zone',  cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))<jupyter_output>AdaBoostClassifier
Train Accuracy: 0.79489
Test Accuracy: 0.75732
Training Time: 3.81s

ExtraTreesClassifier
Train Accuracy: 0.88551
Test Accuracy: 0.78032
Training Time: 38.91s

BaggingClassifier
Train Accuracy: 0.84946
Test Accuracy: 0.75634
Training Time: 20.04s

RandomForestClassifier
Train Accuracy: 0.86461
Test Accuracy: 0.75179
Training Time: 37.78s<jupyter_text>Surface Cover (Ordinal)According to the [USDA reference](https://www.nrcs.usda.gov/wps/portal/nrcs/detail/soils/ref/?cid=nrcs142p2_054253surface_fragments) on soil profiling:1. **(Stony/Bouldery)** — Stones or boulders cover 0.01 to less than 0.1 percent of the surface. The smallest stones are at least 8 meters apart; the smallest boulders are at least 20 meters apart (fig. 3-9).2. **(Very Stony/Very Bouldery)** — Stones or boulders cover 0.1 to less than 3 percent of the surface. The smallest stones are not less than 1 meter apart; the smallest boulders are not less than 3 meters apart (fig. 3-10).3. **(Extremely Stony/Extremely Bouldery)** — Stones or boulders cover 3 to less than 15 percent of the surface. The smallest stones are as little as 0.5 meter apart; the smallest boulders are as little as 1 meter apart (fig. 3-11).4. **(Rubbly)** — Stones or boulders cover 15 to less than 50 percent of the surface. The smallest stones are as little as 0.3 meter apart; the smallest boulders are as little as 0.5 meter apart. In most places it is possible to step from stone to stone or jump from boulder to boulder without touching the soil (fig. 3-12).5. **(Very Rubbly)** — Stones or boulders appear to be nearly continuous and cover 50 percent or more of the surface. The smallest stones are less than 0.03 meter apart; the smallest boulders are less than 0.05 meter apart. Classifiable soil is among the rock fragments, and plant growth is possible (fig. 3-13).<jupyter_code>def surface_cover_original(input_df):
    # Group IDs
    no_desc = [7,8,14,15,16,17,19,20,21,23,35]
    stony = [6,12]
    very_stony = [2,9,18,26]
    extremely_stony = [1,22,24,25,27,28,29,30,31,32,33,34,36,37,38,39,40]
    rubbly = [3,4,5,10,11,13]

    # Create dictionary
    surface_cover = {i:0 for i in no_desc}
    surface_cover.update({i:1 for i in stony})
    surface_cover.update({i:2 for i in very_stony})
    surface_cover.update({i:3 for i in extremely_stony})
    surface_cover.update({i:4 for i in rubbly})
    
    # Create Feature
    df = input_df.copy()
    df['Surface_Cover'] = input_df['Soil_Type'].apply(lambda x: surface_cover[x])
    return df
# AdaBoost
cv_score, oof_preds, test_score = train_original(adaboost, surface_cover_original)

adaboost_scores.append((
    'Surface_Cover', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))

# Extra Trees
cv_score, oof_preds, test_score = train_original(extratrees, surface_cover_original)

extratrees_scores.append((
    'Surface_Cover', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))

# Bagging
cv_score, oof_preds, test_score = train_original(bagging, surface_cover_original)

bagging_scores.append((
    'Surface_Cover', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))

# Random Forest
cv_score, oof_preds, test_score = train_original(randomforest, surface_cover_original)

random_scores.append((
    'Surface_Cover', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))<jupyter_output>AdaBoostClassifier
Train Accuracy: 0.79132
Test Accuracy: 0.75978
Training Time: 3.93s

ExtraTreesClassifier
Train Accuracy: 0.88584
Test Accuracy: 0.78266
Training Time: 40.44s

BaggingClassifier
Train Accuracy: 0.85032
Test Accuracy: 0.75947
Training Time: 23.44s

RandomForestClassifier
Train Accuracy: 0.86937
Test Accuracy: 0.75266
Training Time: 41.04s<jupyter_text>Rock Size (Ordinal)<jupyter_code>def rock_size_original(input_df):
    
    # Group IDs
    no_desc = [7,8,14,15,16,17,19,20,21,23,35]
    stones = [1,2,6,9,12,18,24,25,26,27,28,29,30,31,32,33,34,36,37,38,39,40]
    boulders = [22]
    rubble = [3,4,5,10,11,13]

    # Create dictionary
    rock_size = {i:0 for i in no_desc}
    rock_size.update({i:1 for i in stones})
    rock_size.update({i:2 for i in boulders})
    rock_size.update({i:3 for i in rubble})
    
    df = input_df.copy()
    df['Rock_Size'] = input_df['Soil_Type'].apply(lambda x: rock_size[x])
    return df
# AdaBoost
cv_score, oof_preds, test_score = train_original(adaboost, rock_size_original)

adaboost_scores.append((
    'Rock_Size', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))

# Extra Trees
cv_score, oof_preds, test_score = train_original(extratrees, rock_size_original)

extratrees_scores.append((
    'Rock_Size', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))

# Bagging
cv_score, oof_preds, test_score = train_original(bagging, rock_size_original)

bagging_scores.append((
    'Rock_Size', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))

# Random Forest
cv_score, oof_preds, test_score = train_original(randomforest, rock_size_original)

random_scores.append((
    'Rock_Size', cv_score, test_score,
     *recall_score(y_train, oof_preds, average = None)
))<jupyter_output>AdaBoostClassifier
Train Accuracy: 0.79397
Test Accuracy: 0.75969
Training Time: 3.49s

ExtraTreesClassifier
Train Accuracy: 0.88544
Test Accuracy: 0.78247
Training Time: 41.97s

BaggingClassifier
Train Accuracy: 0.85138
Test Accuracy: 0.75762
Training Time: 20.08s

RandomForestClassifier
Train Accuracy: 0.86798
Test Accuracy: 0.75305
Training Time: 33.46s<jupyter_text>SummaryAll of these features seem promising, so we won't rule any out just yet.<jupyter_code># AdaBoost
pd.DataFrame.from_records(
    data = adaboost_scores,
    columns = ['features','cv_score','holdout','recall_0', 'recall_1','recall_2','recall_3','recall_4','recall_5','recall_6']
).sort_values('cv_score')
# Extra Trees Classifier
pd.DataFrame.from_records(
    data = extratrees_scores,
    columns = ['features','cv_score','holdout','recall_0', 'recall_1','recall_2','recall_3','recall_4','recall_5','recall_6']
).sort_values('cv_score')
# Bagging Classifier
pd.DataFrame.from_records(
    data = bagging_scores,
    columns = ['features','cv_score','holdout','recall_0', 'recall_1','recall_2','recall_3','recall_4','recall_5','recall_6']
).sort_values('cv_score')
# Random Forest
pd.DataFrame.from_records(
    data = random_scores,
    columns = ['features','cv_score','holdout','recall_0', 'recall_1','recall_2','recall_3','recall_4','recall_5','recall_6']
).sort_values('cv_score')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10828402366863905, 'fraction_numerical': 0.06497041420118344, 'mean_word_length': 5.1124773960217, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '497275', 'license': 'MIT', 'n_tokens_mistral': 6906, 'n_tokens_neox': 6100, 'n_words': 1381, 'path': 'notebooks/Notebook 3a - Soil Feature Engineering.ipynb', 'repo_name': 'rsizem2/forest-cover-type-prediction', 'size': '16900'}