<jupyter_start><jupyter_text>Mean and Guessing ModelsLet's build some models that just guess the mean and use smart/weighted guesses on the test set.<jupyter_code>import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import pandas as pd
import pickle
import lightgbm as lgb

from pathlib import Path
from lightgbm import LGBMClassifier
from pprint import pprint

from sklearn.metrics import mean_squared_error, f1_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import KFold, train_test_split, RandomizedSearchCV 
from sklearn.model_selection import cross_val_score, StratifiedKFold

############ USE FOR GOOGLE COLAB ############
# DATA_DIR = Path('/content/drive/MyDrive/Work/Delivery/Current/Earthquake_damage/data')
# SUBMISSIONS_DIR = Path('drive/MyDrive/Work/Delivery/Current/Earthquake_damage/submissions')
# MODEL_DIR = Path('/content/drive/MyDrive/Work/Delivery/Current/Earthquake_damage/models')

# from google.colab import drive
# drive.mount('/content/drive')
#############################################


### USE FOR LOCAL JUPYTER NOTEBOOKS ###
DATA_DIR = Path('data')
SUBMISSIONS_DIR = Path('submissions')
MODEL_DIR = Path('models')
#######################################

# The code runs the same if working on Jupyter or Colab, just need to change the 
# dirs above

X = pd.read_csv(DATA_DIR / 'train_values.csv', index_col='building_id')

categorical_columns = X.select_dtypes(include='object').columns
bool_columns = [col for col in X.columns if col.startswith('has')]
X[categorical_columns] = X[categorical_columns].astype('category')
X[bool_columns] = X[bool_columns].astype('bool')

X = pd.get_dummies(X)
y = pd.read_csv(DATA_DIR / 'train_labels.csv', index_col='building_id')
sns.set()
submission_format = pd.read_csv(DATA_DIR / 'submission_format.csv',
                                index_col='building_id')

test_values = pd.read_csv(DATA_DIR / 'test_values.csv', index_col='building_id')
def submission():
    submission_format = pd.read_csv(DATA_DIR / 'submission_format.csv',
                                    index_col='building_id')

    my_submission = pd.DataFrame(data=predictions,
                                columns=submission_format.columns,
                                index=submission_format.index)
    
    my_submission.to_csv(SUBMISSIONS_DIR / f'{title}.csv')
# Just guessing 2
all_twos = np.ones(len(submission_format), dtype=np.int8) + 1

df_all_twos = pd.DataFrame(data=all_twos,
                           columns=submission_format.columns,
                           index=submission_format.index)

df_all_twos.to_csv(SUBMISSIONS_DIR / 'all_twos.csv')<jupyter_output><empty_output><jupyter_text>Weighted GuessingWe can see taht we have 10% of houses being damage grade 1, 57% being grade 2 and 33% are grade 3. So, let's create a classifier that guesses these options this percentage of the time.<jupyter_code>y.value_counts(sort=False, normalize=True)
len(submission_format)
# Weighted guessing
num_1 = int(len(submission_format) * 0.1)
# Add extra 2 as this is the modal class
num_2 = int(len(submission_format) * 0.57) + 2 
num_3 = int(len(submission_format) * 0.33)
num_1 + num_2 + num_3 == len(submission_format)
num_1 + num_2 + num_3
num_1
weighted_guess = [1 for _ in range(num_1)] + \
                 [2 for _ in range(num_2)] + \
                 [3 for _ in range(num_3)]
np.random.shuffle(weighted_guess)
df_weighted_guess = pd.DataFrame(data=weighted_guess,
                                columns=submission_format.columns,
                                index=submission_format.index)

df_weighted_guess.to_csv(SUBMISSIONS_DIR / 'weighted_guess.csv')<jupyter_output><empty_output><jupyter_text>Calculating F1 Score on X<jupyter_code>ALL_TWO_SUBMISSION_SCORE = 0.5670

all_twos = [2 for _ in range(len(X))]
all_two_score = f1_score(y, all_twos, average='micro')
print('F1 score on X:         ', all_two_score)
print('F1 score on submission:', ALL_TWO_SUBMISSION_SCORE)
def make_weighted_guess(data):
    num_1 = int(len(data) * 0.1)
    num_2 = int(len(data) * 0.57)
    num_3 = int(len(data) * 0.33)
    
    # Above will not be perfectly equal to len(data), so
    # add the difference to modal class 2
    diff = len(data) - num_1 - num_2 - num_3
    num_2 += diff
    
    assert len(data) == num_1 + num_2 + num_3
    
    weighted_guess = [1 for _ in range(num_1)] + \
                     [2 for _ in range(num_2)] + \
                     [3 for _ in range(num_3)]
    
    np.random.shuffle(weighted_guess)
    
    return weighted_guess
WEIGHTED_GUESS_SUBMISSION_SCORE = 0.4441

weighted_pred = make_weighted_guess(X)
weighted_score = f1_score(y, weighted_pred, average='micro')
print('F1 score on X:         ', weighted_score)
print('F1 score on submission:', WEIGHTED_GUESS_SUBMISSION_SCORE)<jupyter_output>F1 score on X:          0.44396222577810524
F1 score on submission: 0.4441<jupyter_text>Calculating F1 Score on X_Val<jupyter_code>from sklearn.model_selection import train_test_split

X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.20,
                                                 random_state=42, stratify=y)
all_twos_X_val = [2 for _ in range(len(X_val))]
all_two_X_val_score = f1_score(y_val, all_twos_X_val, average='micro')
print('F1 score on X:         ', all_two_X_val_score)
print('F1 score on submission:', ALL_TWO_SUBMISSION_SCORE)
weighted_pred_X_val = make_weighted_guess(X_val)
weighted_score_X_val = f1_score(y_val, weighted_pred_X_val, average='micro')
print('F1 score on X:         ', weighted_score_X_val)
print('F1 score on submission:', WEIGHTED_GUESS_SUBMISSION_SCORE)<jupyter_output>F1 score on X:          0.4440244814949828
F1 score on submission: 0.4441
{'fraction_non_alphanumeric': 0.11383658662981516, 'fraction_numerical': 0.02487476248056659, 'mean_word_length': 3.927659574468085, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '340336', 'license': 'MIT', 'n_tokens_mistral': 2104, 'n_tokens_neox': 1986, 'n_words': 432, 'path': 'analysis/4_baseline_models.ipynb', 'repo_name': 'theadammurphy/drivendata_earthquake_damage_competition', 'size': '5789'}