<jupyter_start><jupyter_text>This notebook performs the data preparation for the common classifier dataset<jupyter_code>import pandas as pd
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
# This is a kaggle dataset - https://www.kaggle.com/jsphyg/weather-dataset-rattle-package
df = pd.read_csv('../../data/weatherAUS.csv')
# print(df)
df.shape
df.head(10)
(df.isnull().sum()/len(df))*100
# these columns have large numbers of missing values. We'll drop those
df.drop(columns=['Evaporation','Sunshine','Cloud9am','Cloud3pm'], inplace=True)
# these are important columns, if any empty values, the rows are dropped
df.dropna(subset=['RainToday', 'RainTomorrow'], inplace=True)
# this is dropped as well, we don't want the one-hot encoded data
df.drop(columns=['Location'], inplace=True)
# Fill in numeric columns with missing values with the mean
def fill_missing_numeric_values(data):
    for col in data.select_dtypes(['int','float']):
        data[col] = data[col].fillna(data[col].mean())
    return data

# for other datatypes, just forward fill
def fill_missing_object_values(data):
    for col in data.select_dtypes(['object']):
        data[col] = data[col].fillna(method='ffill')
    return data

df = fill_missing_numeric_values(df)
df = fill_missing_object_values(df)
# the month is an intersting data point, we'll keep that, and drop the date
df['Date'] = pd.to_datetime(df['Date'])
month = df['Date'].dt.month
df.drop(columns=['Date'], inplace=True)
df.insert(loc=0, column='Month', value=month)
# One-hot encode these categorical features, not required for a decision tree model
df = pd.get_dummies(df, columns = ['WindGustDir', 'WindDir9am', 'WindDir3pm'])
# Scale these features
col_names = ['Month', 'MinTemp', 'MaxTemp', 'Rainfall', 'WindGustSpeed', 'WindSpeed9am', 'WindSpeed3pm', 'Humidity9am', 'Humidity3pm', 'Pressure9am', 'Pressure3pm', 'Temp9am', 'Temp3pm']
scaled_features = df[col_names]
# scaler = StandardScaler().fit(scaled_features.values)
scaler = MinMaxScaler().fit(scaled_features.values)
scaled_features = scaler.transform(scaled_features.values)
df[col_names] = scaled_features
# change Yes/No to 1/0, again, not required for a decision tree model
def replace_with_zero_or_one(x):
    if x[0] == 'No':
        return 0
    else:
        return 1
    
# df['RainTomorrow'] = df[['RainTomorrow']].apply(replace_with_zero_or_one, axis=1)
df['RainToday'] = df[['RainToday']].apply(replace_with_zero_or_one, axis=1)
# for decision tree model, this can be skipped
df['RainTomorrowN'] = df['RainTomorrow']
df.drop(columns=['RainTomorrow'], inplace=True)
df.head(10)
df.shape
df.columns
print(len(df.columns))
df.to_csv('../../data/cleanedWeatherAUS.csv', index=False, header=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1414721723518851, 'fraction_numerical': 0.00969479353680431, 'mean_word_length': 6.053164556962026, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '67397', 'license': 'Apache-2.0', 'n_tokens_mistral': 964, 'n_tokens_neox': 921, 'n_words': 231, 'path': 'notebooks/classifier/scikit-learn Classifier - Data Cleanup.ipynb', 'repo_name': 'geoffreydstewart/JavaVsPythonMLlibs', 'size': '2785'}