<jupyter_start><jupyter_text>HTRIA - Notebook<jupyter_code>import numpy as np
import pandas as pd
import seaborn as sns
import datetime as dt
import matplotlib.pyplot as plt
sns.set_palette('magma_r')
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Loading Dataset : Available [Here](https://www.kaggle.com/jsphyg/weather-dataset-rattle-package)<jupyter_code>weather_data = pd.read_csv('../data/weatherAUS.csv')
weather_data.tail()<jupyter_output><empty_output><jupyter_text>EDA and Data-Processing :<jupyter_code>weather_data.select_dtypes('object').info()
weather_data.select_dtypes(np.number).info()
weather_data.select_dtypes(np.number).isna().sum()
weather_data.select_dtypes('object').isna().sum()
weather_data.shape
weather_data['RainTomorrow'].isna().sum()
weather_data = weather_data[weather_data['RainTomorrow'].notna()]
weather_data.shape
weather_data[weather_data['Date'] == '2017-01-01'].head()
weather_data[weather_data['Location'] == 'MountGambier'].head()
num_cols = weather_data.select_dtypes(np.number).columns.tolist()
cat_cols = weather_data.select_dtypes('object').columns.tolist()
weather_data[num_cols].describe()
weather_data['RainTomorrow'].value_counts()/len(weather_data)
weather_data[cat_cols].head()
weather_data['Date'] = weather_data['Date'].apply(pd.to_datetime)
weather_data['Year'] = weather_data['Date'].dt.year
weather_data['Month'] = weather_data['Date'].dt.month
weather_data['Day'] = weather_data['Date'].dt.day
weather_data.drop('Date', axis=1, inplace=True)
for col in num_cols:
    col_median = weather_data[col].median()
    weather_data[col].fillna(col_median, inplace=True)

weather_data[num_cols].describe()
cat_cols.remove('Date')
weather_data[cat_cols].isna().sum()
for col in cat_cols:
    weather_data[col].fillna(weather_data[col].mode()[0], inplace=True)

weather_data[cat_cols].describe()
weather_data.shape
cols_to_keep =[
    'Location',
    'Rainfall',
    'Evaporation',
    'Sunshine',
    'WindGustDir',
    'WindGustSpeed',
    'WindDir3pm',
    'AvgWindSpeed',
    'Humidity3pm',
    'Cloud3pm',
    'RainToday',
    'Pressure3pm',
    'TempDiff',
]
target_col = ['RainTomorrow']
weather_data['TempDiff'] = weather_data['MaxTemp'] - weather_data['MinTemp']
weather_data['AvgWindSpeed'] = (weather_data['WindSpeed9am'] + weather_data['WindSpeed3pm']) / 2<jupyter_output><empty_output><jupyter_text>Preparing dataset for predictions :<jupyter_code>from sklearn.model_selection import train_test_split

X = weather_data[cols_to_keep]
y = weather_data[target_col]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 1)
print(X_train.shape)
num_cols = X.select_dtypes(np.number).columns.tolist()
cat_cols = X.select_dtypes('object').columns.tolist()<jupyter_output><empty_output><jupyter_text>Working with Pipelines :<jupyter_code>from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import ColumnTransformer

num_pipeline = Pipeline([
    ('imputer', SimpleImputer(strategy='median')),
    ('std_scaler', StandardScaler()),
])

full_pipeline = ColumnTransformer([
    ('num', num_pipeline, num_cols),
    ('cat', OneHotEncoder(), cat_cols),
])
piped_X_train = full_pipeline.fit_transform(X_train)
piped_X_test = full_pipeline.fit_transform(X_test)
print(piped_X_train.shape)
print(piped_X_test.shape)<jupyter_output>(99535, 92)
(42658, 92)<jupyter_text>Defining model and Testing :<jupyter_code>from sklearn.linear_model import LogisticRegression
log_reg = LogisticRegression(solver='liblinear', random_state=42)
log_reg.fit(piped_X_train, y_train.values.ravel())
from sklearn.metrics import accuracy_score

y_pred_train = log_reg.predict(piped_X_train)
print('Train:', accuracy_score(y_train, y_pred_train))

y_pred_test = log_reg.predict(piped_X_test)
print('Test:', accuracy_score(y_test, y_pred_test))
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score
from sklearn.metrics import precision_score

print('Recall :', recall_score(y_test, y_pred_test, average="binary", pos_label="No"))
print('Precision:', precision_score(y_test, y_pred_test, average="binary", pos_label="No"))
print('F1_score:', f1_score(y_test, y_pred_test, average="binary", pos_label="No"))<jupyter_output>Recall : 0.9439029542231208
Precision: 0.8698553948832035
F1_score: 0.9053676608923428<jupyter_text>Saving Model and Pipeline for Streamlit :<jupyter_code>import pickle

with open('../pages/pickles/final_model.sav', 'wb') as f:
    pickle.dump(log_reg, f)

with open('../pages/pickles/full_pipeline.sav', 'wb') as f:
    pickle.dump(full_pipeline, f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13900021092596498, 'fraction_numerical': 0.019194262813752375, 'mean_word_length': 8.279843444227005, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '178406', 'license': 'MIT', 'n_tokens_mistral': 1819, 'n_tokens_neox': 1700, 'n_words': 254, 'path': 'model/notebook.ipynb', 'repo_name': 'piyushmohan01/HTRIA', 'size': '4741'}