<jupyter_start><jupyter_text>Data Cleaning<jupyter_code>df.dtypes
df.columns
# Select tornado data for desired years
filtered_df = df.loc[df.yr >= 1995, :]
filtered_df
# Select derired columns
df1 = filtered_df[['yr', 'mo', 'st', 'mag', 'inj', 'fat',\
                   'loss', 'closs', 'len', 'wid', 'slat', 'slon']]
df1
df1.reset_index(inplace = True, drop = True)
df1
# States Data
data = {'StName':['Alabama',
'Alaska',
'Arizona',
'Arkansas',
'California',
'Colorado',
'Connecticut',
'Delaware',
'Florida',
'Georgia',
'Hawaii',
'Idaho',
'Illinois',
'Indiana',
'Iowa',
'Kansas',
'Kentucky',
'Louisiana',
'Maine',
'Maryland',
'Massachusetts',
'Michigan',
'Minnesota',
'Mississippi',
'Missouri',
'Montana',
'Nebraska',
'Nevada',
'New Hampshire',
'New Jersey',
'New Mexico',
'New York',
'North Carolina',
'North Dakota',
'Ohio',
'Oklahoma',
'Oregon',
'Pennsylvania',
'Rhode Island',
'South Carolina',
'South Dakota',
'Tennessee',
'Texas',
'Utah',
'Vermont',
'Virginia',
'Washington',
'West Virginia',
'Wisconsin',
'Wyoming']}

states_df = pd.DataFrame(data)
cw_location = 'http://app02.clerk.org/menu/ccis/Help/CCIS%20Codes/'
cw_filename = 'state_codes.html'

states = pd.read_html(cw_location + cw_filename)[0]
state_code_map = dict(zip(states['Description'], 
                          states['Code']))
states_df['StAbbr'] = states_df['StName'].map(state_code_map)
code_state_map = dict(zip(states['Code'],
                          states['Description']))
states_df['StNameAgain'] = states_df['StAbbr'].map(code_state_map)
states_df = states_df.rename(columns={'stName': 'State_Name', 'StAbbr': 'st'})
states_df.head()
states_df.columns
# Merge dataframes
new_df2 = pd.merge(df1, states_df, on='st', how='left')
# new_df2
df3 = new_df2[[
'yr',
'mo',
'st',
'mag',
'inj',
'fat',
'loss',
'closs',
'len',
'wid',
'slat',
'slon',
'StName']]
df3
df_clean = df3.copy()
df_clean
import seaborn as sns
sns.countplot(df_clean["mag"], label="Count")
plt.show()
# Select the rows where magnitude >= 0
df_magfilt = df_clean[df_clean['mag'] >= 0]
df_magfilt
sns.countplot(df_magfilt["mag"], label="Count")
plt.show()
df_magfilt['adjusted magnitude'] = df_magfilt['mag']
df_magfilt = df_magfilt.replace({'adjusted magnitude': {3: 2, 4: 2, 5:2}})
df_magfilt
df_magfilt["adjusted magnitude"].value_counts()
df_magfilt["mag"].max()
df_magfilt.columns<jupyter_output><empty_output><jupyter_text>Machine Learning<jupyter_code>cleaned = df_magfilt[["yr", "mo", "fat", "loss", "len", "wid", "slat", "slon", "StName", "mag"]]
cleaned
# data is not balanced
from pandas.plotting import scatter_matrix 

scatter_matrix(cleaned.drop('mag', axis=1), figsize=(10,5))
plt.show()<jupyter_output><empty_output><jupyter_text>Predict Tornado Category (magnitude)<jupyter_code># Choose our features
X = cleaned[["fat", "len", "wid", "slat", "slon"]]
X
# Choose our label
y = cleaned['mag']
y<jupyter_output><empty_output><jupyter_text>Preprocessing  Create training and testing sets<jupyter_code>from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)<jupyter_output><empty_output><jupyter_text>Output CSV file for weights for app<jupyter_code># keep columns used as predictors of cleaned dataframe

train_output = X_train[["fat", "len", "wid", "slat", "slon"]]
train_output.head()
# output to csv

train_output.to_csv(r'cleaned.csv',index=False)<jupyter_output><empty_output><jupyter_text>Apply Scaler<jupyter_code>from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)<jupyter_output><empty_output><jupyter_text>Build Models<jupyter_code>## Logistic Regression
from sklearn.linear_model import LogisticRegression
logreg = LogisticRegression()
logreg.fit(X_train_scaled, y_train)

print('Accuracy of Logistic Regression on training data', logreg.score(X_train_scaled, y_train))
print('Accuracy of Logistic Regression on testing data', logreg.score(X_test_scaled, y_test))
## Decision Tree
from sklearn.tree import DecisionTreeClassifier
dt = DecisionTreeClassifier()
dt.fit(X_train_scaled, y_train)

print('Accuracy of Decision Tree on training', dt.score(X_train_scaled, y_train))
print('Accuracy of Decision Tree on testing', dt.score(X_test_scaled, y_test))
# Setting max decision tree depth to help avoid overfitting
dt2 = DecisionTreeClassifier(max_depth=5)
dt2.fit(X_train_scaled, y_train)
print('Accuracy of Decision tree on training', dt2.score(X_train_scaled, y_train))
print('Accuracy of Decision tree on testing', dt2.score(X_test_scaled, y_test))
# K-nearest neighbor 
from sklearn.neighbors import KNeighborsClassifier
knn = KNeighborsClassifier()
knn.fit(X_train_scaled, y_train)
print('Accuracy of KNN on training', knn.score(X_train_scaled, y_train))
print('Accuracy of KNN on testing', knn.score(X_test_scaled, y_test))
# Linear Discriminant Analysis 
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
lda = LinearDiscriminantAnalysis()
lda.fit(X_train_scaled,y_train)
print('Accuracy of Linear Discriminant Analysis on training', lda.score(X_train_scaled, y_train))
print('Accuracy of Linear Discriminant Analysis on testing', lda.score(X_test_scaled, y_test))
# Gaussian Naive Bayes 
from sklearn.naive_bayes import GaussianNB
gnb = GaussianNB()
gnb.fit(X_train_scaled, y_train)
print('Accuracy of GNB on training', gnb.score(X_train_scaled, y_train))
print('Accuracy of GNB on testing', gnb.score(X_test_scaled, y_test))
# Support Vector Machine
from sklearn.svm import SVC
svm = SVC()
svm.fit(X_train_scaled, y_train)
print('Accuracy of SVM on training', svm.score(X_train_scaled, y_train))
print('Accuracy of SVM on testing', svm.score(X_test_scaled, y_test))<jupyter_output>Accuracy of SVM on training 0.6848755583918316
Accuracy of SVM on testing 0.6918953414167198<jupyter_text>Save and load the best model<jupyter_code>import pickle
pickle.dump(dt2, open('model.pkl','wb'))
# Loading model to compare the results
model = pickle.load( open('model.pkl','rb'))<jupyter_output><empty_output><jupyter_text>Make Prediction on tornado category (magnitude)<jupyter_code>user_input = [[50, 100, 900, 31.51, -93.92,]]
inputdf = pd.DataFrame(user_input,columns=["fat", "len", "wid", "slat", "slon"])
original_df = X.copy()
reference = original_df.append(inputdf).reset_index()
new_df = reference[["fat", "len", "wid", "slat", "slon"]]
scaler = MinMaxScaler()
scaled_df = scaler.fit_transform(new_df)
[list(scaled_df[len(scaled_df)-1])]
# scaled_df
model.predict([list(scaled_df[len(scaled_df)-1])])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14906174334140435, 'fraction_numerical': 0.013771186440677966, 'mean_word_length': 7.63921568627451, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '395272', 'license': 'MIT', 'n_tokens_mistral': 2605, 'n_tokens_neox': 2451, 'n_words': 527, 'path': 'Notebooks/tornado_prediction_ML.ipynb', 'repo_name': 'rstrong341/Tornado_Report', 'size': '6608'}