<jupyter_start><jupyter_text>Weather_predictionUsing Decision Tree Classifier to predict the weather conditions of Delhi in the upcoming days.  Gathering dataset and importing necessary modules<jupyter_code>import numpy as np 
import pandas as pd 
import matplotlib.pyplot as plt
import seaborn as sns
import graphviz
import warnings
warnings.filterwarnings('ignore')
df = pd.read_csv("testset.csv")
df.head()<jupyter_output><empty_output><jupyter_text>Data Cleaning and Data Visualization<jupyter_code>df.columns
df.columns = map(lambda x: x.strip(), df.columns)
df.columns
df._conds.value_counts(ascending=False)
plt.figure(figsize=(15, 10));
df._conds.value_counts().head(10).plot(kind='bar');
plt.title("Top 10 most common weather condition")
plt.plot();
plt.figure(figsize=(15, 10));
df._conds.value_counts(ascending=True).head(10).plot(kind="bar");
plt.title("Top 10 least whether condition in delhi");
plt.plot();
df._wdire.value_counts()
plt.figure(figsize=(15, 10));
plt.title("Common wind direction in delhi");
df._wdire.value_counts().plot(kind="bar");
plt.plot();
print("average temperature in delhi:", round(df._tempm.mean(axis=0),2))
def extract_year(value):
    return (value[0:4])
df.head()
def extract_month(value):
    return (value[4:6])
df["year"] = df["datetime_utc"].apply(lambda x:extract_year(x))
df["month"] = df["datetime_utc"].apply(lambda x:extract_month(x))
df.head()
print("max, min: ", df.year.max(), ",", df.year.min())
df.year.value_counts()
df.groupby("year")._tempm.mean()
df_mean = df.groupby("year")._tempm.mean().reset_index().sort_values('_tempm', ascending=True)
df_mean.dtypes
df_mean.year = df_mean.year.astype("float")
df_mean.dtypes
df_mean.plot(kind="scatter", x="year", y="_tempm", figsize=(15, 10))

plt.xticks(df_mean.year);
plt.title("Average temprature change");
plt.plot();
df.isnull().sum()
df.columns
df_filtered = df[['datetime_utc', '_conds', '_dewptm', '_fog', '_hail',
       '_hum', '_pressurem', '_rain', '_snow', '_tempm',
       '_thunder', '_tornado', '_vism', '_wdird', '_wdire'
       , '_wspdm', 'year', "month"]]
df_filtered[df_filtered._dewptm.isnull()]
for index,row in df_filtered[df_filtered._dewptm.isnull()].iterrows():
    mean_val = df_filtered[df_filtered["year"] == row["year"]]._dewptm.mean()
    df_filtered.at[index, "_dewptm"] = mean_val
df_filtered[df_filtered._dewptm.isnull()]
df_filtered.shape
df_filtered.isnull().sum()
df_filtered[df_filtered._hum.isnull()]
for index,row in df_filtered[df_filtered._hum.isnull()].iterrows():
    mean_val = df_filtered[df_filtered["year"] == row["year"]]._hum.mean()
    df_filtered.at[index, "_hum"] = mean_val
df_filtered[df_filtered._hum.isnull()]
df_filtered.isnull().sum()
df_filtered.head()
df_filtered._pressurem.replace(-9999.0, np.nan, inplace=True)
df_filtered.head() 
df_filtered._pressurem.isnull().sum()
for index,row in df_filtered[df_filtered._pressurem.isnull()].iterrows():
    mean_val = df_filtered[df_filtered["year"] == row["year"]]._pressurem.mean()
    df_filtered.at[index, "_pressurem"] = mean_val
df_filtered.isnull().sum()
for index,row in df_filtered[df_filtered._tempm.isnull()].iterrows():
    mean_val = df_filtered[df_filtered["year"] == row["year"]]._tempm.mean()
    df_filtered.at[index, "_tempm"] = mean_val
for index,row in df_filtered[df_filtered._vism.isnull()].iterrows():
    mean_val = df_filtered[df_filtered["year"] == row["year"]]._vism.mean()
    df_filtered.at[index, "_vism"] = mean_val
for index,row in df_filtered[df_filtered._wdird.isnull()].iterrows():
    mean_val = df_filtered[df_filtered["year"] == row["year"]]._wdird.mean()
    df_filtered.at[index, "_wdird"] = mean_val
for index,row in df_filtered[df_filtered._wspdm.isnull()].iterrows():
    mean_val = df_filtered[df_filtered["year"] == row["year"]]._wspdm.mean()
    df_filtered.at[index, "_wspdm"] = mean_val
df_filtered.isnull().sum()
for index,row in df_filtered[df_filtered._wdire.isnull()].iterrows():
    most_frequent = df_filtered[df_filtered["year"] == row["year"]]._wdire.value_counts().idxmax()
    df_filtered.at[index, "_wdire"] = most_frequent
df_filtered.isnull().sum()
for index,row in df_filtered[df_filtered._conds.isnull()].iterrows():
    most_frequent = df_filtered[df_filtered["year"] == row["year"]]._conds.value_counts().idxmax()
    df_filtered.at[index, "_conds"] = most_frequent
df_filtered.isnull().sum()
df_filtered.year = df_filtered.year.astype("object")
df_filtered.month = df_filtered.month.astype("object")
pd.crosstab(df_filtered.year, [df_filtered.month], values=df_filtered._tempm, aggfunc="mean")
plt.figure(figsize=(15, 10));
sns.heatmap(pd.crosstab(df_filtered.year, [df_filtered.month], values=df_filtered._tempm, aggfunc="mean"),
            cmap="coolwarm", annot=True, cbar=True);
plt.title("Average Temprature 1996-2016")
plt.plot();
df_filtered._conds.value_counts()
df_filtered
df_filtered.isnull().values.any()
df_filtered.to_csv("cleaned_weather_dataset.csv")<jupyter_output><empty_output><jupyter_text>LabelEncoding and OneHotEncoding<jupyter_code>from sklearn.preprocessing import LabelEncoder, OneHotEncoder
df_filtered.columns
feature_columns = ['_wdire', '_dewptm', '_fog', '_hail', '_hum',
       '_pressurem', '_rain', '_snow', '_tempm', '_thunder', '_tornado',
       '_vism', '_wdird', '_wspdm', 'year', 'month', '_conds']
df_final = df_filtered[feature_columns]
df_final.head()
df_final.dtypes
df_final._wdire.value_counts()
wdire_dummies = pd.get_dummies(df_final["_wdire"])
df_final = pd.concat([wdire_dummies, df_final], axis=1)
df_final.head()
df_final.columns
df_final.drop("_wdire", inplace=True, axis=1)
X = df_final.iloc[:, 0:-1].values
X.shape
y = df_final.iloc[:, -1].values
label_encoder = LabelEncoder()
y = label_encoder.fit_transform(y)
y.shape<jupyter_output><empty_output><jupyter_text>Splitting of dataset into train and test data<jupyter_code>from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.25, random_state=40)
X_train.shape
X_test.shape<jupyter_output><empty_output><jupyter_text>DecisionTreeClassifier Model for prediction<jupyter_code>from sklearn.tree import DecisionTreeClassifier
clf = DecisionTreeClassifier(criterion="entropy", random_state=40)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
y_pred
from sklearn import metrics
import matplotlib.pyplot as plt
print("DecisionTrees's Accuracy: ", metrics.accuracy_score(y_test, y_pred))<jupyter_output>DecisionTrees's Accuracy:  0.7863197084917617
{'fraction_non_alphanumeric': 0.1642549261083744, 'fraction_numerical': 0.011545566502463054, 'mean_word_length': 7.34017971758665, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '313862', 'license': 'MIT', 'n_tokens_mistral': 2536, 'n_tokens_neox': 2365, 'n_words': 384, 'path': 'Weather_prediction/Weather_Prediction.ipynb', 'repo_name': 'rishusiva/MLH_WeatherDataPrediction', 'size': '6496'}