<jupyter_start><jupyter_text>Exploratory Data Analysis<jupyter_code>#load the libraries
import pandas as pd
import numpy as np
import sweetviz as sv
data1 = pd.read_csv("data_clean.csv")
data1.tail(10)
data1
#Data Structure 
type(data1)
data1.shape
#data types
data1.dtypes<jupyter_output><empty_output><jupyter_text>Data type conversion<jupyter_code>data1.info()
data2=data1.iloc[:,1:]
#The method .copy() is used here so that any changes made in new DataFrame don't get reflected in the original one
data=data2.copy()
data['Month']=pd.to_numeric(data['Month'],errors='coerce')
data['Temp C']=pd.to_numeric(data['Temp C'],errors='coerce')# coerce will introduce NA values for non numeric data in the columns
data['Weather']=data['Weather'].astype('category')           #data['Wind']=data['Wind'].astype('int64')
data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 158 entries, 0 to 157
Data columns (total 9 columns):
 #   Column   Non-Null Count  Dtype   
---  ------   --------------  -----   
 0   Ozone    120 non-null    float64 
 1   Solar.R  151 non-null    float64 
 2   Wind     158 non-null    float64 
 3   Temp C   157 non-null    float64 
 4   Month    157 non-null    float64 
 5   Day      158 non-null    int64   
 6   Year     158 non-null    int64   
 7   Temp     158 non-null    int64   
 8   Weather  155 non-null    category
dtypes: category(1), float64(5), int64(3)
memory usage: 10.3 KB<jupyter_text>Duplicates<jupyter_code>#Count of duplicated rows
data[data.duplicated()].shape
data
#Print the duplicated rows
data[data.duplicated()]
data_cleaned1=data.drop_duplicates()
data_cleaned1<jupyter_output><empty_output><jupyter_text>Drop columns<jupyter_code>data_cleaned2=data_cleaned1.drop('Temp C',axis=1)
data_cleaned2<jupyter_output><empty_output><jupyter_text>Rename the columns<jupyter_code>#rename the Solar column
data_cleaned3 = data_cleaned2.rename({'Solar.R': 'Solar'}, axis=1)
data_cleaned3<jupyter_output><empty_output><jupyter_text>Outlier Detection<jupyter_code># histogram of Ozone
data_cleaned3['Ozone'].hist()
#Box plot
data_cleaned3.boxplot(column=['Ozone'])
#Descriptive stat
data_cleaned3['Ozone'].describe()
data_cleaned3
#Bar plot
data['Weather'].value_counts().plot.bar()<jupyter_output><empty_output><jupyter_text>Missing Values and Imputation<jupyter_code>data_cleaned3.info() #1St method tofina na value
data_cleaned3.isnull().sum()
import seaborn as sns
cols = data_cleaned3.columns 
colours = ['#000099', '#ffff00'] # specify the colours - yellow is missing. blue is not missing.
sns.heatmap(data_cleaned3[cols].isnull(),
            cmap=sns.color_palette(colours))               #2nd method to find na value.
data_cleaned3[data_cleaned3.isnull().any(axis=1)].head()
data_cleaned3.isnull().sum()
#Mean Imputation
mean = data_cleaned3['Ozone'].mean()
print(mean)
data_cleaned3['Ozone'] = data_cleaned3['Ozone'].fillna(mean)
data_cleaned3
#Missing value imputation for categorical vlaue
#Get the object columns
obj_columns=data_cleaned3[['Weather']]
obj_columns.isnull().sum()
#Missing value imputation for categorical vlaue
obj_columns=obj_columns.fillna(obj_columns.mode().iloc[0])
data_cleaned3.shape
obj_columns.shape
#Join the data set with imputed object dataset
data_cleaned4=pd.concat([data_cleaned3,obj_columns],axis=1)
data_cleaned4<jupyter_output><empty_output><jupyter_text>Scatter plot and Correlation analysis<jupyter_code># Seaborn visualization library
import seaborn as sns
# Create the default pairplot
sns.pairplot(data_cleaned3)
#Correlation

data_cleaned3.corr()<jupyter_output><empty_output><jupyter_text>Transformations  Dummy Variable<jupyter_code>#Creating dummy variable for Weather column
data_cleaned4=pd.get_dummies(data,columns=['Weather'])
data_cleaned4
data_cleaned4=data_cleaned4.dropna()
#Normalization of the data
from numpy import set_printoptions
from sklearn.preprocessing import MinMaxScaler
data_cleaned4.values

array = data_cleaned3.values

scaler = MinMaxScaler(feature_range=(0,1))
rescaledX = scaler.fit_transform(array[:,0:5])

# summarize transformed data
set_printoptions(precision=2)
print(rescaledX[0:5,:])

# Standardize data (0 mean, 1 stdev)
from sklearn.preprocessing import StandardScaler
array = data_cleaned4.values
scaler = StandardScaler().fit(array)
rescaledX = scaler.transform(array)

# summarize transformed data
set_printoptions(precision=2)
print(rescaledX[0:5,:])<jupyter_output>[[-0.02  0.05 -0.71 -1.15 -1.53 -1.7   0.   -1.15 -0.64 -0.68  1.28]
 [-0.17 -0.75 -0.54 -0.62 -1.53 -1.59  0.   -0.62  1.57 -0.68 -0.78]
 [-0.9  -0.41  0.77 -0.4  -1.53 -1.48  0.   -0.4  -0.64  1.47 -0.78]
 [-0.72  1.43  0.45 -1.69 -1.53 -1.36  0.   -1.69 -0.64 -0.68  1.28]
 [-0.57  1.27 -0.37 -1.37 -1.53 -1.02  0.   -1.37 -0.64  1.47 -0.78]]<jupyter_text>Speed up the EDA process<jupyter_code>sweet_report = sv.analyze(data)
sweet_report.show_html('weather_report.html')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1299492385786802, 'fraction_numerical': 0.06091370558375635, 'mean_word_length': 5.073982737361282, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '424550', 'license': 'CC0-1.0', 'n_tokens_mistral': 1985, 'n_tokens_neox': 1841, 'n_words': 392, 'path': 'EDA1.ipynb', 'repo_name': 'swaapnil27/Exploratory-Data-Analysis', 'size': '4925'}