<jupyter_start><jupyter_text>Data Wrangling in Python<jupyter_code>import pandas as pd
# Reading the Dataset (csv)
df = pd.read_csv("https://s3-api.us-geo.objectstorage.softlayer.net/cf-courses-data/CognitiveClass/DA0101EN/auto.csv", header=None)
df.head()
# Add headers to the columns
header = ["symboling","normalized-losses","make","fuel-type","aspiration", "num-of-doors","body-style",
         "drive-wheels","engine-location","wheel-base", "length","width","height","curb-weight","engine-type",
         "num-of-cylinders", "engine-size","fuel-system","bore","stroke","compression-ratio","horsepower",
         "peak-rpm","city-mpg","highway-mpg","price"]

df.columns = header
df.head()<jupyter_output><empty_output><jupyter_text>Dealing with the missing values<jupyter_code># All the ? are representing the missing values in the data that should be taken care of
# First step would be to convert the ? to Python's recognized NaN values to allow us to use the built-in Python methods
# Lets replace the ? to NaN in the Dataset
import numpy as np
df.replace("?",np.nan, inplace=True)
# inplace is used here to update the original dataframe after replace
# Calling head method now shows that all the ? are replace by NaN
df.head()
# Its not time to get to know how many NaNs we have in each column to take care of them accordingly
missing_values_df = df.isnull()
missing_values_df
# In the datagrid above, True means missing value or NaN and False means no-Nan means no missing value
# Let count how many missing and non-missing values we have 
for column in missing_values_df.columns.values.tolist():
    print(missing_values_df[column].value_counts())
    print("")
    
# The data above shows we have 7 columns with missing values

#********************************************************
#False    164
#True      41
#Name: normalized-losses, dtype: int64

#False    203
#True       2
#Name: num-of-doors, dtype: int64

#False    201
#True       4
#Name: bore, dtype: int64

#False    201
#True       4
#Name: stroke, dtype: int64

#False    203
#True       2
#Name: horsepower, dtype: int64

#False    203
#True       2
#Name: peak-rpm, dtype: int64

#False    201
#True       4
#Name: price, dtype: int64
#**********************************************************

#Lets work with each one of them to clean the data in each column
# For normalized-losses, lets replace the missing values by the mean of that column
# First lets find out the type of normalized-losses column

df["normalized-losses"].dtype


# Since the type is object and we want to calculate the mean, we need to convert this type to float or int.
# Lets convert the type to float and then find the mean
normalized_losses_mean = df["normalized-losses"].astype("float").mean()
normalized_losses_mean
# Lets replace the Nan value in this column by this mean values
df["normalized-losses"].replace(np.nan, normalized_losses_mean, inplace=True)
# The Nan values are now replaced by the mean
df["normalized-losses"].head()
# lets do the same for  bore column
df["bore"].dtype
# Since the type is object and we want to calculate the mean, we need to convert this type to float or int.
# Lets convert the type to float and then find the mean
bore_mean = df["bore"].astype("float").mean()
bore_mean
# Lets replace the Nan value in this column by this mean values
df["bore"].replace(np.nan, bore_mean, inplace=True)
# The Nan values are now replaced by the mean
df["bore"].head()
# lets do the same for stroke column
df["stroke"].dtype
# Since the type is object and we want to calculate the mean, we need to convert this type to float or int.
# Lets convert the type to float and then find the mean
stroke_mean = df["stroke"].astype("float").mean()
stroke_mean
# Lets replace the Nan value in this column by this mean values
df["stroke"].replace(np.nan, stroke_mean, inplace=True)
# The Nan values are now replaced by the mean
df["stroke"].head()
# lets do the same for horesepower column
df["horsepower"].dtype
# Since the type is object and we want to calculate the mean, we need to convert this type to float or int.
# Lets convert the type to float and then find the mean
horsepower_mean = df["horsepower"].astype("float").mean()
horsepower_mean
# Lets replace the Nan value in this column by this mean values
df["horsepower"].replace(np.nan, horsepower_mean, inplace=True)
# The Nan values are now replaced by the mean
df["horsepower"].head()
# lets do the same for horesepower column
df["peak-rpm"].dtype
# Since the type is object and we want to calculate the mean, we need to convert this type to float or int.
# Lets convert the type to float and then find the mean
peak_rpm_mean = df["peak-rpm"].astype("float").mean()
peak_rpm_mean
# Lets replace the Nan value in this column by this mean values
df["peak-rpm"].replace(np.nan, peak_rpm_mean, inplace=True)
# The Nan values are now replaced by the mean
df["peak-rpm"].head()
# For num-of-doors, taking mean wont make sense, so we replace the missing values by the most popular values in that column
# Lets find out the type of values we have in this column
df["num-of-doors"].value_counts()
# In this case, since we have only 2 type of values, we can see that most of the cars have four door.
# However, we can always use idmax method to find out the most popular value
df["num-of-doors"].value_counts().idxmax()
# Lets now replace the missing values in this column by four
df["num-of-doors"].replace(np.nan, "four", inplace=True)
df.head()
# Finally, we have some missing values in the price column.
# Since we want to base our model on price, it doesnt make sense to use records with missing value in price column
# Hence, lets delete the rows that have missing values in price
df.dropna(subset=["price"], axis=0, inplace=True)

# Its always a good idea to reset the index since we dropped some records
df.reset_index(drop=True, inplace=True)
df.head()<jupyter_output><empty_output><jupyter_text>Fixing the DataTypes<jupyter_code># Let see the datatype for each column
df.dtypes
# Its clear from the result that some column need a change in the data type as we want to perform some mathematical operations
df[["bore", "stroke"]] = df[["bore", "stroke"]].astype("float")
df[["normalized-losses"]] = df[["normalized-losses"]].astype("int")
df[["price"]] = df[["price"]].astype("float")
df[["peak-rpm"]] = df[["peak-rpm"]].astype("float")
# Lets verify the datatype for each column
df.dtypes<jupyter_output><empty_output><jupyter_text>Standardization<jupyter_code># Sometimes, data needs to be standardized according to the use case.
# In this case, instead of having mpg, the data would make more sense in L/100km
# So lets do the necessary conversion
df["city-L/100km"] = 235/df["city-mpg"]

df.head()
# So lets do the necessary conversion for highway-mpg
df["highway-L/100km"] = 235/df["highway-mpg"]
df.head()<jupyter_output><empty_output><jupyter_text>Normalization<jupyter_code># Normalization is the process of transforming values of several variables into a similar range. 
# Typical normalizations include scaling the variable so the variable average is 0, scaling the variable so the variance is 1.
# Scaling variable so the variable values range from 0 to 1
# Lets perform the normalization to length, width and height column
df['length'] = df['length']/df['length'].max()
df['width'] = df['width']/df['width'].max()
df['height'] = df['height']/df['height'].max()
# Lets review the results for the columns after normalization
df[["length","width","height"]].head()<jupyter_output><empty_output><jupyter_text>Binning<jupyter_code># Binning is a process of transforming continuous numerical variables into discrete categorical 'bins', for grouped analysis. 
# Lets perform binning on horsepower column by first converting its type to int
df["horsepower"]=df["horsepower"].astype(int, copy=True)
# Lets create a histogram to see how it looks like in its current form
%matplotlib inline
import matplotlib as plt
from matplotlib import pyplot
plt.pyplot.hist(df["horsepower"])

# set x/y labels and plot title
plt.pyplot.xlabel("horsepower")
plt.pyplot.ylabel("count")
plt.pyplot.title("horsepower bins")
# Lets create equal bins using linspace method in numpy
bins = np.linspace(min(df["horsepower"]), max(df["horsepower"]), 4)
bins
# Lets create groups for the bins to further analyze
group_names = ['Low', 'Medium', 'High']

#We apply the function "cut" the determine what each value of "df['horsepower']" belongs to. 
df['horsepower-binned'] = pd.cut(df['horsepower'], bins, labels=group_names, include_lowest=True )
df[['horsepower','horsepower-binned']].head(20)
# Lets find out the values count for each category
df["horsepower-binned"].value_counts()
# Finally lets plot each category on histogram
%matplotlib inline
import matplotlib as plt
from matplotlib import pyplot
pyplot.bar(group_names, df["horsepower-binned"].value_counts())

# set x/y labels and plot title
plt.pyplot.xlabel("horsepower")
plt.pyplot.ylabel("count")
plt.pyplot.title("horsepower bins")
# Finally lets plot using the 3 bins
%matplotlib inline
import matplotlib as plt
from matplotlib import pyplot

a = (0,1,2)

# draw historgram of attribute "horsepower" with bins = 3
plt.pyplot.hist(df["horsepower"], bins = 3)

# set x/y labels and plot title
plt.pyplot.xlabel("horsepower")
plt.pyplot.ylabel("count")
plt.pyplot.title("horsepower bins")<jupyter_output><empty_output><jupyter_text>Dummy Variable<jupyter_code># An indicator variable (or dummy variable) is a numerical variable used to label categories. 
# They are called 'dummies' because the numbers themselves don't have inherent meaning. 
# They are good for preparing our data for regression
# Lets see our columns again
df.columns
# Fuel-Type column contains values gas and disel, however we want them to be numeric
# let find out the type of values we have
df["fuel-type"].value_counts()
# we can use dummy variable to do that
dummy_variable_df = pd.get_dummies(df["fuel-type"])
dummy_variable_df.head()
# Lets concatenate it with the original dataframe
df = pd.concat([df,dummy_variable_df], axis=1)
# Lets verify it by calling the head
df.head()
# Since we have these two columns now, we dont need fuel-type, so lets drop it
df.drop("fuel-type", axis = 1, inplace=True)
df.head()
# Lets do the same with aspiration column
df["aspiration"].value_counts()
# we can use dummy variable to do that
dummy_aspiration_df = pd.get_dummies(df["aspiration"])
dummy_aspiration_df.head()
# Lets concatenate it with the original dataframe
df = pd.concat([df,dummy_aspiration_df], axis=1)
# Since we have these two columns now, we dont need aspiration, so lets drop it
df.drop("aspiration", axis = 1, inplace=True)
df.head()
# Finally lets export the cleaned csv
df.to_csv('clean_auto_df.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11537392271337225, 'fraction_numerical': 0.00769159484755815, 'mean_word_length': 5.19162363740677, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '467630', 'license': 'MIT', 'n_tokens_mistral': 3391, 'n_tokens_neox': 3226, 'n_words': 1289, 'path': 'Data_Wrangling.ipynb', 'repo_name': 'raziiq/python-data-wrangling', 'size': '10791'}