<jupyter_start><jupyter_text>Loading raw data<jupyter_code>raw_data=pd.read_csv('C:/Users/rjvbb/Desktop/RJv Folder/Folder/Intern docs/Data science/Part_5_Advanced_Statistical_Methods_(Machine_Learning)/S35_L225/1.04. Real-life example.csv')
raw_data.head()<jupyter_output><empty_output><jupyter_text>Data preprocessing<jupyter_code>## exploring describtive statistics of variables
raw_data.describe(include = 'all')<jupyter_output><empty_output><jupyter_text>Determining the variable of interest<jupyter_code>## axis=0- row, axis=1 - columns
data= raw_data.drop(['Model'],axis=1)
data.describe(include = 'all')<jupyter_output><empty_output><jupyter_text>Dealing with missing values<jupyter_code>data.isnull().sum()
## data with no missing value
data_no_mv=data.dropna(axis=0)
data.describe(include = 'all')<jupyter_output><empty_output><jupyter_text>Exploring the pdfs (probability distribution function)<jupyter_code>sns.distplot(data_no_mv['Price'])<jupyter_output>C:\Users\rjvbb\anaconda3\lib\site-packages\seaborn\distributions.py:2551: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).
  warnings.warn(msg, FutureWarning)<jupyter_text>Dealing with outliers<jupyter_code>##removing top 1 percent in other words keeping 99 percentile
q=data_no_mv['Price'].quantile(0.99)
data_1=data_no_mv[data_no_mv['Price']<q]
data_1.describe(include = 'all')
sns.distplot(data_1['Price'])
sns.distplot(data_no_mv['Mileage'])
q=data_no_mv['Mileage'].quantile(0.99)
data_2=data_no_mv[data_no_mv['Mileage']<q]
sns.distplot(data_2['Mileage'])
sns.distplot(data_no_mv['EngineV'])
data_3=data_2[data_2['EngineV']<6.5]
sns.distplot(data_3['EngineV'])
sns.distplot(data_no_mv['Year'])
q=data_3['Year'].quantile(0.01)
data_4=data_3[data_3['Year']>q]
sns.distplot(data_4['Year'])
data_cleaned=data_4.reset_index(drop=True)
data_cleaned.describe(include='all')<jupyter_output><empty_output><jupyter_text>Checking OLS assumptions<jupyter_code># Here we decided to use some matplotlib code, without explaining it
#  can simply use plt.scatter() for each of them (with your current knowledge)
# But since Price is the 'y' axis of all the plots, it made sense to plot them side-by-side (so we can compare them)
f, (ax1, ax2, ax3) = plt.subplots(1, 3, sharey=True, figsize =(15,3)) #sharey -> share 'Price' as y
ax1.scatter(data_cleaned['Year'],data_cleaned['Price'])
ax1.set_title('Price and Year')
ax2.scatter(data_cleaned['EngineV'],data_cleaned['Price'])
ax2.set_title('Price and EngineV')
ax3.scatter(data_cleaned['Mileage'],data_cleaned['Price'])
ax3.set_title('Price and Mileage')


plt.show()
sns.distplot(data_cleaned['Price'])<jupyter_output>C:\Users\rjvbb\anaconda3\lib\site-packages\seaborn\distributions.py:2551: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).
  warnings.warn(msg, FutureWarning)<jupyter_text>realxing the assumptions<jupyter_code>## since price is exponential increasing it doesnt form linear relation. Hence transforming to log function

log_price= np.log(data_cleaned['Price'])
data_cleaned['log_price']= log_price
data_cleaned
## plotting again

f, (ax1, ax2, ax3) = plt.subplots(1, 3, sharey=True, figsize =(15,3)) #sharey -> share 'Price' as y
ax1.scatter(data_cleaned['Year'],data_cleaned['log_price'])
ax1.set_title('Log Price and Year')
ax2.scatter(data_cleaned['EngineV'],data_cleaned['log_price'])
ax2.set_title('Log Price and EngineV')
ax3.scatter(data_cleaned['Mileage'],data_cleaned['log_price'])
ax3.set_title('Log Price and Mileage')


plt.show()
## dropping original price in data frame since it is not needed more as we are getting linear relation in log price value
data_cleaned= data_cleaned.drop(['Price'],axis=1)<jupyter_output><empty_output><jupyter_text>Multi colinearity<jupyter_code>data_cleaned.columns.values<jupyter_output><empty_output><jupyter_text>Checking multi colineairy<jupyter_code># sklearn does not have a built-in way to check for multicollinearity
# one of the main reasons is that this is an issue well covered in statistical frameworks and not in ML ones
## Variance inflation factor

# Here's the relevant module
from statsmodels.stats.outliers_influence import variance_inflation_factor

# To make this as easy as possible to use, we declare a variable where we put
# all features where we want to check for multicollinearity
# since our categorical data is not yet preprocessed, we will only take the numerical ones
variables = data_cleaned[['Mileage','Year','EngineV']]

# we create a new data frame which will include all the VIFs
# note that each variable has its own variance inflation factor as this measure is variable specific (not model specific)
vif = pd.DataFrame()

# here we make use of the variance_inflation_factor, which will basically output the respective VIFs 
vif["VIF"] = [variance_inflation_factor(variables.values, i) for i in range(variables.shape[1])]
# Finally, I like to include names so it is easier to explore the result
vif["Features"] = variables.columns
vif
# Since Year has the highest VIF, I will remove it from the model
# This will drive the VIF of other variables down!!! 
# So even if EngineV seems with a high VIF, too, once 'Year' is gone that will no longer be the case
data_no_multicollinearity = data_cleaned.drop(['Year'],axis=1)<jupyter_output><empty_output><jupyter_text>Creating dmmy variables<jupyter_code># To include the categorical data in the regression, let's create dummies
# There is a very convenient method called: 'get_dummies' which does that seemlessly
# It is extremely important that we drop one of the dummies, alternatively we will introduce multicollinearity
data_with_dummies = pd.get_dummies(data_no_multicollinearity, drop_first=True)
data_with_dummies<jupyter_output><empty_output><jupyter_text>reaaranging<jupyter_code>## manually 
data_with_dummies.columns.values
cols= ['log_price','Mileage', 'EngineV', 'Brand_BMW',
       'Brand_Mercedes-Benz', 'Brand_Mitsubishi', 'Brand_Renault',
       'Brand_Toyota', 'Brand_Volkswagen', 'Body_hatch', 'Body_other',
       'Body_sedan', 'Body_vagon', 'Body_van', 'Engine Type_Gas',
       'Engine Type_Other', 'Engine Type_Petrol', 'Registration_yes']
data_preprocessed=data_with_dummies[cols]
data_preprocessed.head()<jupyter_output><empty_output><jupyter_text>Linear Regression model declare inputs and target<jupyter_code>targets=data_preprocessed['log_price']

## data pre processed with out log price
inputs= data_preprocessed.drop(['log_price'],axis=1)<jupyter_output><empty_output><jupyter_text>Scaling data<jupyter_code># Import the scaling module
from sklearn.preprocessing import StandardScaler

scaler= StandardScaler()
scaler.fit(inputs)
inputs_scaled= scaler.transform(inputs)<jupyter_output><empty_output><jupyter_text>Train test split<jupyter_code># Import the module for the split
from sklearn.model_selection import train_test_split

# Split the variables with an 80-20 split and some random state
# To have the same split as mine, use random_state = 365
x_train, x_test, y_train, y_test = train_test_split(inputs_scaled, targets, test_size=0.2, random_state=365)<jupyter_output><empty_output><jupyter_text>Regression<jupyter_code>reg= LinearRegression()
reg.fit(x_train,y_train)
# Let's check the outputs of the regression
# store them in y_hat as this is the 'theoretical' name of the predictions
y_hat = reg.predict(x_train)
# The simplest way to compare the targets (y_train) and the predictions (y_hat) is to plot them on a scatter plot
# The closer the points to the 45-degree line, the better the prediction
plt.scatter(y_train, y_hat)

#  name the axes
plt.xlabel('Targets (y_train)',size=18)
plt.ylabel('Predictions (y_hat)',size=18)

# Sometimes the plot will have different scales of the x-axis and the y-axis
# This is an issue as we won't be able to interpret the '45-degree line'
# We want the x-axis and the y-axis to be the same
plt.xlim(6,13)
plt.ylim(6,13)
plt.show()
# Another useful check of our model is a residual plot
# We can plot the PDF of the residuals and check for anomalies
sns.distplot(y_train - y_hat)

# Include a title
plt.title("Residuals PDF", size=18)

# In the best case scenario this plot should be normally distributed
# In our case we notice that there are many negative residuals (far away from the mean)
# Given the definition of the residuals (y_train - y_hat), negative values imply
# that y_hat (predictions) are much higher than y_train (the targets)
# This is for thought to improve our model
reg.score(x_train,y_train)<jupyter_output><empty_output><jupyter_text>finding weights and bias<jupyter_code># Obtain the bias (intercept) of the regression
reg.intercept_
# Obtain the weights (coefficients) of the regression
reg.coef_
# Create a regression summary where we can compare them with one-another
reg_summary = pd.DataFrame(inputs.columns.values, columns=['Features'])
reg_summary['Weights'] = reg.coef_
reg_summary
# Check the different categories in the 'Brand' variable
data_cleaned['Brand'].unique()<jupyter_output><empty_output><jupyter_text>Testing<jupyter_code># Once we have trained and fine-tuned our model, we can proceed to testing it
# Testing is done on a dataset that the algorithm has never seen
# Luckily we have prepared such a dataset
# Our test inputs are 'x_test', while the outputs: 'y_test' 
# We SHOULD NOT TRAIN THE MODEL ON THEM, we just feed them and find the predictions
# If the predictions are far off, we will know that our model overfitted
y_hat_test = reg.predict(x_test)
# Create a scatter plot with the test targets and the test predictions
# You can include the argument 'alpha' which will introduce opacity to the graph
plt.scatter(y_test, y_hat_test, alpha=0.2)
plt.xlabel('Targets (y_test)',size=18)
plt.ylabel('Predictions (y_hat_test)',size=18)
plt.xlim(6,13)
plt.ylim(6,13)
plt.show()
# Finally, let's manually check these predictions
# To obtain the actual prices, we take the exponential of the log_price
df_pf = pd.DataFrame(np.exp(y_hat_test), columns=['Prediction'])
df_pf.head()
# We can also include the test targets in that data frame (so we can manually compare them)
df_pf['Target'] = np.exp(y_test)
df_pf

## lot of missing vaues
# After displaying y_test, we find what the issue is
# The old indexes are preserved (recall earlier in that code we made a note on that)
# The code was: data_cleaned = data_4.reset_index(drop=True)

# Therefore, to get a proper result, we must reset the index and drop the old indexing
y_test = y_test.reset_index(drop=True)

# Check the result
y_test.head()
# again putting values after resetiing indexes
df_pf['Target'] = np.exp(y_test)
df_pf
# Additionally, we can calculate the difference between the targets and the predictions
# Note that this is actually the residual (we already plotted the residuals)
df_pf['Residual'] = df_pf['Target'] - df_pf['Prediction']

# Since OLS is basically an algorithm which minimizes the total sum of squared errors (residuals),
# this comparison makes a lot of sense
# Finally, it makes sense to see how far off we are from the result percentage-wise
# Here, we take the absolute difference in %, so we can easily order the data frame
df_pf['Difference%'] = np.absolute(df_pf['Residual']/df_pf['Target']*100)
df_pf
df_pf.describe()
# Sometimes it is useful to check these outputs manually
# To see all rows, we use the relevant pandas syntax
pd.options.display.max_rows = 999
# Moreover, to make the dataset clear, we can display the result with only 2 digits after the dot 
pd.set_option('display.float_format', lambda x: '%.2f' % x)
# Finally, we sort by difference in % and manually check the model
df_pf.sort_values(by=['Difference%'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09915084915084915, 'fraction_numerical': 0.011072261072261072, 'mean_word_length': 6.62246192893401, 'pattern_counts': {'":': 0, '<': 74, '<?xml version=': 0, '>': 74, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '448126', 'license': 'MIT', 'n_tokens_mistral': 4026, 'n_tokens_neox': 3852, 'n_words': 1238, 'path': 'Linear Regression car sales( Practical).ipynb', 'repo_name': 'rjvmhn007/Linear-Regression-car-sales-Practical-', 'size': '12012'}