<jupyter_start><jupyter_text><jupyter_code>#Assignment 1+2: EDA and Regression for ML Part 2
#<NAME>
import pickle 
import numpy as np
import pandas as pd
import sklearn 
with open ('/Users/areejreaz/Downloads/assign-1-radon-data.pickle', 'rb') as pickle_file:
    pickle_data = pickle.load(pickle_file)
pickle_df = pd.DataFrame(pickle_data)
pickle_df

# Examine the data for missing or invalid values.
#Decide what to do about missing or invalid data you observe:  Delete? Replace, Impute?
pickle_df.isnull().sum()
#1 Null item - should be deleted
pd_df1 = pickle_df.dropna()
print(pd_df1)
pd_df1.info()


#so the  Median HH Income - has one blank row and we can just drop this record so then there will be good data present.
#Describe the data by calculating descriptive statistics
print(pd_df1.describe())

#Based on the description below, it shows there are about 7,800,000 deaths on average per year due to lung cancer.
#Radon level seems to have a possible outlier, if Max is at 99.7 and average is about 3.08.
#On Average 29% of the population is obese. 

#Graph Lung Cancer Mortality Rate vs. Radon
#X = ['Radon Rank']
#Y = ['Lung Cancer Mortality']

import matplotlib.pyplot as plt
import pandas as pd

pd_df1.plot(kind='scatter',x='Lung Cancer Mortality',y='Radon',color='blue')
plt.show()


#Lung Cancer Mortality:
#Mortality Rate (Deaths per 100,000 Person-Years)
#Radon:
#County Average Indoor Radon Level (1 decimal place)
# Density Plot and Histogram of all arrival delays
import seaborn as sns
import seaborn as seabornInstance 
#x = pd_df1['Lung Cancer Mortality']
#sns.distplot(x) 

plt.figure(figsize=(15,10))
plt.tight_layout()
seabornInstance.distplot(pd_df1['Lung Cancer Mortality'])
#Train a regression model that predicts Lung Cancer Mortality (LCM) using a cross-validation method.  
#K-fold cross-validation is preferred, but simple hold-out will suffice for this assignment.  
#Evaluate the performance of your model. 

#---------------------------------------------------------------------------------------------#


# seed value for random number generators to obtain reproducible results
RANDOM_SEED = 1


# modeling routines from Scikit Learn packages
import sklearn.linear_model 
from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet
from sklearn.metrics import mean_squared_error, r2_score  
import matplotlib.pyplot as plt  
import seaborn as seabornInstance 
from sklearn.model_selection import train_test_split 
from sklearn import metrics
from math import sqrt  # for root mean-squared error calculation
import sklearn.linear_model 

#--------------------------------------------------------------------------------------------#

X = pd_df1['Radon'].values.reshape(-1,1)
y = pd_df1['Lung Cancer Mortality'].values.reshape(-1,1)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)


regressor = LinearRegression()  
regressor.fit(X_train, y_train) #training the algorithm
#testing the data between the LCM  vs the Radon levels
y_pred = regressor.predict(X_test)
df = pd.DataFrame({'Actual': y_test.flatten(), 'Predicted': y_pred.flatten()})
df.head(10)
plt.scatter(X_test, y_test,  color='green')
plt.plot(X_test, y_pred, color='blue', linewidth=2)
plt.xlabel('LCM')
plt.ylabel('Radon')
plt.title('LCM vs Radon ')
plt.show()
# MSE is an appropriate performance metric.  
#The predictors, or "features" you use to predict LCM, is your choice to make, as long as you can justify it.

print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, y_pred))  
print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))  
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))
#8.	 Train and validate a regression model that predicts LCM after standardizing, or rescaling, your features. 
    #Use a standardization or rescaling method of your choice.  Use K-fold or hold out validation. See: 
#https://scikit-learn.org/stable/modules/preprocessing.html#standardization-or-mean-removal-and-variance-scaling (Links to an external site.)
#https://scikit-learn.org/stable/auto_examples/preprocessing/plot_all_scaling.html#sphx-glr-auto-examples-preprocessing-plot-all-scaling-py (Links to an external site.)
from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler
#from sklearn.preprocessing import MinMaxScaler
import scipy
from pylab import rcParams
import seaborn as sns
from pandas import Series, DataFrame

import matplotlib.pyplot as plt 


xS = pd_df1['Radon'].values.reshape(-1,1)
yS = pd_df1['Lung Cancer Mortality'].values.reshape(-1,1)

XS_train, XS_test, yS_train, yS_test = train_test_split(XS, yS, test_size=0.2, random_state=0)


scaler = preprocessing.StandardScaler().fit(XS_train)
scaler
StandardScaler()

scaler.mean_


scaler.scale_
scaler.transform(XS_train)
print('Mean Absolute Error:', metrics.mean_absolute_error(yS_test, yS_pred))  
print('Mean Squared Error:', metrics.mean_squared_error(yS_test, yS_pred))  
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(yS_test, yS_pred)))
plt.scatter(XS_test, yS_test,  color='blue')
plt.plot(XS_test, yS_pred, color='yellow', linewidth=2)
plt.xlabel('LCM')
plt.ylabel('Radon')
plt.title('LCM vs Radon ')
plt.show()

#Based on the graph below, it seems the standard scalar method I am using is perhaps not working.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13051538746755653, 'fraction_numerical': 0.01167964404894327, 'mean_word_length': 4.544707091469681, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '154538', 'license': 'MIT', 'n_tokens_mistral': 1849, 'n_tokens_neox': 1728, 'n_words': 543, 'path': 'Assn_2_LungCancer_Regression.ipynb', 'repo_name': 'reaz31/CourseProj', 'size': '5398'}