<jupyter_start><jupyter_text>Basic statistical features<jupyter_code>zillow.info()
zillow.describe()<jupyter_output><empty_output><jupyter_text>Detection of outliers<jupyter_code>float_zillow=zillow.select_dtypes(include=['float'])
float_zillow
for col in float_zillow:
    plt.boxplot(float_zillow[col])
    plt.title(col)
    plt.show()
int_zillow = zillow.select_dtypes(include=['int64'])
for col in int_zillow:
    plt.boxplot(int_zillow[col])
    plt.title(col)
    plt.show()
object_zillow = zillow.select_dtypes(include=['object'])
object_zillow.head()<jupyter_output><empty_output><jupyter_text>Since there are many independent variables with numerical values, a box plot is the sensible way to go in order to detect outliers.<jupyter_code>zillow.info()
sns.heatmap(int_zillow.corr(),annot=True,mask=int_zillow.corr()<0.7)
sns.heatmap(float_zillow.corr(),annot=True,mask=float_zillow.corr()<0.0)
high_corr_variables = int_zillow[['OverallQual','YearBuilt','YearRemodAdd','GrLivArea','GarageArea','TotalBsmtSF','SalePrice']]
sns.heatmap(high_corr_variables.corr(),annot=True)
zillow = zillow.select_dtypes(exclude=['object'])
zillow.head()
zillow.info()
zillow.isnull().sum()
zillow = zillow.dropna()
zillow['OverallQual'].value_counts()
zillow.describe()
zillow.iloc[:,16:21]
zillow['FullBath'].value_counts()
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()

normalized_x = scaler.fit_transform(zillow.iloc[:,:-1])
x = zillow.iloc[:,:-1]
y = zillow['SalePrice']

normalized_x
from sklearn.decomposition import PCA
pca = PCA()
pca.fit(normalized_x)
pca.explained_variance_ratio_ * 100
import matplotlib.pyplot as plt
plt.plot(pca.explained_variance_ratio_)
cumulative_variance = np.cumsum(pca.explained_variance_ratio_)
cumulative_variance
plt.plot(cumulative_variance)
A = np.where(cumulative_variance<.99,False,True)
np.where(A == True)
n_components = np.where(A == True)[0].min()
n_components
pca.components_.shape
p_components = pca.fit_transform(normalized_x)
p_components.shape
pca = PCA(n_components=n_components)
pca.fit(normalized_x)
pca.explained_variance_ratio_.sum()
p_components_one = pca.fit_transform(normalized_x) # pca.fit_transform, it transforms the normalized data in terms of principal components. I will have the same no. of rows as dependent variable.
p_components_one.shape
x_pca = p_components_one
y = zillow['SalePrice']
from sklearn.linear_model import LinearRegression
model= LinearRegression()
model.fit(x_pca,y)
# accuracy with x_pca and y
# accuracy with x and y
x = zillow.drop(['SalePrice'],axis=1)
y = zillow['SalePrice']
model_one=LinearRegression()
model_one.fit(x,y)
# x = zillow.iloc[:,:-1]
model.score(x_pca,y)
model_one.score(x,y)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1316270566727605, 'fraction_numerical': 0.007312614259597806, 'mean_word_length': 8.212121212121213, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '522276', 'license': 'Unlicense', 'n_tokens_mistral': 1083, 'n_tokens_neox': 1045, 'n_words': 167, 'path': 'Project github/PCA-(p).ipynb', 'repo_name': 'hhahmedtx/hhahmedtx.github.io', 'size': '2735'}