<jupyter_start><jupyter_text>Importing Libraries<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
sns.set()<jupyter_output><empty_output><jupyter_text>Load CSV into dataframes<jupyter_code>df_1 = pd.read_csv("C:/Users/Mohsen/Desktop/train.csv")
df_1.head(5)<jupyter_output><empty_output><jupyter_text>Classify features<jupyter_code>categorical_features = []
numerical_features = []
def classify_features():
    categorical_features = [feature for feature in df.columns if df[feature].dtypes == 'O']
    #Categorical features that are encoded with numerical values
    categorical_features = categorical_features + ['MSSubClass', 'OverallQual', 'OverallCond']
    numerical_features = [feature for feature in df.columns if feature not in categorical_features and feature != 'Id']
    return categorical_features, numerical_features
categorical_features, numerical_features = classify_features()
categorical_features
numerical_features<jupyter_output><empty_output><jupyter_text>Handle Null values<jupyter_code>def get_null_percent(df):
    return {feature: (df[feature].isnull().sum()/len(df))*100 
                for feature in df.columns if df[feature].isnull().sum() > 0}
null_features = get_null_percent(df)
null_features<jupyter_output><empty_output><jupyter_text>Remove features with large percent of null values.<jupyter_code>del_features = [key for key, val in null_features.items() if val > 75]
del_features
df.drop(del_features, axis = 1, inplace = True)
categorical_features, numerical_features = classify_features()
null_features = get_null_percent(df)<jupyter_output><empty_output><jupyter_text>Handle Null values in numerical features<jupyter_code>for feature in numerical_features:
    feature_median = df[feature].median()
    df[feature].fillna(value = feature_median, inplace=True)<jupyter_output><empty_output><jupyter_text>Handle Null values in categorical features<jupyter_code>for feature in categorical_features:
    feature_mode = df[feature].mode()[0]
    df[feature].fillna(value = feature_mode, inplace=True)
null_features = get_null_percent(df)
null_features<jupyter_output><empty_output><jupyter_text>Handling Outliers in the data<jupyter_code>len(numerical_features)<jupyter_output><empty_output><jupyter_text>Boxplots for outlier detection<jupyter_code>fig, axs = plt.subplots(11, 3)
# fig.suptitle('Outliers in the data')
fig.set_figheight(50)
fig.set_figwidth(17)
feature_idx = 0
for row_idx in range(0, 11):
    for col_idx in range(0, 3):
        feature = numerical_features[feature_idx]
        axs[row_idx, col_idx].boxplot(df[feature])
        axs[row_idx, col_idx].set_title('Outliers in {}'.format(feature))
        feature_idx += 1
plt.show()
len(numerical_features)<jupyter_output><empty_output><jupyter_text>Histogram for understanding distribution<jupyter_code>fig, axs = plt.subplots(8, 3)
# fig.suptitle('Outliers in the data')
fig.set_figheight(30)
fig.set_figwidth(17)
feature_idx = 0
for row_idx in range(0, 8):
    for col_idx in range(0, 3):
        feature = numerical_features[feature_idx]
        axs[row_idx, col_idx].hist(df[feature], bins=25)
        axs[row_idx, col_idx].set_title('Hist of {}'.format(feature))
        feature_idx += 1
plt.show()<jupyter_output><empty_output><jupyter_text>Skewness and Kurtosis<jupyter_code>a = pd.DataFrame(df.skew(), columns=['Skewness'])
b = pd.DataFrame(df.kurt(), columns=['Kurtosis'])
tdf = a.join(b)
tdf.head(70)
def get_iqr_limit(df, feature):
    Q1 =df[feature].quantile(0.25)
    Q3 = df[feature].quantile(0.75)
    IQR = Q3 - Q1
    return (Q1 - (IQR*1.5)), (Q3 + (IQR*1.5))
f_lst = []
for feature in numerical_features:
    lower_limit, upper_limit = get_iqr_limit(df, feature)
    if lower_limit == 0 and upper_limit == 0:
        # remove features with mostly zero values
        df.drop(columns=feature, axis = 1, inplace=True)
        f_lst.append(feature)
    else:
        # indexes of the outliers
        outlier_idxs = np.where(np.logical_or(df[feature].values < lower_limit, 
                                              df[feature].values > upper_limit))[0]
        feature_median = df[feature].median()
        df.loc[list(outlier_idxs), feature] = feature_median
categorical_features, numerical_features = classify_features()
df.shape
a = pd.DataFrame(df.skew(), columns=['Skewness'])
b = pd.DataFrame(df.kurt(), columns=['Kurtosis'])
tdf = a.join(b)
tdf.head(70)
df.to_csv('df_clean.csv', index=False)
import pandas as pd
import numpy as np
from scipy.stats import f_oneway
df = pd.read_csv("C:/Users/Mohsen/Desktop/df_clean.csv")
df.head(10)
# Question1
a = max(df.LotArea)
print('house the maximum is ', a)
b = min(df.LotArea)
print('house the minimum is ', b)

# Question2
df.groupby(['LotShape']).size()
!pip install seaborn
import seaborn as sns
import matplotlib.pyplot as plt
sns.catplot(x="LotShape", kind="count", data=df)

# Question3,How many houses are contained  Lots configuration on the corner, it is 263
df.groupby(['LotConfig']).size()
sns.catplot(x="LotConfig", kind="count", data=df)
# Question 4,it is 445
df.groupby(['HouseStyle']).size()
sns.catplot(x="HouseStyle", kind="count", data=df)
#Question 5, most of the people prefer 1Story(726) than 2Story(445)
# Question6, Total number of houses with best overall condition according to the reviews from 8 or more than 8 on the scale of 10 is 94
df.groupby(['OverallCond']).size()
sns.catplot(x="OverallCond", kind="count", data=df)
# Question 7,the oldest built house and latest newly built house in this particular area
d = max(df.YearBuilt)
print('The latest built house is',d)
e = min(df.YearBuilt)
print('The oldest built house is',e)

# Question 8,Which roof style is more popular in that area
f = max(df.RoofStyle)
print('The popular roofstyle  is',f)
sns.catplot(x="RoofStyle", kind="count", data=df)
# Question 9,Number of buildings which used MetalSd material as their first exteriors material, it is 220
df.groupby(['Exterior1st']).size()
sns.catplot(x="Exterior1st", kind="count", data=df)
# Question 10,Which technique is used to made the foundation of most of the houses
g = max(df.Foundation)
print('The foundation of most of the houses  is',g)
sns.catplot(x="Foundation", kind="count", data=df)
# Question 11, What is the percentage of central air adopted houses is 0.93493150684
df.groupby(['CentralAir']).size()
sns.catplot(x="CentralAir", kind="count", data=df)
#Question 12,Most of the people preferred to live in 3 bhk house(804) than 2 bhk house(358)  in this area according to this dataset
df.groupby(['BedroomAbvGr']).size()
sns.catplot(x="BedroomAbvGr", kind="count", data=df)
#Question13,Number of houses without fireplace is 690
df.groupby(['Fireplaces']).size()
sns.catplot(x="Fireplaces", kind="count", data=df)
# Question 14,  88 of houses is there in which car garage are built in the house
df.groupby(['GarageType']).size()
sns.catplot(x="GarageType", kind="count", data=df)
# Question 15, The percentage of houses without fencing is 1179
df_1['Fence'].isna().sum()

# Question 16,Find the most expensive house in the area is 755000
h = max(df.SalePrice)
print('the most expensive house is ', h)
# Question 17, In which year most number of houses were sold
I = max(df.YrSold)
print('The year most number of houses were sold is ', I)
sns.catplot(x="YrSold", kind="count", data=df)
# Question 18, Is there any relation between Yearbuilt and salePrice,
#it shows that there is a relation between yearbuilt and price,new home are more expensive
f_oneway(df.YearBuilt, df.SalePrice)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.121787267694741, 'fraction_numerical': 0.020166073546856466, 'mean_word_length': 5.6328671328671325, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '527155', 'license': 'Apache-2.0', 'n_tokens_mistral': 2698, 'n_tokens_neox': 2530, 'n_words': 663, 'path': 'BDAT1005,Final Project.ipynb', 'repo_name': 'Mohsenselseleh/My-Projects', 'size': '7587'}