<jupyter_start><jupyter_text>from scratch<jupyter_code># creating a random distribution of numbers(sample data)
def create_number(mu=54,sigma=15,seed=44,samples=100):

    np.random.seed(seed)
    sample_numer = np.random.normal(mu,sigma,samples) #to create_normal distribution
    sample_number = np.round(sample_numer,10)

    return sample_number
sample = create_number()
sample
sns.displot(sample,bins=20)
sns.boxplot(sample)
ser = pd.Series(sample)
ser
ser.describe()
# to find the lower limit and upper limit we can calucalte by Iqr
IQR = 62.535927 - 43.040896 # 75% - 25% of the
IQR
lower_limit = 43.040896 - (1.5*IQR) # 25% - 1.5*Iqr
lower_limit
upper_limit = 62.535927 + (1.5*IQR)
upper_limit 
# another method other than pandas we can do with numpy aslo
q25,q75 = np.percentile(sample,[25,75])
IQR = q75 - q25
IQR
# from the box plot we can identify the outliers is present below the lowerlimit so we can remove the outiler by
a = ser[ser> lower_limit]
a
len(a) # therefore one oulier is removed<jupyter_output><empty_output><jupyter_text>Working in the realworld Data sets<jupyter_code>pd.set_option('display.max_rows',None)
df = pd.read_csv(r'D:\ml_ineuron\modular\Ml_class_ineuron\data\Ames_Housing_Data.csv')
df.head()
df.corr()
#so it soo big lets check only relation between the dependent and independent only
df.corr()['SalePrice'].sort_values(ascending=False)
#from the corr we can say the Overall Qual is strongly positive correlated wiith sale price.
sns.scatterplot(x='Overall Qual',y='SalePrice',data=df)
# as the overall qual increae sale price aslo increase
# but from the chart we can identify that there are 3 outliers above 8 which is increased
# lets check with another higly corr feature 
sns.scatterplot(x='Gr Liv Area',y='SalePrice',data=df)

# we conclude that after 4000 that 3 points didnt increase
# lets find the id of the points to remove them
df[(df['Overall Qual'] > 8) & (df['SalePrice'] < 200000)]
df[(df['Gr Liv Area'] > 4000) & (df['SalePrice'] < 400000)]
drp_ind = df[(df['Gr Liv Area'] > 4000) & (df['SalePrice'] < 400000)].index
df = df.drop(drp_ind,axis=0)
#we drop the three outliers rows lets check now
sns.scatterplot(x='Gr Liv Area',y='SalePrice',data=df)
df.info()
#pid is unneccory lets drop that
df = df.drop('PID',axis=1)
df.isnull().sum()
#lets convert the missing info into percentiles 
def percent_missing(df):
    nan_per = (df.isnull().sum()/len(df))*100
    percent_miss = nan_per[nan_per>0].sort_values()

    return percent_miss
percent_miss = percent_missing(df)
percent_miss
plt.figure(figsize=(5,3),dpi=100)
sns.barplot(y=percent_miss,x= percent_miss.index)
plt.xticks(rotation=90)
plt.figure(figsize=(5,3), dpi=100)
sns.barplot(x=percent_miss.index, y= percent_miss, palette='viridis')
plt.xticks(rotation=90)
plt.ylim(0,1) #for visulaizing 1% data
# dropping the row of feature which is 1 percent of missing values 
percent_miss[percent_miss< 1.0]
percent_miss[percent_miss > 1.0]
df[df['Electrical'].isnull()]
df[df['Garage Area'].isnull()]
df[df['Total Bsmt SF'].isnull()]
df = df.dropna(axis=0, subset=['Electrical', 'Garage Cars','Total Bsmt SF'])
percent_nan = percent_missing(df)
plt.figure(figsize=(5,3), dpi=100)
sns.barplot(x=percent_nan.index, y= percent_nan, palette='viridis')
plt.xticks(rotation=90)
plt.ylim(0,1) #for visulaizing 1% data
# BSMT Numeric Columns - fillna "0"
bsmt_num_cols = ['BsmtFin SF 1','BsmtFin SF 2', 'Bsmt Unf SF', 'Total Bsmt SF','Bsmt Full Bath', 'Bsmt Half Bath']
df[bsmt_num_cols] = df[bsmt_num_cols].fillna(0) #0 Imputation

#BSMT Ctegorical Columns - fill "None"
bsmt_str_cols = ['Bsmt Qual', 'Bsmt Cond', 'Bsmt Exposure', 'BsmtFin Type 1', 'BsmtFin Type 2']
df[bsmt_str_cols] = df[bsmt_str_cols].fillna('None') #None Imputation
percent_nan = percent_missing(df)
plt.figure(figsize=(5,3), dpi=100)
sns.barplot(x=percent_nan.index, y= percent_nan, palette='viridis')
plt.xticks(rotation=90)
plt.ylim(0,1) #for visulaizing 1% data
df['Mas Vnr Area'].isnull().sum()

df['Mas Vnr Type'] = df['Mas Vnr Type'].fillna("None") #categorical feature
df["Mas Vnr Area"] = df['Mas Vnr Area'].fillna("0") #numerical feature
percent_nan = percent_missing(df)
percent_nan
plt.figure(figsize=(5,8),dpi=100)
sns.barplot(x=percent_nan.index,y = percent_nan, palette= 'viridis')
plt.xticks(rotation=90)

#from fence to poolqc the nan percentage is above 80 percent it is better to drop that
df = df.drop(['Fence','Alley','Misc Feature','Pool QC'],axis = 1)
garage_df = df[['Garage Type','Garage Yr Blt','Garage Finish','Garage Qual','Garage Cond']]
#garage_df[garage_df.isnull()] # all the row of the garage is null but there are 157 colums we cant drop lets impute with None
garage_df.isnull().sum()
grg_str = ['Garage Type', 'Garage Finish', 'Garage Qual', 'Garage Cond']
df[grg_str] = df[grg_str].fillna('No_garage')
df['Garage Yr Blt'] = df['Garage Yr Blt'].fillna(0) #there is no garage lets impute with 0

percent_nan = percent_missing(df)
percent_nan
plt.figure(figsize=(5,8),dpi=100)
sns.barplot(x=percent_nan.index,y = percent_nan, palette= 'viridis')
plt.xticks(rotation=90)
df['Fireplace Qu'].value_counts()
df['Fireplace Qu'].isnull().sum()
df['Fireplace Qu'] = df['Fireplace Qu'].fillna('No_fireplace')
plt.figure(figsize=(8,11), dpi=100)
sns.boxplot(x='Lot Frontage', y='Neighborhood', data=df)
plt.figure(figsize=(8,11), dpi=100)
sns.boxplot(x='Lot Frontage', y='Lot Config', data=df)
df.groupby('Neighborhood')['Lot Frontage'].mean()
df.groupby('Lot Config')['Lot Frontage'].mean()
df['Lot Frontage'] = df.groupby('Lot Config')['Lot Frontage'].transform(lambda value : value.fillna(value.mean()))
percent_nan = percent_missing(df)
percent_nan
# atlast there is no null value<jupyter_output><empty_output><jupyter_text>Encoding the categorical Value<jupyter_code>df['MS SubClass'].dtype
df['MS SubClass'] = df['MS SubClass'].apply(str)
df.info()
object_df = df.select_dtypes(include='object')
object_df.head()
numeric_df = df.select_dtypes(exclude='object') #return all numeric columns
numeric_df.head()
df_object_dummies = pd.get_dummies(object_df, drop_first = True) #encoded dataframe
df_object_dummies.head()
final_df = pd.concat([numeric_df,df_object_dummies],axis=1)<jupyter_output><empty_output><jupyter_text>modelling<jupyter_code>X =final_df.drop('SalePrice', axis=1)
X.head()
y =final_df['SalePrice']
from sklearn.model_selection import train_test_split , GridSearchCV
from sklearn.linear_model import ElasticNet
from sklearn.preprocessing import StandardScaler

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=101)
scaler = StandardScaler()
scaled_X_train = scaler.fit_transform(X_train)
scaled_X_test = scaler.fit_transform(X_test)
base_elastic_model = ElasticNet(max_iter = 100000) # max_iter used for to overcome the conergence problem with

param_grid = {'alpha': [200,250,300], 'l1_ratio':[0.1, 0.5, 1]}
grid_model = GridSearchCV(base_elastic_model,param_grid,scoring='neg_mean_squared_error', cv=5,verbose=2)
grid_model.fit(scaled_X_train,y_train)

grid_model.best_params_
from sklearn.metrics import mean_squared_error
y_pred = grid_model.predict(scaled_X_test)
mean_squared_error(y_test,y_pred,squared=False)
np.mean(y)
#percentage of the error
(23821.18675693739/180850.35807113544)*100<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14052197802197802, 'fraction_numerical': 0.03571428571428571, 'mean_word_length': 6.974808324205915, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '327091', 'license': 'MIT', 'n_tokens_mistral': 2937, 'n_tokens_neox': 2680, 'n_words': 640, 'path': 'Practise_realdata/Outliers.ipynb', 'repo_name': 'yokesh492/Ml_class_ineuron', 'size': '7280'}