<jupyter_start><jupyter_text>1. Treating target variable<jupyter_code>## treating outliers
sns.boxplot(x = data['Sale Price'])
q1 = data['Sale Price'].quantile(0.25)
q3 = data['Sale Price'].quantile(0.75)
iqr = q3 - q1
upper_limit = q3 + 1.5*iqr
lower_limit = q1 - 1.5*iqr
print(upper_limit, lower_limit)
def limit_imputer(value):
    if value > upper_limit:
        return upper_limit
    if value < lower_limit:
        return lower_limit
    else:
        return value
data['Sale Price']= data['Sale Price'].apply(limit_imputer)
data['Sale Price'].describe()
sns.boxplot(x = data['Sale Price'])
## treating missing values in target variable
data.dropna(inplace = True, axis = 0, subset = ['Sale Price'])
data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 21609 entries, 0 to 21612
Data columns (total 21 columns):
 #   Column                                     Non-Null Count  Dtype  
---  ------                                     --------------  -----  
 0   ID                                         21609 non-null  int64  
 1   Date House was Sold                        21609 non-null  object 
 2   Sale Price                                 21609 non-null  float64
 3   No of Bedrooms                             21609 non-null  int64  
 4   No of Bathrooms                            21605 non-null  float64
 5   Flat Area (in Sqft)                        21600 non-null  float64
 6   Lot Area (in Sqft)                         21600 non-null  float64
 7   No of Floors                               21609 non-null  float64
 8   Waterfront View                            21609 non-null  object 
 9   No of Times Visited                        21609 non-null  object 
 10  Condition of the House[...]<jupyter_text>2. Treating independent variables<jupyter_code>## treating missing values
data.isnull().sum()
# missing values in numerical variables 
num_columns = ['No of Bathrooms','Flat Area (in Sqft)','Lot Area (in Sqft)',
              'Area of the House from Basement (in Sqft)','Latitude','Longitude',
               'Living Area after Renovation (in Sqft)']
from sklearn.impute import SimpleImputer
imputer = SimpleImputer( missing_values = np.nan, strategy = 'median')
data[num_columns] = imputer.fit_transform(data[num_columns])
# missing values in categorical variable
data['Zipcode'].shape
column = data['Zipcode'].values.reshape(-1,1)
column.shape
imputer2 = SimpleImputer(missing_values = np.nan, strategy = 'most_frequent')
data['Zipcode'] = imputer2.fit_transform(column)
## changing data type of num variable which should be categorical variable 
data['Zipcode'] = data['Zipcode'].astype(object)
## converting data from cat. var. to numerical
data['No of Times Visited'].unique()
mapping = {'None':'0','Once':'1','Twice':'2', 'Thrice':'3','Four':'4'}
data['No of Times Visited']= data['No of Times Visited'].map(mapping)
data['No of Times Visited'].unique()
## creating new imp column from existing data
data['Ever renovate']= np.where(data['Renovated Year']== 0,'No', 'Yes')
data['Purchase year']= pd.DatetimeIndex(data['Date House was Sold']).year
data.head()
data['Years since Renovation']= np.where(data['Ever renovate']== 'Yes',
                                        abs(data['Purchase year']- data['Renovated Year']), 0 )
data.drop(columns = ['Purchase year','Date House was Sold','Renovated Year'], inplace = True)
data.head()
## now we have four var. which are categorical type, we want to decide whether to keep or not
## we will check they have impact on target variable or not, and decide accordingly
## for this we use ANOVA

from statsmodels.formula.api import ols
import statsmodels.api as sm
data = data.rename(columns = {'Sale Price':'Sale_Price'})
data = data.rename(columns = {'Waterfront View':'Waterfront_view'})
data = data.rename(columns = {'Condition of the House':'Condition'})
data = data.rename(columns = {'Ever renovate':'Ever_renovate'})
mod = ols('Sale_Price ~ Condition', data = data).fit()
sm.stats.anova_lm(mod, typ = 2)
mod = ols ('Sale_Price ~ Waterfront_view', data= data).fit()
sm.stats.anova_lm(mod, typ=2)
mod = ols('Sale_Price ~ Zipcode', data = data).fit()
sm.stats.anova_lm(mod, typ=2)
mod = ols('Sale_Price ~ Ever_renovate',data= data).fit()
sm.stats.anova_lm(mod, typ = 2)
## we concluded that all 4 variables has high impact on target variable
## we will create dummy variables for all of them.
## getting dummies for first three (not zipcode)
data = pd.get_dummies(data, columns = ['Condition','Waterfront_view','Ever_renovate'], drop_first = True)
zip_table = data.groupby('Zipcode').agg({'Sale_Price':'mean'}).sort_values('Sale_Price',ascending= True)
zip_table['zip_group']= pd.cut(zip_table['Sale_Price'],bins=10,
                              labels = ['0','1','2','3','4','5','6','7','8','9'], include_lowest = True)
zip_table= zip_table.drop(columns=['Sale_Price'])
zip_table
data = pd.merge(data, zip_table,
               left_on = 'Zipcode',
               how = 'left',
               right_index = True)
data.info()
data = pd.get_dummies(data, columns= ["zip_group"], drop_first= True)
data
data = data.drop(columns= ['Zipcode'])
data.info()
data.to_csv('Preprocessed_linear_regression_project', index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1355157974876285, 'fraction_numerical': 0.02626570232204035, 'mean_word_length': 3.1939345570630486, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '144052', 'license': 'Apache-2.0', 'n_tokens_mistral': 1741, 'n_tokens_neox': 1612, 'n_words': 462, 'path': 'linear_regression_project_preprocesssing.ipynb', 'repo_name': 'Adityakurhade/Housing_prices_USA_internshala', 'size': '5254'}