<jupyter_start><jupyter_text>Filter Outliers To git rid of outliers, we need to filter:    * Properties sold before 2019    * landsize over 2000 sqm    * carspace,bedroom over 6    * Properties price over 3M<jupyter_code>df['sold_date']= df['sold_date'].astype('datetime64[ns]')
df['sold_date'] = df['sold_date'].dt.strftime('%Y')
df['sold_date'] = df['sold_date'].astype('int')
df['rent_date']= df['rent_date'].astype('datetime64[ns]')
df['rent_date'] = df['rent_date'].dt.strftime('%Y')
df['rent_date'] = df['rent_date'].astype('int')
df.dtypes
# After2019_df = df[(df['sold_date']>2019)]
# After2019_df
Less2000sqm_df = df[(df['land_size']<2000)]
Less2000sqm_df
carspaceLessThan7_df = Less2000sqm_df[(Less2000sqm_df['car_space']<7)]
carspaceLessThan7_df
BedroomsLessthan7_df = carspaceLessThan7_df[(carspaceLessThan7_df['bedrooms']<7)]
BedroomsLessthan7_df
filtered_df = BedroomsLessthan7_df[(BedroomsLessthan7_df['price']<3000000)]<jupyter_output><empty_output><jupyter_text>Data Preporcessing<jupyter_code># filtered_df = df
#only getting landed properties for the machine learning
house = filtered_df[(filtered_df['property_type']=='House') | (filtered_df['property_type']=='Villa') | (filtered_df['property_type']=='Townhouse')]
house
house.columns
house = pd.get_dummies(house, columns=["suburb"])
house
house = pd.get_dummies(house, columns=["property_type"])
house
# for i in range(0,len(house['sale_id'])):
#     if house.iloc[i,13] =='Villa':
#         house.iloc[i,26] = 1
#     elif house.iloc[i,13] =='Townhouse':
#         house.iloc[i,13]= 2
#     elif house.iloc[i,13] =='House':
#         house.iloc[i,13] = 3
    
# #     house.iloc[i,15] = int(house.iloc[i,15].split('-')[0])
# #     if house.iloc[i,15]== 1900:
# #         house.iloc[i,15] = 0
# house
# house['median_income'] = ''
# for i in range(0,len(house['sale_id'])):
#     if house.iloc[i,19] =='Perth':
#         house.iloc[i,20] = 1750
#     elif house.iloc[i,19] =='Crawley':
#         house.iloc[i,20]= 1145
#     elif house.iloc[i,19] =='Nedlands':
#         house.iloc[i,20] = 2217
#     elif house.iloc[i,19] =='Northbridge':
#         house.iloc[i,20] = 1385
#         elif house.iloc[i,19] =='Northbridge':
#         house.iloc[i,20] = 1385
# house['perth'] = ''
# house['east_perth'] = ''
# house['west_perth'] = ''
# house['northbridge'] = ''
# house['crawley'] = ''
# house['nedlands'] = ''
# house['Villa'] = ''
# house['Townhouse'] = ''
# house['House'] = ''
# for i in range(0,len(house['sale_id'])):
#     if house.iloc[i,19] =='Perth':
#         house.iloc[i,20] = 1
#         house.iloc[i,21:26] = 0
#     elif house.iloc[i,19] =='East Perth':
#         house.iloc[i,21] = 1
#         house.iloc[i,20] = 0
#         house.iloc[i,22:26] = 0
#     elif house.iloc[i,19] =='West Perth':
#         house.iloc[i,22] = 1
#         house.iloc[i,20:22] = 0
#         house.iloc[i,23:26] = 0
#     elif house.iloc[i,19] =='Northbridge':
#         house.iloc[i,23] = 1
#         house.iloc[i,20:23] = 0
#         house.iloc[i,24:26] = 0
#     elif house.iloc[i,19] =='Crawley':
#         house.iloc[i,24] = 1
#         house.iloc[i,20:24] = 0
#         house.iloc[i,25] = 0
#     elif house.iloc[i,19] =='Nedlands':
#         house.iloc[i,25] = 1
#         house.iloc[i,20:25] = 0
        
#     if house.iloc[i,13] =='Villa':
#         house.iloc[i,26] = 1
#         house.iloc[i,27:29] = 0
#     elif house.iloc[i,13] =='Townhouse':
#         house.iloc[i,27]= 1
#         house.iloc[i,26] = 0
#         house.iloc[i,28] = 0
#     elif house.iloc[i,13] =='House':
#         house.iloc[i,28] = 1
#         house.iloc[i,26:28] = 0

# house
# Assign the data to X and y
X = house.drop("price",axis=1)
y = house["price"].values.reshape(-1, 1)
print(X.shape, y.shape)
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=70)
X_train.shape,X_test.shape
import seaborn  as sns
plt.figure(figsize=(12,10))
cor = X_train.corr()
sns.heatmap(cor, annot = True, cmap= plt.cm.CMRmap_r)
def correlation(dataset, threshold):
    col_corr = set()
    corr_matrix = dataset.corr()
    for i in range(len(corr_matrix.columns)):
        for j in range(i):
            if abs(corr_matrix.iloc[i,j]) > threshold:
                colname = corr_matrix.columns[i]
                col_corr.add(colname)
    return col_corr
corr_features = correlation(X_train, 0.9)
len(set(corr_features))
corr_features
X_train = X_train.drop(corr_features, axis=1)
X_train.columns
X_train = X_train[['bathrooms', 'bedrooms', 'building_size',
       'built_date', 'car_space', 'land_size', 'lat', 'lng', 'rent', 'rent_date', 'sold_date', 'suburb_Crawley', 'suburb_East Perth', 'suburb_Nedlands',
       'suburb_Northbridge', 'suburb_Perth', 'suburb_West Perth',
       'property_type_House', 'property_type_Villa']]
X_train
X_test = X_test[['bathrooms', 'bedrooms', 'building_size',
       'built_date', 'car_space', 'land_size', 'lat', 'lng', 'rent', 'rent_date', 'sold_date', 'suburb_Crawley', 'suburb_East Perth', 'suburb_Nedlands',
       'suburb_Northbridge', 'suburb_Perth', 'suburb_West Perth',
       'property_type_House', 'property_type_Villa']]
X_test
# house.sort_values(by = ['price'], ascending=False)
# # Assign the data to X and y
# X = house[["bedrooms", "bathrooms", "car_space", "land_size", "building_size", "built_date", "postcode"]]
# y = house["price"].values.reshape(-1, 1)
# print(X.shape, y.shape)
# # Assign the data to X and y
# X = house[["bedrooms", "bathrooms", "car_space", "land_size", "building_size", "built_date", "perth", "west_perth", "east_perth", "northbridge", "crawley", "nedlands"]]
# y = house["price"].values.reshape(-1, 1)
# print(X.shape, y.shape)
# from sklearn.model_selection import train_test_split

# X_train, X_test, y_train, y_test = train_test_split(X, y)
from sklearn.preprocessing import StandardScaler

# Create a StandardScater model and fit it to the training data

X_scaler = StandardScaler().fit(X_train)
y_scaler = StandardScaler().fit(y_train)
# Transform the training and testing data using the X_scaler and y_scaler models

X_train_scaled = X_scaler.transform(X_train)
X_test_scaled = X_scaler.transform(X_test)
y_train_scaled = y_scaler.transform(y_train)
y_test_scaled = y_scaler.transform(y_test)
# Create the model using LinearRegression

from sklearn.linear_model import LinearRegression
model = LinearRegression()
model.fit(X_train_scaled, y_train_scaled)

# #evaluate the model (intercept and slope)
# print(model.intercept_)
# print(model.coef_)
# Make predictions using a fitted model
predictions = model.predict(X_test_scaled)
model.fit(X_train_scaled, y_train_scaled)
plt.scatter(model.predict(X_train_scaled), y_train_scaled - model.predict(X_train_scaled), c="blue", label="Training Data")
plt.scatter(model.predict(X_test_scaled), y_test_scaled - model.predict(X_test_scaled), c="orange", label="Testing Data")
plt.legend()
plt.hlines(y=0, xmin=y_test_scaled.min(), xmax=y_test_scaled.max())
plt.title("Residual Plot")
plt.show()
#compare actual output values with predicted values
df1 = X_test
df1['Actual'] = y_test.reshape(1,-1)[0]
df1['Linear_Regression_Predicted'] = y_scaler.inverse_transform(model.predict(X_test_scaled))
df1.head(10)
# Fit the model to the training data and calculate the scores for the training and testing data

training_score = model.score(X_train_scaled, y_train_scaled)
testing_score = model.score(X_test_scaled, y_test_scaled)

print(f"Training Score: {training_score}")
print(f"Testing Score: {testing_score}")
# Used X_test_scaled, y_test_scaled, and model.predict(X_test_scaled) to calculate MSE and R2

from sklearn.metrics import mean_squared_error

MSE = mean_squared_error(y_test_scaled, predictions)
r2 = model.score(X_test_scaled, y_test_scaled)

print(f"MSE: {MSE}, R2: {r2}")
#test
# suburb needs to be categorical
X_test = X_scaler.transform([[4,3,2,175,186,2019,0,0,0,0,1,0]])
predictions = model.predict(X_test)
results = y_scaler.inverse_transform(predictions)
results<jupyter_output><empty_output><jupyter_text>LASSO model<jupyter_code># LASSO model
# Note: Use an alpha of .01 when creating the model for this activity
from sklearn.linear_model import Lasso


lasso = Lasso(alpha=.01).fit(X_train_scaled, y_train_scaled)

lasso_predictions = lasso.predict(X_test_scaled)

MSE = mean_squared_error(y_test_scaled, lasso_predictions) #error to a model (closer to 0 the better)
r2 = lasso.score(X_test_scaled, y_test_scaled) #nearer to 1 the better

print(f"MSE: {MSE}, R2: {r2}")
# find optimal alpha with grid search
alpha = [0.001, 0.01, 0.1, 1, 10, 100, 1000]

param_grid = dict(alpha=alpha)

lasso_grid = GridSearchCV(estimator=lasso, param_grid=param_grid, scoring='r2', verbose=1, n_jobs=-1)

lasso_grid_result = lasso_grid.fit(X_train_scaled, y_train_scaled)

# lasso_grid_predictions = lasso_grid_model(X_test_scaled)

# MSE = mean_squared_error(y_test_scaled, lasso_grid_predictions) #error to a model (closer to 0 the better)
# r2 = lasso_grid_model.score(X_test_scaled, y_test_scaled) #nearer to 1 the better

# print(f"MSE: {MSE}, R2: {r2}")
print('Best Score: ', lasso_grid_result.best_score_)
print('Best Params: ', lasso_grid_result.best_params_)
best_lasso = Lasso(alpha=0.001).fit(X_train_scaled, y_train_scaled)

best_lasso_predictions = best_lasso.predict(X_test_scaled)

MSE = mean_squared_error(y_test_scaled, best_lasso_predictions)
r2 = best_lasso.score(X_test_scaled, y_test_scaled)

print(f"MSE: {MSE}, R2: {r2}")
df1['Lasso_Predicted'] = y_scaler.inverse_transform(best_lasso.predict(X_test_scaled))
df1.head(10)<jupyter_output><ipython-input-50-96dcefbde04c>:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  df1['Lasso_Predicted'] = y_scaler.inverse_transform(best_lasso.predict(X_test_scaled))<jupyter_text>Ridge model<jupyter_code># Ridge model
# Note: Use an alpha of .01 when creating the model for this activity
from sklearn.linear_model import Ridge


ridge = Ridge(alpha=.01).fit(X_train_scaled, y_train_scaled)

ridge_predictions = ridge.predict(X_test_scaled)

MSE = mean_squared_error(y_test_scaled, ridge_predictions)
r2 = ridge.score(X_test_scaled, y_test_scaled)

print(f"MSE: {MSE}, R2: {r2}")
# find optimal alpha with grid search
alpha = [0.001, 0.01, 0.1, 1, 10, 100, 1000]

param_grid = dict(alpha=alpha)

ridge_grid = GridSearchCV(estimator=ridge, param_grid=param_grid, scoring='r2', verbose=1, n_jobs=-1)

ridge_grid_result = ridge_grid.fit(X_train_scaled, y_train_scaled)

# ridge_grid_predictions = ridge_grid(X_test_scaled)

# MSE = mean_squared_error(y_test_scaled, ridge_grid_predictions) #error to a model (closer to 0 the better)
# r2 = ridge_grid.score(X_test_scaled, y_test_scaled) #nearer to 1 the better

# print(f"MSE: {MSE}, R2: {r2}")
print('Best Score: ', ridge_grid_result.best_score_)
print('Best Params: ', ridge_grid_result.best_params_)
best_ridge = Ridge(alpha=10).fit(X_train_scaled, y_train_scaled)

best_ridge_predictions = best_ridge.predict(X_test_scaled)

MSE = mean_squared_error(y_test_scaled, best_ridge_predictions)
r2 = best_ridge.score(X_test_scaled, y_test_scaled)

print(f"MSE: {MSE}, R2: {r2}")
df1['Ridge_Predicted'] = y_scaler.inverse_transform(best_ridge.predict(X_test_scaled))
df1.head(10)<jupyter_output><ipython-input-56-4ab57f1676a3>:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  df1['Ridge_Predicted'] = y_scaler.inverse_transform(best_ridge.predict(X_test_scaled))<jupyter_text>ElasticNet model<jupyter_code># ElasticNet model
# Note: Use an alpha of .01 when creating the model for this activity
from sklearn.linear_model import ElasticNet

elasticnet = ElasticNet(alpha=.01).fit(X_train_scaled, y_train_scaled)

elas_predictions = elasticnet.predict(X_test_scaled)

MSE = mean_squared_error(y_test_scaled, elas_predictions)
r2 = elasticnet.score(X_test_scaled, y_test_scaled)

print(f"MSE: {MSE}, R2: {r2}")
# find optimal alpha with grid search
alpha = [0.001, 0.01, 0.1, 1, 10, 100, 1000]

param_grid = dict(alpha=alpha)

elasticnet_grid = GridSearchCV(estimator=elasticnet, param_grid=param_grid, scoring='r2', verbose=1, n_jobs=-1)

elasticnet_grid_result= elasticnet_grid.fit(X_train_scaled, y_train_scaled)

# elasticnet_grid_predictions = elasticnet_grid(X_test_scaled)

# MSE = mean_squared_error(y_test_scaled, elasticnet_grid_predictions) #error to a model (closer to 0 the better)
# r2 = elasticnet_grid.score(X_test_scaled, y_test_scaled) #nearer to 1 the better

# print(f"MSE: {MSE}, R2: {r2}")
print('Best Score: ', elasticnet_grid_result.best_score_)
print('Best Params: ', elasticnet_grid_result.best_params_)
best_elasticnet = ElasticNet(alpha=0.001).fit(X_train_scaled, y_train_scaled)

best_elasticnet_predictions = best_elasticnet.predict(X_test_scaled)

MSE = mean_squared_error(y_test_scaled, best_elasticnet_predictions)
r2 = best_elasticnet.score(X_test_scaled, y_test_scaled)

print(f"MSE: {MSE}, R2: {r2}")
df1['elasticnet_Predicted'] = y_scaler.inverse_transform(best_elasticnet.predict(X_test_scaled))
df1.head(10)<jupyter_output><ipython-input-60-30feeaf9fe4a>:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  df1['elasticnet_Predicted'] = y_scaler.inverse_transform(best_elasticnet.predict(X_test_scaled))<jupyter_text>Save the best Model<jupyter_code>import joblib
joblib.dump(best_lasso, "best_model.pkl")
my_model = joblib.load("best_model.pkl")

X_test = X_scaler.transform([[4,3,2,175,186,2019,0,0,0,0,1,0]])
predictions = my_model.predict(X_test)
results = y_scaler.inverse_transform(predictions)
results
y_pred = my_model.predict(X_test_scaled)
from sklearn import metrics
print('MAE:', metrics.mean_absolute_error(y_test_scaled, y_pred))  
print('MSE:', metrics.mean_squared_error(y_test_scaled, y_pred))  
print('RMSE:', np.sqrt(metrics.mean_squared_error(y_test_scaled, y_pred)))
print('VarScore:',metrics.explained_variance_score(y_test_scaled,y_pred))
# Visualizing Our predictions
fig = plt.figure(figsize=(10,5))
plt.scatter(y_test_scaled,y_pred)
# Perfect predictions
plt.plot(y_test_scaled,y_test_scaled,'r')<jupyter_output>MAE: 0.26162743760137125
MSE: 0.11450046069356608
RMSE: 0.33837916705016885
VarScore: 0.8591439426886013
{'fraction_non_alphanumeric': 0.14232864733974573, 'fraction_numerical': 0.03692742315194727, 'mean_word_length': 5.845303867403315, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 26, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '534090', 'license': 'MIT', 'n_tokens_mistral': 6243, 'n_tokens_neox': 5781, 'n_words': 1051, 'path': 'Notebooks/ML-LinearRegression-Models-with coorelation.ipynb', 'repo_name': 'helenamin/Project3', 'size': '14867'}