<jupyter_start><jupyter_text>Real estate cost and price prediction  Goal: Predict real estate construction cost and sale prices using linear regression<jupyter_code>%matplotlib inline

# Imports
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>raw_data = pd.read_excel('Residential-Building-Data-Set.xlsx',header=1)

# Construct dataframe with target feature labels decoded and placed in the first two columns
price = raw_data['V-9']
cost = raw_data['V-10']
raw_data.drop(labels=['V-9','V-10'], axis=1,inplace = True)
re_df = raw_data
re_df.insert(0, 'Price', price)
re_df.insert(0, 'Cost', cost)
re_df.head()<jupyter_output><empty_output><jupyter_text>Visualize the distribution of the two target variables and relationship to each other<jupyter_code>plt.figure(figsize=(16,5))
ax1 = plt.subplot(131);
plt.hist(re_df['Cost'],bins=20);
ax1.set_title('Cost');
ax1.set_xlabel('Rial (10,000s)');
ax1.set_ylabel('Count');

ax2 = plt.subplot(132);
plt.hist(re_df['Price'], bins=20);
ax2.set_title('Price');
ax2.set_xlabel('Rial (10,000s)');
ax2.set_ylabel('Count');

ax3 = plt.subplot(133);
plt.scatter(re_df['Cost'], re_df['Price']);
ax3.set_title('Cost vs. Price');
ax3.set_xlabel('Cost in Rial (10,000s)');
ax3.set_ylabel('Price in Rial (10,000s)');<jupyter_output><empty_output><jupyter_text>Visualize relationship between one potential predictor and target variables<jupyter_code>plt.figure(figsize=(12,5))
ax1 = plt.subplot(121);
plt.scatter(re_df['V-3'],re_df['Cost']);
ax1.set_xlabel('Floor area of building (m$^2$)');
ax1.set_ylabel('Cost in Rial (10,000s)');
ax1.set_xlim(0,2000);

ax2 = plt.subplot(122);
plt.scatter(re_df['V-3'],re_df['Price']);
ax2.set_xlabel('Floor area of building (m$^2$)');
ax2.set_ylabel('Price in Rial (10,000s)');
ax2.set_xlim(0,2000);<jupyter_output><empty_output><jupyter_text>Create an initial regression model<jupyter_code># Create a linear regression object to predict cost and price
X = raw_data.drop(labels=['Cost','Price'], axis=1)
cost_lm = LinearRegression()
cost_lm.fit(X,cost);
price_lm = LinearRegression()
price_lm.fit(X,price);
# Create a df to store the coefficients
cost_coef = pd.DataFrame(zip(list(X),cost_lm .coef_), columns=['Feature','Coefficient'])
price_coef = pd.DataFrame(zip(list(X),price_lm .coef_), columns=['Feature','Coefficient'])
cost_coef.head() # View the coefficients
# Predict using the model
cost_pred = cost_lm.predict(X)
price_pred = price_lm.predict(X)
# Visualize predicted and actual cost and price
plt.figure(figsize=(12,5))
ax1 = plt.subplot(121);
plt.scatter(cost,cost_pred);
ax1.set_xlabel('Actual cost in Rials (10,000s)');
ax1.set_ylabel('Predicted cost in Rials (10,000s)');

ax2 = plt.subplot(122);
plt.scatter(price,price_pred);
ax2.set_xlabel('Actual price in Rials (10,000s)');
ax2.set_ylabel('Predicted price in Rials (10,000s)');
# Calculate mean square error and root mean square error
mse_cost = np.mean((cost_pred-cost)**2)
rmse_cost = np.sqrt(mse_cost)

mse_price = np.mean((price_pred-price)**2)
rmse_price = np.sqrt(mse_price)<jupyter_output><empty_output><jupyter_text>Refit and evaluate model using training and validation sets<jupyter_code>X_cost_train, X_cost_test, Y_cost_train, Y_cost_test = train_test_split(X,cost, test_size=0.25, random_state = 42)
X_price_train, X_price_test, Y_price_train, Y_price_test = train_test_split(X,price, test_size=0.25, random_state = 42)

# Retrain models using only train data set
cost_lm.fit(X_cost_train,Y_cost_train)
price_lm.fit(X_price_train,Y_price_train)

# Predict using test data set
cost_pred = cost_lm.predict(X_cost_test)
price_pred = price_lm.predict(X_price_test)
# Visualize predicted and actual values for cost and price
plt.figure(figsize=(12,5))
ax1 = plt.subplot(121);
plt.scatter(Y_cost_test,cost_pred);
ax1.set_xlabel('Actual cost in Rials (10,000s)');
ax1.set_ylabel('Predicted cost in Rials (10,000s)');

ax2 = plt.subplot(122);
plt.scatter(Y_price_test,price_pred);
ax2.set_xlabel('Actual price in Rials (10,000s)');
ax2.set_ylabel('Predicted price in Rials (10,000s)');
# Calculate mean square error and root mean square error
mse_cost = np.mean((cost_pred-Y_cost_test)**2)
rmse_cost = np.sqrt(mse_cost)

mse_price = np.mean((price_pred-Y_price_test)**2)
rmse_price = np.sqrt(mse_price)

print 'The RMSE of cost is ' + str(round(rmse_cost,2)) + ' and of price is ' + str(round(rmse_price,2))
# Plot residuals
plt.figure(figsize=(12,5))
ax1 = plt.subplot(121);
plt.scatter(cost_pred,Y_cost_test - cost_pred);
plt.hlines(y=0,xmin=0,xmax=700)
ax1.set_xlabel('Predicted cost in Rials (10,000s)');
ax1.set_ylabel('Residuals');
ax1.set_ylim(-110,110)

ax2 = plt.subplot(122);
plt.scatter(price_pred,Y_price_test - price_pred);
plt.hlines(y=0,xmin=0,xmax=6000)
ax2.set_xlabel('Predicted price in Rials (10,000s)');
ax2.set_ylabel('Residuals');
ax2.set_ylim(-800,800);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14724534284581847, 'fraction_numerical': 0.04676971858898137, 'mean_word_length': 7.964476021314387, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '303647', 'license': 'MIT', 'n_tokens_mistral': 2102, 'n_tokens_neox': 1969, 'n_words': 384, 'path': 'Real_estate_regression_nb.ipynb', 'repo_name': 'mostofa-ahsan/Financial-Data-Analysis', 'size': '5046'}