<jupyter_start><jupyter_text>Housing Prices WorksheetPredicts Housing Prices Using Housing Prices.<jupyter_code>import numpy as np
import pandas as pd

from scipy import stats
import statsmodels.api as sm 
import matplotlib.pyplot as plt

from sklearn.metrics import mean_absolute_error
from sklearn.tree import DecisionTreeRegressor

from sklearn.model_selection import train_test_split
from sklearn.impute import SimpleImputer<jupyter_output><empty_output><jupyter_text>Importing the Datasets<jupyter_code>train_set = pd.read_csv(r"C:\Users\<NAME>\Desktop\Housing Prices\Housing Price\train.csv")
test_set = pd.read_csv(r"C:\Users\<NAME>\Desktop\Housing Prices\Housing Price\test.csv")<jupyter_output><empty_output><jupyter_text>Display Training Datasets<jupyter_code>train_set.describe()<jupyter_output><empty_output><jupyter_text>Replace Missing Values<jupyter_code>features = ['LotArea', "LotFrontage", 'OverallQual', 'BedroomAbvGr', 'TotRmsAbvGrd', 'GarageArea', "YearBuilt", 'MSSubClass', "WoodDeckSF", "GrLivArea", "TotalBsmtSF", "YearRemodAdd", "OverallCond"]
# Fill in the lines below: imputation
my_imputer = SimpleImputer()# Your code here

train_set = train_set.fillna(0)

imputed_X_train = pd.DataFrame(my_imputer.fit_transform(train_set[features]))
imputed_X_valid = pd.DataFrame(my_imputer.transform(test_set[features]))

# Fill in the lines below: imputation removed column names; put them back

imputed_X_valid.columns = test_set[features].columns
train_set.columns = train_set.columns<jupyter_output><empty_output><jupyter_text>Defining Functions<jupyter_code>def tooLarge(lotArea, amount):
    tooLarge = []
    tooLarge_ID = []
    i = 0
    for i in range(len(lotArea)):
        try:
            if lotArea[i] > amount:
                tooLarge_ID.append(i)
        except KeyError:
            i = 1 + 1
                
    return tooLarge_ID

def printMatches(dataset, ids):
    ids = np.array(ids)
    print(dataset.iloc[ids])       
    return

def neighCoding(neighbor):
    neighbor = neighbor.copy()
    misspelling = []
    for n in range(len(neighbor)):
        try:
            if neighbor.iloc[n] == "Blmngtn":
                neighbor.iloc[n] = 1
            elif neighbor.iloc[n] == "Blueste":
                neighbor.iloc[n] = 2
            elif neighbor.iloc[n] == "BrDale":
                neighbor.iloc[n] = 3
            elif neighbor.iloc[n] == "BrkSide":
                neighbor.iloc[n] = 4
            elif neighbor.iloc[n] == "ClearCr":
                neighbor.iloc[n] = 5
            elif neighbor.iloc[n] == "CollgCr":
                neighbor.iloc[n] = 6
            elif neighbor.iloc[n] == "Crawfor":
                neighbor.iloc[n] = 7
            elif neighbor.iloc[n] == "Edwards":
                neighbor.iloc[n] = 8
            elif neighbor.iloc[n] == "Gilbert":
                neighbor.iloc[n] = 9
            elif neighbor.iloc[n] == "IDOTRR":
                neighbor.iloc[n] = 10
            elif neighbor.iloc[n] == "MeadowV":
                neighbor.iloc[n] = 11
            elif neighbor.iloc[n] == "Mitchel":
                neighbor.iloc[n] = 12
            elif neighbor.iloc[n] == "NAmes":
                neighbor.iloc[n] = 13
            elif neighbor.iloc[n] == "NoRidge":
                neighbor.iloc[n] = 14
            elif neighbor.iloc[n] == "NPkVill":
                neighbor.iloc[n] = 15
            elif neighbor.iloc[n] == "NridgHt":
                neighbor.iloc[n] = 16
            elif neighbor.iloc[n] == "NWAmes":
                neighbor.iloc[n] = 17
            elif neighbor.iloc[n] == "OldTown":
                neighbor.iloc[n] = 18
            elif neighbor.iloc[n] == "SWISU":
                neighbor.iloc[n] = 19
            elif neighbor.iloc[n] == "Sawyer":
                neighbor.iloc[n] = 20
            elif neighbor.iloc[n] == "SawyerW":
                neighbor.iloc[n] = 21
            elif neighbor.iloc[n] == "Somerst":
                neighbor.iloc[n] = 22
            elif neighbor.iloc[n] == "StoneBr":
                neighbor.iloc[n] = 23
            elif neighbor.iloc[n] == "Timber":
                neighbor.iloc[n] = 24
            elif neighbor.iloc[n] == "Veenker":
                neighbor.iloc[n] = 25
        except KeyError:
            n = n + 1
    return neighbor<jupyter_output><empty_output><jupyter_text>Recoding Neighborhood Data For Analysis<jupyter_code>train_set["Neighborhood"] = neighCoding(train_set["Neighborhood"])<jupyter_output><empty_output><jupyter_text>Plotting Neighborhood Data with Price Data<jupyter_code>train_set["Neighborhood"].head()
plt.figure(figsize = (10,8))
plt.scatter(train_set["LotFrontage"], train_set["SalePrice"])
plt.show<jupyter_output><empty_output><jupyter_text>Finding Outliers in 1st Floor Area<jupyter_code>large1stArea = []
large1stArea = (tooLarge(train_set["1stFlrSF"], 4000))
printMatches(train_set, large1stArea)
plt.figure(figsize = (10,8))
plt.boxplot(train_set["YearRemodAdd"])
plt.show()<jupyter_output><empty_output><jupyter_text>Finding Outliers with Lot Area<jupyter_code>largeLotArea = []
largeLotArea = (tooLarge(train_set["LotArea"], 100000))
printMatches(train_set, largeLotArea)<jupyter_output>Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \
249  250          50       RL          0.0   159000   Pave     0      IR2   
313  314          20       RL        150.0   215245   Pave     0      IR3   
335  336         190       RL          0.0   164660   Grvl     0      IR1   
706  707          20       RL          0.0   115149   Pave     0      IR2   

    LandContour Utilities  ... PoolArea PoolQC  Fence MiscFeature MiscVal  \
249         Low    AllPub  ...        0      0      0        Shed     500   
313         Low    AllPub  ...        0      0      0           0       0   
335         HLS    AllPub  ...        0      0      0        Shed     700   
706         Low    AllPub  ...        0      0      0           0       0   

    MoSold YrSold  SaleType  SaleCondition  SalePrice  
249      6   2007        WD         Normal     277000  
313      6   2009        WD         Normal     375000  
335      8   2008        WD         Normal     228950  
706 [...]<jupyter_text>Finding Frontage Outliers<jupyter_code>largeFrontage = []
largeFrontage = (tooLarge(train_set["LotFrontage"], 300))
printMatches(train_set, largeFrontage)<jupyter_output>Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \
934    935          20       RL        313.0    27650   Pave     0      IR2   
1298  1299          60       RL        313.0    63887   Pave     0      IR3   

     LandContour Utilities  ... PoolArea PoolQC  Fence MiscFeature MiscVal  \
934          HLS    AllPub  ...        0      0      0           0       0   
1298         Bnk    AllPub  ...      480     Gd      0           0       0   

     MoSold YrSold  SaleType  SaleCondition  SalePrice  
934      11   2008        WD         Normal     242000  
1298      1   2008       New        Partial     160000  

[2 rows x 81 columns]<jupyter_text>Finding Outliers in Price Data<jupyter_code>highPricesID = []
highPricesID = tooLarge(train_set["SalePrice"], 700000)
printMatches(train_set, highPricesID)

train_set["OverallQual"].iloc[highPricesID]<jupyter_output>Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \
691    692          60       RL        104.0    21535   Pave     0      IR1   
1182  1183          60       RL        160.0    15623   Pave     0      IR1   

     LandContour Utilities  ... PoolArea PoolQC  Fence MiscFeature MiscVal  \
691          Lvl    AllPub  ...        0      0      0           0       0   
1182         Lvl    AllPub  ...      555     Ex  MnPrv           0       0   

     MoSold YrSold  SaleType  SaleCondition  SalePrice  
691       1   2007        WD         Normal     755000  
1182      7   2007        WD        Abnorml     745000  

[2 rows x 81 columns]<jupyter_text>Dropping the outlier rows<jupyter_code>train_set = train_set.drop(highPricesID)
train_set = train_set.drop(largeLotArea)
train_set = train_set.drop(large1stArea)

sum(train_set["MSSubClass"].isna())<jupyter_output><empty_output><jupyter_text>Seperating Training data with the labels<jupyter_code>X = train_set.drop(['Id'], axis=1)
Y = train_set["SalePrice"]
train, test, train_labels, test_labels = train_test_split(X, Y, test_size = 0.30)
train_labels = np.array(train_labels)<jupyter_output><empty_output><jupyter_text>Testing Regression Scenario One: All Variables<jupyter_code>features = ['LotArea', 'LotFrontage', 'OverallQual', 'BedroomAbvGr', 'TotRmsAbvGrd', 'GarageArea', "YearBuilt", 'MSSubClass', "WoodDeckSF", "GrLivArea", "TotalBsmtSF", "YearRemodAdd", "OverallCond"]

X1 = sm.add_constant(train[features])
reg = sm.OLS(train_labels, X1).fit()

reg.summary()<jupyter_output>C:\Users\<NAME>\Anaconda3\lib\site-packages\numpy\core\fromnumeric.py:2389: FutureWarning: Method .ptp is deprecated and will be removed in a future version. Use numpy.ptp instead.
  return ptp(axis=axis, out=out, **kwargs)<jupyter_text>Scenario Two<jupyter_code>X1 = sm.add_constant((train[["LotFrontage"]]))
reg = sm.OLS(train_labels, X1).fit()

reg.summary()<jupyter_output><empty_output><jupyter_text>Random Forests<jupyter_code>iowa_model = DecisionTreeRegressor(random_state=1)
# Fit Model
iowa_model.fit(train[features], train_labels)

# Make validation predictions and calculate mean absolute error
val_predictions = iowa_model.predict(test[features])
val_mae = mean_absolute_error(val_predictions, test_labels)
print("Validation MAE when not specifying max_leaf_nodes: {:,.0f}".format(val_mae))
# Using best value for max_leaf_nodes
iowa_model = DecisionTreeRegressor(max_leaf_nodes=100, random_state=1)
iowa_model.fit(train[features], train_labels)

val_predictions = iowa_model.predict(test[features])
val_mae = mean_absolute_error(val_predictions, test_labels)
print("Validation MAE for best value of max_leaf_nodes: {:,.0f}".format(val_mae))<jupyter_output>Validation MAE for best value of max_leaf_nodes: 23,880<jupyter_text>Preparing Submissions<jupyter_code>#sub_data["Neighborhood"] = neighCoding(sub_data["Neighborhood"])


sub_predictions = iowa_model.predict(imputed_X_valid)
sub_predictions
sub_predictions = pd.DataFrame(sub_predictions, index = test_set["Id"], columns = ["SalePrice"])
sub_predictions.head()<jupyter_output><empty_output><jupyter_text>Export Submission<jupyter_code>sub_predictions.to_csv(r"C:\Users\<NAME>\Desktop\Housing Prices\Housing Price\Submission.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10241820768136557, 'fraction_numerical': 0.0413466097676624, 'mean_word_length': 2.6541926541926544, 'pattern_counts': {'":': 25, '<': 72, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '518369', 'license': 'MIT', 'n_tokens_mistral': 3892, 'n_tokens_neox': 3556, 'n_words': 725, 'path': 'Housing Prices Code.ipynb', 'repo_name': 'malcolmRite/House-Price-Predictions', 'size': '10577'}