<jupyter_start><jupyter_text>Data Cleaning<jupyter_code># total_bedrooms has missing values, we can fix that with one of these next three options
# option 1 ~ this will drop the rows from entire dataframe where the column/feature "total_bedrooms" is empty
housing.dropna(subset=["total_bedrooms"])


# option 2 ~ this will remove the entire column/feature from dataframe
housing.drop("total_bedrooms", axis=1)

# option 3 ~ this will calculate the median value for the 'total_bedroom' column/feature and replace any missing values with...
#                the calculated median value. The 'inplace' flag must be true or your dataframe will not make the changes to...
#                your dataframe but will just execute the command which. The 'inplace' flag will apply the changes directly...
#                to your dataframe
median = housing["total_bedrooms"].median()
housing["total_bedrooms"].fillna(median, inplace=True)
# sklearn has an alternate method for filling in missing values
from sklearn.impute import SimpleImputer

# First we must create an instance of the SimpleImputer class
imputer = SimpleImputer(strategy="median")

# SimpleImputer will fail if there exists categorical data/labels, we must remove to use SimpleImputer
housing_num = housing.drop("ocean_proximity", axis=1)

# We must fit the Imputer to the remaining all numberical dataframe
imputer.fit(housing_num)

print("Imputer Statistics:\n{}\n".format(imputer.statistics_))
print("Dataframe Statistics:\n{}\n".format(housing_num.median().values))

# Now we can replace the missing values with the trained values of the imputer, this will return a numpy array
X = imputer.transform(housing_num)

# Put numpy array back into a dataframe
housing_tr = pd.DataFrame(X, columns=housing_num.columns, index=housing_num.index)<jupyter_output>Imputer Statistics:
[-118.51     34.26     29.     2119.5     433.     1164.      408.
    3.5409]

Dataframe Statistics:
[-118.51     34.26     29.     2119.5     433.     1164.      408.
    3.5409]<jupyter_text>Categorical Data<jupyter_code>housing_cat = housing[["ocean_proximity"]]
housing_cat.head(10)
# Convert the categorical data in the 'ocean_proximity' column/feature into numerical
from sklearn.preprocessing import OrdinalEncoder
ordinal_encoder = OrdinalEncoder()
housing_cat_encoded = ordinal_encoder.fit_transform(housing_cat)
print("The first 10 values in housing_cat_encoded: \n{}".format(housing_cat_encoded[:10]))
# This will then allow you to associate those categorical values into numerical and the OrdinalEncoder...
#   saves the mapping between the two so you are able to go back and forth as needed.
print("Ordinal Encorder Categories:\n{}\n".format(ordinal_encoder.categories_))
# Machine learning will assume distance between values corresponds to similarity. So it would assume that "<1H OCEAN"...
#    is more similar to "INLAND" when it is much more similar to "NEAR OCEAN".
# To Solve this we can one-hot encode 
from sklearn.preprocessing import OneHotEncoder
cat_encoder = OneHotEncoder()

# housing_cat_1hot will be returned as a sparse array ~ thus the 0 data is not included, only locations where non-zero...
#    this will save memory as a vast majority of the array will be 0's. You can convert into a dense array, one which...
#    contains the 0's populated by using "housing_cat_1hot.toarray()"
housing_cat_1hot = cat_encoder.fit_transform(housing_cat)

# The categories are available using the fit encoder
print("One Hot Encoder Categories:\n{}\n".format(cat_encoder.categories_))<jupyter_output>One Hot Encoder Categories:
[array(['<1H OCEAN', 'INLAND', 'ISLAND', 'NEAR BAY', 'NEAR OCEAN'],
      dtype=object)]<jupyter_text>Custom Transformers<jupyter_code>from sklearn.base import BaseEstimator, TransformerMixin
rooms_ix, bedrooms_ix, population_ix, households_ix = 3, 4, 5, 6

class CombinedAttributesAdder(BaseEstimator, TransformerMixin):
    def __init__(self, add_bedrooms_per_room=True):
        self.add_bedrooms_per_room = add_bedrooms_per_room
    def fit(self, X, y=None):
        return self
    def transform(self, X):
        rooms_per_household = X[:, rooms_ix] / X[:, households_ix]
        population_per_household = X[:, population_ix] / X[:, households_ix]
        if self.add_bedrooms_per_room:
            bedrooms_per_room = X[:, bedrooms_ix] / X[:, rooms_ix]
            return np.c_[X, rooms_per_household, population_per_household, bedrooms_per_room]
        else:
            return np.c_[X, rooms_per_household, population_per_household]
attr_adder = CombinedAttributesAdder(add_bedrooms_per_room=False)
housing_extra_attribs = attr_adder.transform(housing.values)
print("Housing Extra Attribs:\n{}".format(housing_extra_attribs))<jupyter_output>Housing Extra Attribs:
[[-121.89 37.29 38.0 ... '<1H OCEAN' 4.625368731563422 2.094395280235988]
 [-121.93 37.05 14.0 ... '<1H OCEAN' 6.008849557522124 2.7079646017699117]
 [-117.2 32.77 31.0 ... 'NEAR OCEAN' 4.225108225108225 2.0259740259740258]
 ...
 [-116.4 34.09 9.0 ... 'INLAND' 6.34640522875817 2.742483660130719]
 [-118.01 33.82 31.0 ... '<1H OCEAN' 5.50561797752809 3.808988764044944]
 [-122.45 37.77 52.0 ... 'NEAR BAY' 4.843505477308295 1.9859154929577465]]<jupyter_text>Transformation Pipelines<jupyter_code>from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

# The Pipeline class takes a list of steps, this case we use the 'imputer' first, then 'attribs_adder', then "std_scaler"
num_pipeline = Pipeline([("imputer", SimpleImputer(strategy="median")),
                         ("attribs_adder", CombinedAttributesAdder()),
                         ("std_scaler", StandardScaler())])
housing_num_tr = num_pipeline.fit_transform(housing_num)
# Column based transformation pipelines

from sklearn.compose import ColumnTransformer

# List of numerical column names ~ just a list of housing_num which was a dataframe we made previously containing only numerical data
num_attribs = list(housing_num)

# List of categorical columns ~ since we only have one categorical column "ocean_proximity", we manually specify it
cat_attribs = ["ocean_proximity"]


# full_pipeline works by first calling our 'num_pipeline', we specified above. This will run through its pipeline first. Since...
#    this is the ColumnTransformer, it works on each column and then concatenates the results. Finally we use OneHotEncoding...
#    on the categorical column.
full_pipeline = ColumnTransformer([("num", num_pipeline, num_attribs),
                                   ("cat", OneHotEncoder(), cat_attribs)])

housing_prepared = full_pipeline.fit_transform(housing)<jupyter_output><empty_output><jupyter_text>Training a Model<jupyter_code># Linear Regression is used in this example but it would require there to be a linear relationship between the data...
#    for this algorithm to perform well. A root mean square error of 68628 is fairly high we can conclude a linear relationship...
#    may exist but to better capture the data/patterns we need more degrees of freedom.

from sklearn.linear_model import LinearRegression

lin_reg = LinearRegression()
lin_reg.fit(housing_prepared, housing_labels)

some_data = housing.iloc[:5]
some_labels = housing_labels.iloc[:5]
some_data_prepared = full_pipeline.transform(some_data)

print("\nPredictions:", lin_reg.predict(some_data_prepared))
print("\nLabels:", list(some_labels))

from sklearn.metrics import mean_squared_error

housing_predictions = lin_reg.predict(housing_prepared)
lin_mse = mean_squared_error(housing_labels, housing_predictions)
lin_rmse = np.sqrt(lin_mse)
print("\nLinear Regression Root Mean Square Error: {}\n".format(lin_rmse))
# Decision Tree ~ should be able to fit the data much better but typically will overfit without some form of prunning...
#    Prunning removes some specified number of leaves or even hold nodes, based on how the prunning is implemented...
#    Some will remove nodes where the information gain is not above a certain threshold, others may remove leaves which...
#    do not contain a thresholded number of instances.

from sklearn.tree import DecisionTreeRegressor

tree_reg = DecisionTreeRegressor()

# Fitting the model is training the model
tree_reg.fit(housing_prepared, housing_labels)

housing_predictions = tree_reg.predict(housing_prepared)
tree_mse = mean_squared_error(housing_labels, housing_predictions)
tree_rmse = np.sqrt(tree_mse)
print("\n Decision Tree Regression Root Mean Square Error: {}\n".format(tree_rmse))

# We can see the Decision Tree Regressor has a RMSE of 0.0 and that is being it does not have prunning implemented, thus...
#    has overfit the training data perfectly (which is bad).<jupyter_output>Decision Tree Regression Root Mean Square Error: 0.0<jupyter_text>Cross-Validation<jupyter_code># Cross-Validation will take a subset of the dataset and train the model on that, evaluate and compare. CV=10, means we...
#    will be doing this evaluation 10 times with 10 randomly selected subsets.
from sklearn.model_selection import cross_val_score

scores = cross_val_score(tree_reg, housing_prepared, housing_labels, scoring="neg_mean_squared_error", cv=10)
tree_rmse_scores = np.sqrt(-scores)
def display_scores(scores):
    print("\nScores: {}\n".format(scores))
    print("\nMean: {}\n".format(scores.mean()))
    print("\nStandard Deviation: {}\n".format(scores.std()))
display_scores(tree_rmse_scores)
lin_scores = cross_val_score(lin_reg, housing_prepared, housing_labels, scoring="neg_mean_squared_error", cv=10)

lin_rmse_scores = np.sqrt(-lin_scores)
display_scores(lin_rmse_scores)<jupyter_output>Scores: [66782.73843989 66960.118071   70347.95244419 74739.57052552
 68031.13388938 71193.84183426 64969.63056405 68281.61137997
 71552.91566558 67665.10082067]


Mean: 69052.46136345083


Standard Deviation: 2731.674001798349<jupyter_text>Ensemble Learning<jupyter_code>from sklearn.ensemble import RandomForestRegressor

forest_reg = RandomForestRegressor()
forest_reg.fit(housing_prepared, housing_labels)

forest_scores = cross_val_score(forest_reg, housing_prepared, housing_labels, scoring="neg_mean_squared_error", cv=10)
forest_rmse_scores = np.sqrt(-forest_scores)

display_scores(forest_rmse_scores)<jupyter_output>Scores: [49292.85950783 48055.31023134 49856.02411341 52290.46534913
 49588.4656239  53707.43457337 48821.56324338 48146.07333957
 53108.20778939 50475.7016243 ]


Mean: 50334.21053956099


Standard Deviation: 1925.0791231381<jupyter_text>GridSearch<jupyter_code>from sklearn.model_selection import GridSearchCV

param_grid = [{"n_estimators": [3, 10, 30], "max_features": [2, 4, 6, 8]}, 
              {"bootstrap": [False], "n_estimators": [3, 10], "max_features": [2, 3, 4]}]

forest_reg = RandomForestRegressor()

grid_search = GridSearchCV(forest_reg, param_grid=param_grid, cv=5,
                           scoring="neg_mean_squared_error", return_train_score=True)

grid_search.fit(housing_prepared, housing_labels)
cvres = grid_search.cv_results_

for mean_score, params  in zip(cvres["mean_test_score"], cvres["params"]):
    print(np.sqrt(-mean_score), params)
feature_importance = grid_search.best_estimator_.feature_importances_
print("\nFeature Importance: \n{}\n".format(feature_importance))

extra_attribs = ["rooms_per_hhold", "pop_per_hhold", "bedrooms_per_room"]
cat_encoder = full_pipeline.named_transformers_["cat"]
cat_one_hot_attribs = list(cat_encoder.categories_[0])
attributes = num_attribs + extra_attribs + cat_one_hot_attribs

resultes = sorted(zip(feature_importance, attributes), reverse=True)

print("Results of Feature and Importance: \n{}\n".format(resultes))<jupyter_output>Feature Importance: 
[7.36835432e-02 6.60209832e-02 4.32508326e-02 1.79704209e-02
 1.66741402e-02 1.71823240e-02 1.57151407e-02 3.09919405e-01
 5.23568125e-02 1.11449485e-01 9.71765078e-02 7.48631868e-03
 1.64381584e-01 7.21134769e-05 2.93499437e-03 3.72539473e-03]

Results of Feature and Importance: 
[(0.30991940465228257, 'median_income'), (0.16438158389745144, 'INLAND'), (0.11144948514925004, 'pop_per_hhold'), (0.09717650780301433, 'bedrooms_per_room'), (0.07368354316719972, 'longitude'), (0.06602098315691106, 'latitude'), (0.05235681250375809, 'rooms_per_hhold'), (0.04325083255474583, 'housing_median_age'), (0.0179704208793358, 'total_rooms'), (0.017182324039167397, 'population'), (0.016674140219758854, 'total_bedrooms'), (0.0157151407167709, 'households'), (0.007486318683109752, '<1H OCEAN'), (0.003725394731043233, 'NEAR OCEAN'), (0.0029349943692654097, 'NEAR BAY'), (7.211347693567463e-05, 'ISLAND')]<jupyter_text>Evaluate The Model<jupyter_code>final_model = grid_search.best_estimator_

X_test = strat_test_set.drop("median_house_value", axis=1)
y_test = strat_test_set["median_house_value"].copy()

X_test_prepared = full_pipeline.transform(X_test)

final_predictions = final_model.predict(X_test_prepared)

final_mse = mean_squared_error(y_test, final_predictions)
final_rmse = np.sqrt(final_mse)

print("\nThe Final Model Root Mean Squared Error: {}\n".format(final_rmse))

from scipy import stats
confidence = 0.95
squared_errors = (final_predictions - y_test) ** 2

final_results = np.sqrt(stats.t.interval(confidence, len(squared_errors) - 1,
                       loc=squared_errors.mean(),
                       scale=stats.sem(squared_errors)))

print("\nFinal Results: {}\n".format(final_results))<jupyter_output>The Final Model Root Mean Squared Error: 47831.695767396704


Final Results: [45848.76831204 49735.6279164 ]
{'fraction_non_alphanumeric': 0.09862079193237432, 'fraction_numerical': 0.09083493993771319, 'mean_word_length': 5.252665739452944, 'pattern_counts': {'":': 5, '<': 35, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '36724', 'license': 'Apache-2.0', 'n_tokens_mistral': 5277, 'n_tokens_neox': 4398, 'n_words': 1161, 'path': 'Chapter 2 - End-to-End Machine Learning Project/.ipynb_checkpoints/Housing-checkpoint.ipynb', 'repo_name': 'Jadams29/Hands-On-Machine-Learning-with-Scikit-Learn-Keras-Tensorflow-2nd', 'size': '13486'}