<jupyter_start><jupyter_text>Setup<jupyter_code>import pandas as pd
import numpy as np
import os
np.random.seed(24)
housing = pd.read_csv("datasets/housing/housing.csv")
housing.head()<jupyter_output><empty_output><jupyter_text>Data Split by stratification<jupyter_code>housing["income_cat"] = np.ceil(housing["median_income"]/1.5)
housing["income_cat"].where(housing["income_cat"]<5, 5.0, inplace=True)
housing["income_cat"].value_counts()
from sklearn.model_selection import StratifiedShuffleSplit
split = StratifiedShuffleSplit(n_splits = 1, test_size=0.2, random_state=42)
for train_index, test_index in split.split(housing, housing["income_cat"]):
    strat_train_set = housing.loc[train_index]
    strat_test_set = housing.loc[test_index]
print("Train set:", strat_train_set.shape)
print("Test set:", strat_test_set.shape)
# %matplotlib inline
import matplotlib.pyplot as plt
f,ax = plt.subplots(1,2,sharey=True,figsize=(8,6))
# ax[0].hist(strat_train_set["income_cat"])
# ax[0].set_title("training set")
# ax[1].hist(housing["income_cat"])
# ax[1].set_title("full set")
strat_train_set["income_cat"].plot(kind="hist", ax=ax[0], title="training set")
housing["income_cat"].plot(kind="hist", ax=ax[1], title="full dataset")
plt.show()
strat_train_set.drop(["income_cat"], axis=1, inplace=True)
strat_train_set.head(2)<jupyter_output><empty_output><jupyter_text>Data Preparation* For numerical attributes, fill in missing data, standardization, add additional attribute* For categorical attributes, apply one-hot-encoder<jupyter_code>housing = strat_train_set.drop("median_house_value", axis=1)
housing_label = strat_train_set["median_house_value"].copy()
print(housing.head(2), "\n")
print(housing_label.head(2))
from sklearn.base import BaseEstimator, TransformerMixin
class CustomBinarizer(BaseEstimator, TransformerMixin):
    def fit(self, X, y=None,**fit_params):
        return self
    def transform(self, X):
        return LabelBinarizer().fit(X).transform(X)
from sklearn.base import BaseEstimator, TransformerMixin

# column index
rooms_ix, bedrooms_ix, population_ix, household_ix = 3, 4, 5, 6
class CombinedAttributesAdder(BaseEstimator, TransformerMixin):
    def __init__(self, add_bedrooms_per_room = True):
        self.add_bedrooms_per_room = True
    def fit(self, X, y=None):
        return self
    def transform(self, X, y=None):
        rooms_per_household = X[:, rooms_ix] / X[:, household_ix]
        population_per_household = X[:, population_ix] / X[:, household_ix]
        if self.add_bedrooms_per_room: 
            bedrooms_per_room = X[:, rooms_ix]/X[:, household_ix]
            return np.c_[X, rooms_per_household, population_per_household, bedrooms_per_room]
        else:
            return np.c_[X, rooms_per_household, population_per_household]
class DataFrameSelector(BaseEstimator, TransformerMixin):
    def __init__(self, attribute_names):
        self.attribute_names = attribute_names
    def fit(self, X, y=None):
        return self # do nothing
    def transform(self, X, y=None):
        return X[self.attribute_names].values # return numpy array
num_attribs = housing.columns.values.tolist()
num_attribs.remove("ocean_proximity")
num_attribs
cat_attribs = ["ocean_proximity"]
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import Imputer
from sklearn.preprocessing import LabelBinarizer
num_pipeline = Pipeline([
        ('selector', DataFrameSelector(num_attribs)), # select numeric columns
        ('imputer', Imputer(strategy="median")), # fill missing values with median
        ('attribs_adder', CombinedAttributesAdder()), # add extra columns
        ('std_scaler', StandardScaler()), # standardize
    ])

cat_pipeline = Pipeline([
        ('selector', DataFrameSelector(cat_attribs)),  # select categorical columns
        ('label_binarizer', CustomBinarizer()), # binarize categorical attributes
    ])

from sklearn.pipeline import FeatureUnion
full_pipeline = FeatureUnion(transformer_list = [
    ("num_pipeline", num_pipeline),
    ("cat_pipeline", cat_pipeline),
])
housing_pred = full_pipeline.fit_transform(housing)
housing_pred
housing_pred.shape<jupyter_output><empty_output><jupyter_text>Practice  Question 1 Try a Support Vector Machine regressor (`sklearn.svm.SVR`), with various hyperparameters such as `kernel="linear"` (with various values for the `C` hyperparameter) or `kernel="rbf"` (with various values for the `C` and `gamma` hyperparameters). Don't worry about what these hyperparameters mean for now. How does the best `SVR` predictor perform?<jupyter_code>from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVR
param_grid = [{"kernel": ["linear"], "C": [10., 30., 100., 300., 1000., 3000., 10000., 30000.0]},
              {"kernel": ["rbf"], "C": [1.0, 3.0, 10., 30., 100., 300., 1000.0], 
              "gamma": [0.01, 0.03, 0.1, 0.3, 1.0, 3.0]}]
svm_reg = SVR()
grid_search = GridSearchCV(svm_reg, param_grid, cv=5, scoring="neg_mean_squared_error", verbose=2, n_jobs=4)
grid_search.fit(housing_pred, housing_label)
grid_search.best_estimator_
negative_mse = grid_search.best_score_
print(np.sqrt(-negative_mse))
grid_search.best_params_<jupyter_output><empty_output><jupyter_text>Question 2  Try replacing `GridSearchCV` with `RandomizedSearchCV`.<jupyter_code>from sklearn.model_selection import RandomizedSearchCV
param_hist ={"kernel": ["linear", "rbf"], "C": np.linspace(1.0, 400.0, 10), "gamma": np.linspace(0.1, 3.0,10)}
svm_reg = SVR()
rnd_search = RandomizedSearchCV(estimator=svm_reg, cv=5, param_distributions=param_hist, 
                                n_jobs=4, scoring="neg_mean_squared_error", verbose=4)
rnd_search.fit(housing_pred, housing_label)
rnd_search.best_estimator_
rmse =np.sqrt(-rnd_search.best_score_)
rmse
rnd_search.best_params_<jupyter_output><empty_output><jupyter_text>Question 3  Try adding a transformer in the preparation pipeline to select only the most important attributes.<jupyter_code>from sklearn.model_selection import RandomizedSearchCV
from sklearn.ensemble import RandomForestRegressor
forest_reg = RandomForestRegressor(random_state=42)
param_hist = {"n_estimators": np.arange(10, 500, 20),
             "max_depth": np.arange(5, 50, 5),
             "max_features": [2,4,6,8]}
rnd_search = RandomizedSearchCV(estimator=forest_reg, cv=5, param_distributions=param_hist,
                                scoring="neg_mean_squared_error",
                                n_jobs=4,verbose=4)
rnd_search.fit(housing_pred, housing_label)
cvre = rnd_search.cv_results_
for mean_score, params in zip(cvre["mean_test_score"], cvre["params"]):
    print(np.sqrt(-mean_score), params)
rnd_search.best_estimator_
feature_importances = rnd_search.best_estimator_.feature_importances_
extra_attribs = ["room_per_household", "pop_per_household", "bedroom_per_room"]
encoder = LabelBinarizer()
housing_cat_encoded = encoder.fit_transform(housing["ocean_proximity"])
cat_attribs = encoder.classes_.tolist()
attribs = num_attribs + extra_attribs + cat_attribs
sorted(zip(feature_importances, attribs))
# pick the feature of important
from sklearn.base import BaseEstimator, TransformerMixin
def indices_of_top_k(arr, k):
    return(np.sort(np.argpartition(np.array(arr), -k)[-k:]))
class TopFeatureSelector(BaseEstimator, TransformerMixin):
    def __init__(self, feature_importance, k):
        self.feature_importance = feature_importance
        self.k = k
    def fit(self, X, y=None):
        self.feature_indices_ = indices_of_top_k(self.feature_importance, self.k)
        return self
    def transform(self, X, y=None):
        return(X[:, self.feature_indices_])
indices_of_top_k(feature_importances, 5) # the top five important features
np.array(attribs)[indices_of_top_k(feature_importances, 5)]
preparation_and_feature_selection_pipeline = Pipeline([
    ("preparation", full_pipeline),
    ("feature_selection", TopFeatureSelector(feature_importances, 5))])
housing_prepared_top_k_features = preparation_and_feature_selection_pipeline.fit_transform(housing)
housing_prepared_top_k_features[:3,:]
from sklearn.model_selection import cross_val_score
rnd_reg=RandomForestRegressor(max_features=8, n_estimators=30)
scores = cross_val_score(rnd_reg, X=housing_pred, y=housing_label, 
                           scoring="neg_mean_squared_error", cv=5, 
                           n_jobs=-1,verbose=4)
rmse = [np.sqrt(-i) for i in scores]
print(rmse)
# using n_estimator = 350
rnd_reg = RandomForestRegressor(max_features=8, n_estimators=350)
scores = cross_val_score(estimator=rnd_reg, cv=5, X=housing_pred, y=housing_label, 
                        scoring="neg_mean_squared_error", n_jobs=-1, verbose=4)
rmse = [np.sqrt(-i) for i in scores]
print(rmse)<jupyter_output>[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   44.0s remaining:  1.1min<jupyter_text>Question 4 Try creating a single pipeline that does the full data preparation plus the final prediction.<jupyter_code>prepare_prediction_pipeline = Pipeline([
    ("preparation", full_pipeline),
    ("feature_selection", TopFeatureSelector(feature_importances, 6)), 
    ("prediction", rnd_search.best_estimator_)
])
some_data = housing.iloc[:4]
some_data_label = housing_label[:4]
some_data
prepare_prediction_pipeline.fit(some_data, some_data_label)
print("Label", some_data_label.values)
print("Prediction", prepare_prediction_pipeline.predict(some_data))<jupyter_output>Label [ 286600.  340600.  196900.   46300.]
Prediction [ 273038.66666667  296892.66666667  202522.66666667  138553.33333333]<jupyter_text>Question 5 Automatically explore some preparation options using GridSearchCV<jupyter_code>from sklearn.model_selection import GridSearchCV
param_grid = [
        {'preparation__num_pipeline__imputer__strategy': ['mean', 'median', 'most_frequent'],
         'feature_selection__k': [4, 5, 6]}
]

grid_search_prep = GridSearchCV(preparation_and_feature_selection_pipeline, param_grid, cv=3,
                                scoring='neg_mean_squared_error', verbose=4, n_jobs=-1)
grid_search_prep.fit(housing, housing_label)<jupyter_output>Fitting 3 folds for each of 9 candidates, totalling 27 fits
{'fraction_non_alphanumeric': 0.1151467556689899, 'fraction_numerical': 0.025326396387552762, 'mean_word_length': 5.611291369240753, 'pattern_counts': {'":': 11, '<': 31, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '161630', 'license': 'Apache-2.0', 'n_tokens_mistral': 3615, 'n_tokens_neox': 3395, 'n_words': 675, 'path': '02_practice.ipynb', 'repo_name': 'yaozhou86/handson-ml', 'size': '10187'}