<jupyter_start><jupyter_text>Preprocessing<jupyter_code>from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedShuffleSplit
df.head()
split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)

housing = df.copy()
strat_train_set = []
strat_test_set = []

# Divide by 1.5 to limit the number of income categories
housing["income_cat"] = np.ceil(housing["median_income"] / 1.5)
# Label those above 5 as 5
housing["income_cat"].where(housing["income_cat"] < 5, 5.0, inplace=True)

for train_index, test_index in split.split(housing, housing["income_cat"]):
    strat_train_set = housing.loc[train_index]
    strat_test_set = housing.loc[test_index]
strat_train_set.info()
strat_test_set.info()
strat_train_set.head(2)
# X ...
housing = strat_train_set.drop('median_house_value', axis=1)
housing.head(2)
# y ...
housing_label = strat_train_set['median_house_value'].copy()
housing_label[:2]<jupyter_output><empty_output><jupyter_text>Cleaning Data / Missing Data<jupyter_code>from sklearn.impute import SimpleImputer
housing.dropna(subset=['total_bedrooms']).head(2)
housing.shape
# delete rows which has empty values ...
housing.dropna(subset=['total_bedrooms']).shape
# delete the feature from dataset ...
housing.drop('total_bedrooms', axis=1).shape
# # fill the empty value of features ...
# medain = housing['total_bedrooms'].median()
# housing['total_bedrooms'].fillna(medain, inplace=True)
housing.info()
imputer = SimpleImputer(strategy='median')
imputer
housing_num = housing.drop('ocean_proximity', axis=1)
housing_num.shape
imputer.fit(housing_num)
imputer.statistics_
for k in housing_num.columns:
    print('{0} median : {1}'.format(k, housing_num[k].median()))
X = imputer.transform(housing_num)
X<jupyter_output><empty_output><jupyter_text>Category feature<jupyter_code>from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import LabelBinarizer
label_encoder = LabelEncoder()

housing_cat = housing['ocean_proximity']
housing_cat_encoded = label_encoder.fit_transform(housing_cat)
housing_cat_encoded
label_encoder.classes_
one_hot_encoder = OneHotEncoder()

housing_cat_1hot = one_hot_encoder.fit_transform(housing_cat_encoded.reshape(-1,1))
housing_cat_1hot
housing_cat_1hot.toarray()
label_binarizer_encoder = LabelBinarizer()

housing_cat_1hot = label_binarizer_encoder.fit_transform(housing_cat)
housing_cat_1hot
label_binarizer_encoder.classes_<jupyter_output><empty_output><jupyter_text>Custom Transformer [[1.1.]](resources)<jupyter_code>from sklearn.base import BaseEstimator, TransformerMixin
housing.columns
# get the right column indices: safer than hard-coding indices 3, 4, 5, 6
rooms_ix, bedrooms_ix, population_ix, household_ix = [
    list(housing.columns).index(col)
    for col in ("total_rooms", "total_bedrooms", "population", "households")]
#
# This code come from book [1.1.], page 82.
#

class CombinedAttributesAdder(BaseEstimator, TransformerMixin):
    def __init__(self, add_bedrooms_per_room = True): # no *args or **kwargs
        self.add_bedrooms_per_room = add_bedrooms_per_room

    def fit(self, X, y=None):
        return self  # nothing else to do

    def transform(self, X, y=None):
        rooms_per_household = X[:, rooms_ix] / X[:, household_ix]
        population_per_household = X[:, population_ix] / X[:, household_ix]
        if self.add_bedrooms_per_room:
            bedrooms_per_room = X[:, bedrooms_ix] / X[:, rooms_ix]
            return np.c_[X, rooms_per_household, population_per_household,
                         bedrooms_per_room]
        else:
            return np.c_[X, rooms_per_household, population_per_household]

attr_adder = CombinedAttributesAdder(add_bedrooms_per_room=False)
housing_extra_attribs = attr_adder.transform(housing.values)

housing_extra_attribs.shape<jupyter_output><empty_output><jupyter_text>Feature Scaling<jupyter_code>from sklearn.preprocessing import MinMaxScaler, StandardScaler
xx = np.array([3,4,5]).reshape(-1, 1)
# sacle into range [0,1]
MinMaxScaler().fit_transform(xx)
## the smallest value 2 after scaling not equall 0
xx/xx.max()
# ...
StandardScaler().fit_transform(xx)<jupyter_output><empty_output><jupyter_text>different range of MinMax ....<jupyter_code>MinMaxScaler(feature_range=(0,1)).fit_transform(xx)
MinMaxScaler(feature_range=(0.5,1)).fit_transform(xx)
MinMaxScaler(feature_range=(-1,0)).fit_transform(xx)
MinMaxScaler(feature_range=(2,3)).fit_transform(xx)<jupyter_output><empty_output><jupyter_text>Pipelines<jupyter_code>from sklearn.pipeline import Pipeline
from sklearn.pipeline import FeatureUnion
num_pipeline = Pipeline([
        ('imputer', SimpleImputer(strategy="median")),
        ('attribs_adder', CombinedAttributesAdder()),
        ('std_scaler', StandardScaler()),
    ])
housing_num_tr = num_pipeline.fit_transform(housing_num)
housing_num_tr
# Create a class to select numerical or categorical columns 
class DataFrameSelector(BaseEstimator, TransformerMixin):
    def __init__(self, attribute_names):
        self.attribute_names = attribute_names

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        return X[self.attribute_names].values
# ...
# num_attribs = housing_num.columns
# num_attribs = housing_num.keys()
num_attribs = list(housing_num)

# ...
cat_attribs = ["ocean_proximity"]

# ...
num_pipeline = Pipeline([
        ('selector', DataFrameSelector(num_attribs)),
        ('imputer', SimpleImputer(strategy="median")),
        ('attribs_adder', CombinedAttributesAdder()),
        ('std_scaler', StandardScaler()),
    ])

cat_pipeline = Pipeline([
        ('selector', DataFrameSelector(cat_attribs)),
        ('cat_encoder', OneHotEncoder(sparse=False)),
    ])
full_pipeline = FeatureUnion(transformer_list=[
    ('num_pipeline', num_pipeline),
    ('cat_pipeline', cat_pipeline),
])
old_housing_prepared = full_pipeline.fit_transform(housing)
old_housing_prepared
 old_housing_prepared.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11751075802714334, 'fraction_numerical': 0.009599470374048328, 'mean_word_length': 6.034924330616996, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '344540', 'license': 'MIT', 'n_tokens_mistral': 2101, 'n_tokens_neox': 2058, 'n_words': 357, 'path': 'notes/machine-learning-project-checklist/demo/ex1/main.ipynb', 'repo_name': 'SalAlba/machine-learning', 'size': '6042'}