<jupyter_start><jupyter_text>Download the Data<jupyter_code>import os
import tarfile
import urllib


DOWNLOAD_ROOT = "https://raw.githubusercontent.com/ageron/handson-ml2/master/"
HOUSING_PATH = os.path.join("datasets", "housing")
HOUSING_URL = DOWNLOAD_ROOT + "datasets/housing/housing.tgz"


def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):
    """Download the Data and extract them"""
    os.makedirs(housing_path, exist_ok=True)
    tgz_path = os.path.join(housing_path, "housing.tgz")
    urllib.request.urlretrieve(housing_url, tgz_path)
    housing_tgz = tarfile.open(tgz_path)
    housing_tgz.extractall(path=housing_path)
    housing_tgz.close()   
fetch_housing_data()<jupyter_output><empty_output><jupyter_text>load the data<jupyter_code>import pandas as pd


def load_housing_data(housing_path:str =HOUSING_PATH) -> pd.DataFrame:
    """read the data from csv and return panda data from,"""
    csv_path = os.path.join(housing_path,"housing.csv")
    return pd.read_csv(csv_path)
df_housing = load_housing_data()
df_housing.head()
df_housing.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 20640 entries, 0 to 20639
Data columns (total 10 columns):
 #   Column              Non-Null Count  Dtype  
---  ------              --------------  -----  
 0   longitude           20640 non-null  float64
 1   latitude            20640 non-null  float64
 2   housing_median_age  20640 non-null  float64
 3   total_rooms         20640 non-null  float64
 4   total_bedrooms      20433 non-null  float64
 5   population          20640 non-null  float64
 6   households          20640 non-null  float64
 7   median_income       20640 non-null  float64
 8   median_house_value  20640 non-null  float64
 9   ocean_proximity     20640 non-null  object 
dtypes: float64(9), object(1)
memory usage: 1.6+ MB<jupyter_text>Check on particular parameters<jupyter_code>ocean_crt= df_housing["ocean_proximity"].value_counts()
print(ocean_crt)<jupyter_output><1H OCEAN     9136
INLAND        6551
NEAR OCEAN    2658
NEAR BAY      2290
ISLAND           5
Name: ocean_proximity, dtype: int64<jupyter_text>Check other parameters:<jupyter_code>df_housing.describe()<jupyter_output><empty_output><jupyter_text>Histogram<jupyter_code>import matplotlib.pyplot as plt


df_housing.hist(bins=50, figsize=(22,15))
plt.show()<jupyter_output><empty_output><jupyter_text>Create a Test Set<jupyter_code># import numpy as np
# from typing import Tuple


# def split_train_test(data:pd.DataFrame,test_ratio:float) -> Tuple[pd.DataFrame,pd.DataFrame]:
    # """Split the  dataframe to two train set and test set"""
    # shuffled_indices = np.random.permutation(len(data))
    # test_set_size = int(len(data)*test_ratio)
    # test_indices = shuffled_indices[:test_set_size]
    # train_indices = shuffled_indices[test_set_size:]
    # return data.iloc[train_indices], data.iloc[test_indices]


# np.random.seed(42)
# train_set, test_set = split_train_test(df_housing, 0.2)
# train_set.head()
# from zlib import crc32


# def test_set_check(identifier, test_ratio):
    # return crc32(np.int64(identifier)) & 0xffffffff < test_ratio * 2**32
# 
# def split_train_test_by_id(data, test_ratio, id_column):
    # ids = data[id_column]
    # in_test_set = ids.apply(lambda id_: test_set_check(id_, test_ratio))
    # return data.loc[~in_test_set], data.loc[in_test_set]
# housing_with_id = df_housing.reset_index()
# train_set, test_set = split_train_test_by_id(housing_with_id, 0.2, "index")
from sklearn.model_selection import train_test_split
train_set, test_set = train_test_split(df_housing, test_size=0.2, random_state=42)
#train_set.head()<jupyter_output><empty_output><jupyter_text>Categorizing to check the distribution<jupyter_code>import numpy as np

df_housing["income_cat"] = pd.cut(df_housing["median_income"],
                                bins=[0., 1.5, 3.0, 4.5, 6., np.inf],
                                labels=[1, 2, 3, 4, 5])

df_housing["income_cat"].hist()

from sklearn.model_selection import StratifiedShuffleSplit

split = StratifiedShuffleSplit(n_splits=1,test_size=0.2,random_state=42)
for train_index, test_index in split.split(df_housing,df_housing["income_cat"]):
    strat_train_set = df_housing.loc[train_index]
    strat_test_set = df_housing.loc[test_index]

strat_test_set["income_cat"].value_counts() / len(strat_test_set)
for set_ in (strat_train_set, strat_test_set):
    set_.drop("income_cat", axis=1, inplace=True)
strat_train_set.head()
HOUSING_PATH = os.path.join("datasets", "housing")


def save_train_test_data(housing_path:str =HOUSING_PATH) -> None:
    """save the test and train data in csv"""
    test_path = os.path.join(housing_path,"test.csv")
    train_path = os.path.join(housing_path,"train.csv")
    strat_test_set.to_csv(test_path,index=False)
    strat_train_set.to_csv(train_path,index=False)

save_train_test_data()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11221590909090909, 'fraction_numerical': 0.033076298701298704, 'mean_word_length': 4.613895216400911, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 31, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '256225', 'license': 'Apache-2.0', 'n_tokens_mistral': 1932, 'n_tokens_neox': 1799, 'n_words': 319, 'path': 'chapter2/Get_data.ipynb', 'repo_name': 'FarhadManiCodes/handson-ml2', 'size': '4928'}