<jupyter_start><jupyter_text>创建测试集<jupyter_code>import pandas as pd
import os

HOUSING_PATH=os.path.join("datasets","housing")

def load_housing_data(housing_path=HOUSING_PATH):
    csv_path=os.path.join(housing_path,"housing.csv");
    return pd.read_csv(csv_path)

housing=load_housing_data()<jupyter_output><empty_output><jupyter_text>方法一：通过随机数列划分训练集和测试集,将随机数列作为列索引划分    np.random.permutation 生成随机数列<jupyter_code>import numpy as np
np.random.seed(42)
def split_train_test(data,test_ration):
    shuffled_indices=np.random.permutation(len(data))
    test_set_size=int(len(data)*test_ration)
    test_indices=shuffled_indices[:test_set_size]
    tran_indices=shuffled_indices[test_set_size:]
    tran_set=data.iloc[tran_indices]
    test_set=data.iloc[test_indices]
    return tran_set,test_set
train_set,test_set=split_train_test(housing,0.2)<jupyter_output><empty_output><jupyter_text>方法二：通过计算主键的hash值，只取hash值的    使用pd.apply方法<jupyter_code>import hashlib

def test_set_check(identifier, test_ratio, hash=hashlib.md5):
    return hash(np.int64(identifier)).digest()[-1] < 256 * test_ratio

def split_train_test_by_id(data, test_ratio, id_column):
    ids = data[id_column]
    in_test_set = ids.apply(lambda id_: test_set_check(id_, test_ratio))
    return data.loc[~in_test_set], data.loc[in_test_set]<jupyter_output><empty_output><jupyter_text>方法三：通过Scikit-Learn 提供的函数划分<jupyter_code>from sklearn.model_selection import train_test_split

train_set,test_set=train_test_split(housing,test_size=0.2,random_state=42)<jupyter_output><empty_output><jupyter_text>分层抽样划分测试集<jupyter_code>import matplotlib.pyplot as plt
housing['median_income'].hist(bins=50)
plt.show()
# 限制收入类别的数量
housing['income_cat']=np.ceil(housing['median_income']/1.5)

#将所有大于5的类别合并为类别5
housing['income_cat'].where(housing['income_cat']<5,5.0,inplace=True)

housing['income_cat'].value_counts()<jupyter_output><empty_output><jupyter_text>收入划分层级<jupyter_code>housing["income_cat"] = pd.cut(housing["median_income"],
                               bins=[0., 1.5, 3.0, 4.5, 6., np.inf],
                               labels=[1, 2, 3, 4, 5])
housing["income_cat"].value_counts().sort_index()<jupyter_output><empty_output><jupyter_text>使用Stratified-Shuffle Split 类进行分层抽样<jupyter_code>from sklearn.model_selection import StratifiedShuffleSplit

split=StratifiedShuffleSplit(n_splits=1,test_size=0.2,random_state=42)

for train_index,test_index in split.split(housing,housing['income_cat']):
    strat_train_set=housing.loc[train_index]
    strat_test_set=housing.loc[test_index]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12446518864255153, 'fraction_numerical': 0.01633605600933489, 'mean_word_length': 7.930555555555555, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '205916', 'license': 'Apache-2.0', 'n_tokens_mistral': 1114, 'n_tokens_neox': 1097, 'n_words': 94, 'path': '02_split_train_test.ipynb', 'repo_name': 'JaredZen/handson-ml', 'size': '2571'}