<jupyter_start><jupyter_text>train_test_split- numpy arrays<jupyter_code>X, y = np.arange(10).reshape((5, 2)), range(5)
print(X) # Column first
print(y)<jupyter_output>[[0 1]
 [2 3]
 [4 5]
 [6 7]
 [8 9]]
range(0, 5)<jupyter_text>用于生成数据索引<jupyter_code>num_samples = 50000 # Same as MNIST
val_split = 0.2
# train_test_split
# Input: a list of int
# Outputs: train indices, validation indices
train_idx, val_idx = train_test_split(list(range(num_samples)), test_size=val_split, shuffle=False)<jupyter_output><empty_output><jupyter_text>结合 PyTorch Datasets 使用<jupyter_code>def train_val_dataset(dataset, val_split=0.25):
    train_idx, val_idx = train_test_split(list(range(len(dataset))), test_size=val_split)
    datasets = {}
    datasets['train'] = Subset(dataset, train_idx)
    datasets['val'] = Subset(dataset, val_idx)
    return datasets
len_dataset = 1000
val_split = 0.25
# 随机的，因此固定random_state
random_state = 42
train_idx, val_idx = train_test_split(
    list(range(len_dataset)), test_size=val_split, random_state=random_state)
print(train_idx[:10])  # 只要random_state一致结果就一直


#### 可视化索引序列
plt.scatter(list(range(len(train_idx))), sorted(train_idx))
# - 稀疏性
# - 随机性
from torchvision import datasets, transforms
kwargs = {'batch_size': 32}

# prepare transform
transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.1307,), (0.3081,)),
    transforms.Lambda(lambda x: torch.flatten(x)
                        )  # Add for 1D inputs
])

data_root = os.path.expanduser("~/.datasets")
mnist_train = datasets.MNIST(
    data_root, train=True, download=True, transform=transform)
mnist_test = datasets.MNIST(
    data_root, train=False, transform=transform)
train_loader = torch.utils.data.DataLoader(mnist_train, **kwargs)
test_loader = torch.utils.data.DataLoader(mnist_test, **kwargs)
# mnist_train.__dir__()
mnist_train.targets
from torch.utils.data import Subset
from sklearn.model_selection import train_test_split
def train_val_dataset(dataset, val_split=0.25, random_state=None):
    train_idx, val_idx = train_test_split(list(range(len(dataset))), test_size=val_split, random_state=random_state)
    datasets = {}
    datasets['train'] = Subset(dataset, train_idx)
    datasets['val'] = Subset(dataset, val_idx)
    return datasets
mnist_train_val = train_val_dataset(mnist_train, random_state=42)
train_loader = torch.utils.data.DataLoader(mnist_train_val["train"], **kwargs)
valid_loader = torch.utils.data.DataLoader(mnist_train_val["val"], **kwargs)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1293512658227848, 'fraction_numerical': 0.022151898734177215, 'mean_word_length': 6.373177842565598, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '494487', 'license': 'MIT', 'n_tokens_mistral': 971, 'n_tokens_neox': 937, 'n_words': 160, 'path': 'sklearn_basics/sklearn.model_selection.train_test_split_test.ipynb', 'repo_name': 'KellyHwong/deeplearning-basics', 'size': '2528'}