<jupyter_start><jupyter_text>Part 1: create txt file with train and val id's of data<jupyter_code>import os.path as osp
import os
import random as rand
import cv2
root_path = "S2DSdevkit"
part_path = "part2"
images_path = "images"
labels_path = "labels"

folder_segmentation = "Segmentation"
images_dir_path = osp.join(root_path, part_path, images_path)
# create list of files in the folder
list_paths = os.listdir(images_dir_path)
# get only images ending on .jpg
list_images_with_postfix = list((filter(lambda k: '.jpg' in k, list_paths)))
# remove the file ending .jpg
list_images_without_postfix = list((map(lambda k: osp.splitext(k)[0], list_paths)))
for ele in range(5):
    print(list_images_without_postfix[ele])
# shuffel list
rand.shuffle(list_images_without_postfix)
for ele in range(5):
    print(list_images_without_postfix[ele])
print("\nTotal number of elements: " + str(len(list_images_without_postfix)) )
train_txt_path = osp.join(root_path, part_path, folder_segmentation, "train.txt")
with open(train_txt_path,'w') as f:
    f.write('\n'.join(list_images_without_postfix[:200]))
val_txt_path = osp.join(root_path, part_path, folder_segmentation, "val.txt")
with open(val_txt_path,'w') as f:
    f.write('\n'.join(list_images_without_postfix[200:]))<jupyter_output><empty_output><jupyter_text>Part 2: load data using yaml file and visulize some data pairs.<jupyter_code>from libs.datasets import get_dataset
from addict import Dict
import yaml
import torch
import numpy as np
# Load data using yaml and show some
config_path = 'configs/S2DS.yaml'
with open(config_path, 'r') as file:
    CONFIG = Dict(yaml.load(file, Loader=yaml.FullLoader))

for key in CONFIG:
    print(key + ": " + str(CONFIG[key]))
    print("--------")
# Dataset
dataset = get_dataset(CONFIG.DATASET.NAME)(
    root=CONFIG.DATASET.ROOT,
    part=CONFIG.DATASET.PART,
    split=CONFIG.DATASET.SPLIT.TRAIN,
    ignore_label=CONFIG.DATASET.IGNORE_LABEL,
    mean_bgr=(CONFIG.IMAGE.MEAN.B, CONFIG.IMAGE.MEAN.G, CONFIG.IMAGE.MEAN.R),
    augment=False,
    base_size=CONFIG.IMAGE.SIZE.BASE,
    crop_size=CONFIG.IMAGE.SIZE.TRAIN,
    scales=CONFIG.DATASET.SCALES,
    flip=True,
)
print(dataset)
# DataLoader
loader = torch.utils.data.DataLoader(
    dataset=dataset,
    batch_size=1,#CONFIG.SOLVER.BATCH_SIZE.TRAIN,
    num_workers=CONFIG.DATALOADER.NUM_WORKERS,
    shuffle=True,
)
loader_iter = iter(loader)
# Load data
image_ids, images, labels = next(loader_iter)
print("ID: " + str(image_ids))
print("Image shape: " + str(images.shape))
print("Label shape: " + str(labels.shape))
first_img = images[0].permute(1,2,0)
first_lbl = labels[0]
# choose first element of the batch
print("ID: " + str(image_ids))
print("Image shape: " + str(first_img.shape))
print("Label shape: " + str(first_lbl.shape))
dataset.show_pair(image_ids, images, labels)
testlst = range(1,10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12954624177346727, 'fraction_numerical': 0.008313127814340146, 'mean_word_length': 5.716279069767442, 'pattern_counts': {'":': 1, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '157028', 'license': 'MIT', 'n_tokens_mistral': 1049, 'n_tokens_neox': 1046, 'n_words': 219, 'path': 'CreateSplitFiles.ipynb', 'repo_name': 'Madave94/deeplab-pytorch', 'size': '2887'}