<jupyter_start><jupyter_text>download from S3<jupyter_code>session = boto3.Session(aws_access_key_id=access_key_id, aws_secret_access_key=secret_key_id)
s3 = session.resource('s3')

unrestricted = pd.read_csv('unrestricted_zshek_5_6_2022_5_34_30.csv')

ii = 0

id_list = []
age_list = []
gender_list = []

for subj_id in unrestricted['Subject']:

    id_ = str(subj_id)
    KEY = 'HCP/' + id_ + '/unprocessed/3T/T2w_SPC1/'+ id_ <KEY>'
    DST = 'D:/HCP/data/T2w/' + id_ + '.nii.gz'
    ii += 1
    try:
        s3.Bucket('hcp-openaccess').download_file(KEY, DST)
    except botocore.exceptions.ClientError:
        continue
    id_list.append(id_)
    age_list.append(unrestricted['Age'][ii-1])
    gender_list.append(unrestricted['Gender'][ii-1])
    print(str(ii-1) + ': ' + DST)

    if ii > 4:
        break
custom_HCP_Dataframe = pd.DataFrame(list(zip(id_list, age_list, gender_list)), columns = ['id', 'age', 'gender'])
print(custom_HCP_Dataframe)
custom_HCP_Dataframe.to_csv('custom_HCP.csv')
class HCP_3T_T2w_Dataset(Dataset):

    def __init__(self, csv_file, root_dir, transform=None):

        self.id_unrestricted_frame = pd.read_csv(csv_file)
        self.root_dir = root_dir
        self.transform = transform

    def __len__(self):
        return len(self.id_unrestricted_frame)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        img_name = os.path.join(self.root_dir,
                                str(self.id_unrestricted_frame.iloc[idx, 1])+'.nii.gz')
        image = nib.load(img_name)

        id_ = self.id_unrestricted_frame.iloc[idx, 1]
        age = self.id_unrestricted_frame.iloc[idx, 2]
        gender = self.id_unrestricted_frame.iloc[idx, 3]
        sample = {'image': image, 'id_': id_, 'age': age, 'gender': gender}

        if self.transform:
            sample = self.transform(sample)

        return sample
HCP_dataset = HCP_3T_T2w_Dataset(csv_file='custom_HCP.csv', root_dir='data/T2w/')
plt.figure()
for i in range(len(HCP_dataset)):
    sample = HCP_dataset[i]

    print(i, sample['image'].shape, sample['id_'])
    slice_ = sample['image'].get_fdata()
    slice_ = slice_[:, :, 160]
    plt.imshow(slice_, cmap='gray')<jupyter_output>0 (256, 320, 320) 100307
1 (256, 320, 320) 100408
2 (256, 320, 320) 101006
{'fraction_non_alphanumeric': 0.14310494362532525, 'fraction_numerical': 0.0372940156114484, 'mean_word_length': 3.4365384615384613, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '488445', 'license': 'MIT', 'n_tokens_mistral': 961, 'n_tokens_neox': 872, 'n_words': 145, 'path': '220524_HCP_pipeline.ipynb', 'repo_name': 'Zshek001/HKU_BISP_2022_Shihao', 'size': '2323'}