<jupyter_start><jupyter_text>Translating Images:Make directoy '/data/datasets/chestxray/' and run the following cells<jupyter_code>for data_name in ['nih', 'chex', 'kaggle']:

    rng = default_rng()
    indices = rng.choice(data[data_name]['size'], size=data[data_name]['size'], replace=False)    
    print(indices.shape)
    
    count=0    
    for case in ['train', 'val', 'test']:        
        size= data[data_name][case+'_size']
        ids=[]
        
        count_l0=0
        count_l1=0
        count_lim=int(size/2)
        
        while count_l0 < count_lim or count_l1 < count_lim:

            index= indices[count].item()
            task = xrv.datasets.default_pathologies.index('Pneumonia')            
            label= data[data_name]['obj'][index]['lab'][task]
            count+=1
            
            if np.isnan(label):
                continue
            else:
                
                if label == 0:
                    if count_l0 < count_lim:
#                         print('Label 0')
                        count_l0+= 1
                    else:
                        continue

                if label ==1:
                    if count_l1 < count_lim:
#                         print('Label 1')
                        count_l1+= 1
                    else:
                        continue
                
                ids.append(index) 
                    
        ids= np.array(ids)
        
        print(count_l0, count_l1)
        base_dir='/data/datasets/chestxray/'
        np.save(base_dir + data_name + '_' + case  + '_' + 'indices.npy', ids)        
base_dir='/data/datasets/chestxray/'
        
for data_name in ['nih', 'chex', 'kaggle']:
    
    indices= np.random.randint(0, data[data_name]['size'], data[data_name]['size'] )
    print(indices.shape)
        
    count=0    
    for case in ['train', 'val', 'test']:
        
        size= data[data_name][case+'_size']            
        imgs=[]
        labels=[]    
        imgs_org=[]
        
        indices= np.load(base_dir + data_name + '_' + case  + '_' + 'indices.npy')        
        
        count_l0=0
        count_l1=0
        for idx in range(indices.shape[0]):
            
            index= indices[idx].item()
            task = xrv.datasets.default_pathologies.index('Pneumonia')
            
            img= data[data_name]['obj'][index]['img']
            img_org= data[data_name]['obj'][index]['img']
            label= data[data_name]['obj'][index]['lab'][task]
            count+=1
            
            if np.isnan(data[data_name]['obj'][index]['lab'][task]):
                print('Error: Nan in the labels')
                
            if label == 0:
                count_l0+=1
                
            if label == 1:
                count_l1+=1
            
            label=torch.tensor(label).long()
            label= label.view(1)
            
            img= to_tensor( to_augment( translate(img, label, data_name, index, 1) ) )                
            img_org= to_tensor( translate(img, label, data_name, index, 1) )
                
            
            img= img.view(1, img.shape[0], img.shape[1], img.shape[2])
            img_org= img_org.view(1, img_org.shape[0], img_org.shape[1], img_org.shape[2])
                                
#             print('Data: ', data_name, count, img.shape, label)
            imgs.append(img)
            imgs_org.append(img_org)
            labels.append(label)
            
            if torch.all(torch.eq(img, img_org)):
                print('Error:')
                
                
        imgs=torch.cat(imgs)
        imgs_org=torch.cat(imgs_org)
        labels=torch.cat(labels)
        print(imgs.shape, imgs_org.shape, labels.shape, count_l0, count_l1)
        
        torch.save(imgs, base_dir + data_name + '_trans_' + case  + '_' + 'image.pt')
        torch.save(imgs_org, base_dir + data_name + '_trans_' + case  + '_' + 'image_org.pt')
        torch.save(labels, base_dir + data_name + '_trans_' + case  + '_' + 'label.pt')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1291819291819292, 'fraction_numerical': 0.011721611721611722, 'mean_word_length': 1.430860534124629, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '480357', 'license': 'MIT', 'n_tokens_mistral': 1277, 'n_tokens_neox': 1257, 'n_words': 213, 'path': 'docs/notebooks/ChestXRay_Translate.ipynb', 'repo_name': 'nijinjose/robustdg', 'size': '4095'}