<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Generate a dictionary of all specific folders for analysis on a specific patient1.1&nbsp;&nbsp;Copying image files from patient derived folders into image specific folder types with renaming of files to include all pertinent information is held1.2&nbsp;&nbsp;Dicom writing patients to folder with specific file name str format1.3&nbsp;&nbsp;Ground writing patients to folder with specific file name format2&nbsp;&nbsp;Stripping out file information Patient id etc from file name using regex3&nbsp;&nbsp;Generate Fold text files denoting validation and training and test folders for analysis4&nbsp;&nbsp;Move data on a per patient basis to different folder set aka test or validation folders5&nbsp;&nbsp;Rename Ground files to match image source file names6&nbsp;&nbsp;Function calls for generating file_dictionaries, file tuple sets7&nbsp;&nbsp;Sub sampler creating folder called fold1 and fold2 for specific analysis in set folder8&nbsp;&nbsp;Sub sampler by patient to specific destination folders8.1&nbsp;&nbsp;Grey level intensity methods for grey level setting9&nbsp;&nbsp;Determine sub sampling based on class balance<jupyter_code>import os
import pandas as pd
import numpy as np
import re
from shutil import copyfile
import ipdb
import filecmp
import random

import imageio
from difflib import SequenceMatcher
import errno
import imageio
cd F:\Biomedical images\Train\NIFTI_MR_512x512\NIFTI_MR_512x512\NIFTI_MR_512x512_png_256grey_lvl<jupyter_output><empty_output><jupyter_text>Generate a dictionary of all specific folders for analysis on a specific patient<jupyter_code>imgs_modes=['t1dual','t2spir','inphase','outphase','dicom_anon','ground','source']
src_dir=r'F:\Biomedical images\Train\NIFTI_MR_256x256 seg_rename'#os.getcwd()
pair_keyword=[['t1dual_inphase','ground'],['t1dual_outphase','ground'],['t2spir','ground']]
dst_grnd_path=''
dst_img_path=''
grnd_sub_repl_tple_dict={'_ground':'_inphase','.png':'_grey_lvl_256.png'}
src_sub_repl_tple_dict={}

file_dict=gen_img_fileset_dict(src_dir,imgs_modes,key_word_sp=1,len_file_sp=-1)
file_dict_src_keys=[k for k,v in file_dict.items() if k.find('source')!=-1]
file_dict_src_keys
for k,v in file_dict.items():
    print(type(k))
    if k.find('source')!=-1:
        print(k)<jupyter_output><empty_output><jupyter_text>Copying image files from patient derived folders into image specific folder types with renaming of files to include all pertinent information is held<jupyter_code>src_dir=r'D:\Biomedical images\Train\Preprocess_steps\NIFTI_MR_N4_DICOM\8'
trgt_dir=r'D:\Biomedical images\Train\NIFTI_MR_256x256_seg_rename'
trgt_keyword='masks'
img_sub_str=['t2spir','inphase','outphase']
grey_lvl_sub_str=sorted(['64grey_lvl','256grey_lvl'])
img_types_broad=['t1dual','t2spir']<jupyter_output><empty_output><jupyter_text>Dicom writing patients to folder with specific file name str format<jupyter_code>import ipdb
#Generating directory directory bsaed on keyword in parameters above. 
trgt_dir_roots=[trgt_root for trgt_root,_,_ in os.walk(trgt_dir) if trgt_root.lower().find(trgt_keyword)!=-1]

for root,subdirs,files in os.walk(src_dir):
    
    if len(files)>0 and root.lower().find('dicom_anon')!=-1:
        
        #Getting target directories to write files to. 
        img_match=[img_type for img_type in img_sub_str if root.lower().find(img_type)!=-1]
        trgt_path_match=[trgt_root_dir for trgt_root_dir in trgt_dir_roots if trgt_root_dir.find(img_match[0])!=-1]
        
        
        #Iterate through files for renaming clause
        for file in files:
            #ipdb.set_trace()
            trgt_path_match_dict=dict(zip(grey_lvl_sub_str,trgt_path_match))
            file_b_name=os.path.splitext(file)[0]
            #Getting patient name inserted into file
            int_in_f_name=re.findall('d|\d+',file_b_name)
            pat_idx=file_b_name.find(int_in_f_name[0])

            dst_file_pat=insert_str(file_b_name,
                                              int_in_f_name[0],
                                              '_',pat_idx)

            #splitting braod image types in file name
            img_type_brd=[x for x in img_types_broad if file_b_name.lower().find(x)!=-1]
            img_brd_idx=dst_file_pat.lower().find(img_type_brd[0])
            dst_file_img_typ_brd=insert_str(dst_file_pat,img_type_brd[0], '_',img_brd_idx)
            
            #reading file in for renaming file for method.
            src_file_path=os.path.join(root,file) 
            tmp_arr=imageio.imread(src_file_path)
            
            
            #adding grey scale and image size
            for k,v in trgt_path_match_dict.items():
                final_f_name='pat_id_'+dst_file_img_typ_brd+'_'+k+'_256x256.png'
                strp_dbl_undr_scr=final_f_name.replace('__','_')
                trgt_path_match_dict[k]=os.path.join(v,strp_dbl_undr_scr).lower()
                
                #Bytescaling image for analysis
                tmp_arr_256=bytescaling(tmp_arr)
                
                if k.find('256grey_lvl')!=-1:
                    #print(trgt_path_match_dict[k]) 
                    imageio.imwrite(trgt_path_match_dict[k],tmp_arr_256)
                
                elif k.find('64grey_lvl')!=-1:

                    tmp_arr_64=quantize_igs(tmp_arr_256,64,'igs')
                    imageio.imwrite(trgt_path_match_dict[k],tmp_arr_64)<jupyter_output><empty_output><jupyter_text>Ground writing patients to folder with specific file name format<jupyter_code>src_dir=r"F:\Biomedical images\Train\MR_raw\8"
trgt_dir=r"F:\Biomedical images\Train\NIFTI_MR_256x256_seg_rename"
trgt_keyword='masks'
img_sub_str=['t2spir','t1dual']
grey_lvl_sub_str=['64grey_lvl','256grey_lvl']
img_types_broad=['t1dual','t2spir']
img_types_spec=['t1dual','t2spir','inphase','outphase']
#Generating directory directory bsaed on keyword in parameters above. 
trgt_dir_roots=[trgt_root for trgt_root,_,_ in os.walk(trgt_dir) if trgt_root.lower().find(trgt_keyword)!=-1]

for root,subdirs,files in os.walk(src_dir):
    
    if len(files)>0 and root.lower().find('ground')!=-1:#
        #print(root)
        #ipdb.set_trace()
        #Getting target directories to write files to. 
        img_match=[img_type for img_type in img_types_spec if root.lower().find(img_type)!=-1]
        trgt_path_match=[trgt_root_dir for trgt_root_dir in trgt_dir_roots if trgt_root_dir.find(img_match[0])!=-1]
        #sorting file into appropriate naming list
        files=sorted(files)
        file_info=root.split('\\')
        pat_no=file_info[4]
        print(pat_no)
        img_type=file_info[5].lower()
        
        #Getting broad image types        
        slc_no_st=1
        #Iterate through files for renaming clause
        for file in files:
            #Creating dictoinary of values
            
            if img_type=='t1dual':
                st_file_name=['pat_id_'+pat_no+'_'+img_type+'_inphase_slice_no_'+str(slc_no_st),
                             'pat_id_'+pat_no+'_'+img_type+'_outphase_slice_no_'+str(slc_no_st)]
                grey_lvl_sub_str=sorted(['t1o_64grey_lvl','t1o_256grey_lvl','t1i_64grey_lvl','t1i_256grey_lvl'])
            elif img_type=='t2spir':
                st_file_name=['pat_id_'+pat_no+'_'+img_type+'_slice_no_'+str(slc_no_st)]
                grey_lvl_sub_str=sorted(['64grey_lvl','256grey_lvl'])
            else:
                print('error no matching values')
                
            
            trgt_path_match_dict=dict(zip(grey_lvl_sub_str,trgt_path_match))
            
            #reading file in for renaming file for method.
            src_file_path=os.path.join(root,file) 
            tmp_arr=imageio.imread(src_file_path)

            #adding grey scale and image size
            for k,v in trgt_path_match_dict.items():
                if k.find('t1i_')!=-1:
                    final_f_name=st_file_name[0]+'_'+k[len('tli_'):]+'_256x256.png'
                elif k.find('t1o_')!=-1:
                    final_f_name=st_file_name[0]+'_'+k[len('t1o_'):]+'_256x256.png'
                else:
                    final_f_name=st_file_name[0]+'_'+k+'_256x256.png'
                strp_dbl_undr_scr=final_f_name.replace('__','_')
                trgt_path_match_dict[k]=os.path.join(v,strp_dbl_undr_scr).lower()
                imageio.imwrite(trgt_path_match_dict[k],tmp_arr)
                #Bytescaling image for analysis
            if int(pat_no)>9:
                print(trgt_path_match_dict)
                

        #Increasing slice no by 1 
            slc_no_st+=1
                    
            
def insert_str(string,sub_str, str_to_insert, index):
    """the purpose of this method is to insert strings at specific indices"""
    return string[:index+len(sub_str)] + str_to_insert + string[index+len(sub_str):]<jupyter_output><empty_output><jupyter_text>Stripping out file information Patient id etc from file name using regex<jupyter_code>final_lst=[]
for file in mask_file_list:
    
    Pat_id_act=int(file[file.lower().find(Pat_str)+len(Pat_str):file.lower().find(img_type_str)])
    
    tmp_img_type=[x for x in img_types_lst if file.lower().find(x)!=-1]
    
    #Getting slice no from file name
    slic_no_lwr_bnd=file.lower().find('slice_no_')
    slic_no_uppr_bnd=file.lower().rfind('_512x512')
    slice_no=int(file[slic_no_lwr_bnd+len('slice_no_'):slic_no_uppr_bnd])
    
    tmp_img=imageio.imread(file)
    
    tmp_img_uniq_val=np.unique(tmp_img,return_counts=True)
    tmp_dict={}
    for int_val,pixel_cnts in list(zip(tmp_img_uniq_val[0],tmp_img_uniq_val[1])):
        int_val_key=mask_intensities[int_val]
        
        tmp_dict[int_val_key]=pixel_cnts
    
    tmp_dict['slice_no']=slice_no
    tmp_dict['image_type']=tmp_img_type[0]
    tmp_dict['Patient_id']=Pat_id_act
    
    final_lst.append(tmp_dict)<jupyter_output><empty_output><jupyter_text>Rewriting ground file intensity values to between 0 and 4<jupyter_code>src_dir=r'D:\Biomedical images\Train\NIFTI_MR_256x256_seg_rename - Copy'
dst_dir=r'D:\Biomedical images\Train\NIFTI_MR_256x256_seg_rename - Copy'
int_conv_dict={0:0,63:1,126:2,189:3,252:4}
conv_grnd_img_sing_label(src_dir,dst_dir,int_conv_dict)
def conv_grnd_img_sing_label(src_dir,dst_dir,int_conv_dict):
    """Generate single number iterative from i.e from intensity values 0,63,126,189,252 to 0,1,2,3,4,5"""
    
    for root,subdir,files in os.walk(src_dir):
        #ipdb.set_trace()
        if (root.find('ground')!=-1 or root.find('masks')!=-1) and len(files)>0:
            #Generating temporary file for analysis
            for file in files:
                
                #Importing image for analysis 
                tmp_path=os.path.join(root,file)
                print(tmp_path)
                tmp_img=imageio.imread(tmp_path)
                
                #Performing image conversion to single integer
                try:
                    tmp_img_conv=np.vectorize(int_conv_dict.get,otypes=[np.uint8])(tmp_img)
                    print(np.unique(tmp_img_conv))
                    
               
                    imageio.imwrite(tmp_path,tmp_img_conv)
                except TypeError as e:
                    print('Image already converted:',tmp_path)<jupyter_output><empty_output><jupyter_text>Generate Fold text files denoting validation and training and test folders for analysis<jupyter_code>#Analysis being be performed via fold method
str('x')<jupyter_output><empty_output><jupyter_text>Move data on a per patient basis to different folder set aka test or validation folders<jupyter_code>import shutil
dst_path=r'F:\Biomedical images\validation\1_pat_no_32'
pat_no=[32]

def non_subsample_patients(dst_path,src_path,pat_no,substr='inphase'):
    """the purpose of this method is to copy all patients that match number
    list pat_no and place into train and validation folders."""

    pat_no_str=['pat_id_'+str(x)+'_' for x in pat_no]
    for root,subdir,files in os.walk(src_path):
        if len(files)>0 and root.lower().find(substr)!=-1:
            root_split=root.split('\\')
            #Creating new directory based o nfile names
            root_split_rejoin='\\'.join(root_split[-3:])
            dst_dir=os.path.join(dst_path,root_split_rejoin)
            if not os.path.exists(dst_dir):
                os.makedirs(dst_dir)
            file_nm_lst=[]
            for file in files:
                
                file_chk=[x for x in pat_no_str if file.find(x)!=-1]
                #Copying file sthat match patient id. 
                if len(file_chk)==1:
                    #ipdb.set_trace()
                    print(file)
                    src_img_path=os.path.join(root,file)
                    dst_img_path=os.path.join(dst_dir,file)
                    shutil.copy(src_img_path, dst_img_path)
                    file_nm_lst.append(os.path.basename(file))
    #Return tuple list of test file.
    return file_nm_lst<jupyter_output><empty_output><jupyter_text>Rename Ground files to match image source file names<jupyter_code>cd F:\Biomedical images\Train\NIFTI_MR_512x512 seg_rename\NIFTI_MR_512x512
grey_lvl_str=['128','256','64']
phase_lvl_str={'inphase':'_inphase','outphase':'_outphase','t2spir':''}
for root,subdir,files in os.walk(os.getcwd()):
    
    if root.find('ground')!=-1:
        #Determine grey level setpoint for analysis 
        grey_lvl_sp=[x for x in grey_lvl_str if root.find(x)!=-1]
        grey_lvl_str_add='_grey_lvl_'+grey_lvl_sp[0]+'.png'
        #Determine ground source level
        phase_lvl_sp=[v for k,v in phase_lvl_str.items() if root.find(k)!=-1]
        
        grnd_sub_repl_tple_dict={'_ground':phase_lvl_sp[0],'.png':grey_lvl_str_add}

        for file in files:
            new_file_name=repl_sub_string(file,grnd_sub_repl_tple_dict)
            old_img=os.path.join(root,file)
            new_img=os.path.join(root,new_file_name)
            #renaming tuple to match srounce image
            os.rename(old_img,new_img)

#Check to confirm for os path to determine method for analysis 
for root,subdir,files in os.walk(os.getcwd()):
    if len(subdir)==2:        

        dir_lst=[os.path.join(root,x) for x in subdir]
        
        file_lst_tple=list(zip(sorted(os.listdir(dir_lst[0])),
                               sorted(os.listdir(dir_lst[1]))))
                           
        for file_src,file_grnd in file_lst_tple:
            
            if file_src!=file_grnd:
                print(file_src)
                print(file_grnd)<jupyter_output><empty_output><jupyter_text>Function calls for generating file_dictionaries, file tuple sets<jupyter_code>def gen_new_dir(dir_name):
    """Generate directory based on directory existing or not"""

    
    if not os.path.exists(directory):
        
        try:
            os.makedirs(directory)
        except OSError as error:
            if error.errno != errno.EEXIST:
                raise
    with open(filepath, 'w') as my_file:
        do_stuff(my_file)
def gen_img_fileset_dict(src_dir,imgs_modes,key_word_sp=1,len_file_sp=0):
    """The purpose of this method is to create a keyword dictoinary of a folders subfolder file directory paths"""
    file_dict={}
    for subdir,root,files in os.walk(src_dir):
        print(subdir)

        if len(files)!=len_file_sp:
            gt_img_name=[]
            gt_img_name=[''.join(x) for x in imgs_modes if x in subdir.lower()]
            #If two categories 
            if len(gt_img_name)>=key_word_sp:

                tmp_file_lst=[os.path.join(subdir,file) for file in files]
                seperator = '_'
                seperator =seperator.join(gt_img_name)
                file_dict[seperator]=sorted(tmp_file_lst)
    #Sett original name for analysis and munging
    return file_dict
def gen_file_tpl_set_lists(file_dict:dict,pair_keyword: list)->list:
    """The purpose of this method is to generate keyword base file tuple sets"""
    #final list to return list
    rtrn_lst=[]
    #Creating list of keys to iterate through 
    file_sets=list(file_dict.keys()) 
    
    for key in pair_keyword:
        #ipdb.set_trace()
        #Get main image set initially b yimage type i.e. t1dual
        img_type_lst=[x for x in file_sets if x.lower().find(key[0])!=-1]
        #Getting the ground key
        img_grnd_key=[x for x in img_type_lst if x.lower().find(key[1])!=-1]
        #Getting non ground by removing that value from the list 
        img_type_lst.remove(img_grnd_key[0])
        #Generating file list tuple pairs for anaylysis=
        file_pair=list(zip(file_dict[img_type_lst[0]],file_dict[img_grnd_key[0]]))
            
        #Appending final list of tuple for analysis
        rtrn_lst.append(file_pair)
            
    return rtrn_lst

def repl_sub_string(str_val:str,str_rep_dict:dict):
    """The purpose of this method is to itertivley replace substrings in a string with other values"""
    
    # Add escape characters to string keys for regex estimation
    rep = dict((re.escape(k), v) for k, v in str_rep_dict.items()) 
    #Python 3 renamed dict.iteritems to dict.items so use rep.items() for latest versions
    pattern = re.compile("|".join(str_rep_dict.keys()))
    
    return pattern.sub(lambda m: rep[re.escape(m.group(0))],str_val)
def rename_grnd_src_img(file_set_list_tuple:list,
                                     grnd_sub_repl_tple_dict:dict,
                                    src_sub_repl_tple_dict:dict,
                                     dst_grnd_path:str,
                                    dst_img_path:str):
    """The purpose of this method is to detail how to rename ground images to the source image and write them both 
    to a new folder. """
    
    
    for grnd_nm_path,img_nm_path in file_set_list_tuple:
        
        grnd_nm=os.path.basename(grnd_nm_path)
        img_nm=os.path.basename(img_nm_path)
        
        strp_grnd_nm=repl_sub_string(grnd_nm,grnd_sub_repl_tple_dict)
        strp_img_nm=repl_sub_string(img_nm,src_sub_repl_tple_dict)
        
        dst_grnd_path=os.path.join(dst_grnd_path,strp_grnd_nm)
        dst_img_path=os.path.join(dst_img_path,strp_img_nm)
        
        copyfile(grnd_nm_path, dst_grnd_path)
        copyfile(img_nm_path,dst_img_path)<jupyter_output><empty_output><jupyter_text>Sub sampler creating folder called fold1 and fold2 for specific analysis in set folder<jupyter_code>set_up_dir=os.path.join(os.path.basename(dst_directory_src),'setup')


gen_fold_text_folder(set_up_dir,'fold1.txt',file_tuple_set_train)
def gen_fold_text_folder(dst_dir:str,txt_f_nm_trn:str,file_tuple_set:list):
    """Generate test file to write files to folders"""
    final_lst=[]
    for file in file_tuple_set:
        if type(file) is tuple and len(file)>1:
            
            final_lst.append(os.path.splitext(os.path.basename(file[0]))[0])
        else:
            final_lst.append(os.path.splitext(os.path.basename(file))[0])
    dst_file=os.path.join(dst_dir,txt_f_nm_trn)
    #Writing file of list strings to final folder. 
    
    with open(dst_file,'w') as f_dst:
        
        for item in final_lst:
            f_dst.write("%s\n" % item)
    
    return final_lst
        
        
    
file_tuple_set_train<jupyter_output><empty_output><jupyter_text>Sub sampler by patient to specific destination folders<jupyter_code>dst_directory_src<jupyter_output><empty_output><jupyter_text>Sub sample for U-net<jupyter_code>#Analysis 
file_list_nm=os.listdir('F:\Biomedical images\Train\MR_raw')
file_list_nm_test_val=['3','8','39','2','32']
file_list_nm_val=['3','32']
file_list_nm_test=[x for x in file_list_nm_test_val if x not in file_list_nm_val]
file_list_nm_train=[x for x in file_list_nm if x not in file_list_nm_test_val]

samp_size=50
src_path=r'F:\Biomedical images\Train\NIFTI_MR_256x256_seg_rename'
dst_path=r'F:\Biomedical images\Train\sub sample sets\NIFTI_MR_256x256_'+str(samp_size)+'_train_Unet'
dst_train=os.path.join(dst_path,'train')
dst_val=os.path.join(dst_path,'valid')
dst_test=os.path.join(dst_path,'test')

pat_subsampler(src_path,dst_train,
               file_list_nm_train,
               samp_size)

val_list_files=non_subsample_patients(dst_val,
                                      src_path,
                                      file_list_nm_val)

test_list_files=non_subsample_patients(dst_test,
                                       src_path,
                                       file_list_nm_test)<jupyter_output><empty_output><jupyter_text>Sub sample for Segcaps<jupyter_code>#Analysis 
file_list_nm=os.listdir('F:\Biomedical images\Train\MR_raw')
file_list_nm_test_val=['3','8','39','2','32']
file_list_nm_val=['3','32']
file_list_nm_test=[x for x in file_list_nm_test_val if x not in file_list_nm_val]
file_list_nm_train=[x for x in file_list_nm if x not in file_list_nm_test_val]
samp_size=50

#Destination to path to analysis 
dst_path=r'F:\Biomedical images\Train\sub sample sets\NIFTI_MR_256x256_'+str(samp_size)+'_train_segcaps'
dst_train=os.path.join(dst_path,'train')
pat_subsampler(r'F:\Biomedical images\Train\NIFTI_MR_256x256_seg_rename',
               dst_train,
               file_list_nm_train,samp_size)

val_list_files=non_subsample_patients(dst_train,
                                  r'F:\Biomedical images\Train\NIFTI_MR_256x256_seg_rename',
                                  file_list_nm_val)

test_list_files=non_subsample_patients(dst_train,
                                  r'F:\Biomedical images\Train\NIFTI_MR_256x256_seg_rename',
                                  file_list_nm_test)
file_setup_path=os.path.join(dst_path,r'train\NIFTI_MR_256x256_png_256grey_lvl\t1dual_inphase\setup')
file_setup_path
#file_setup_path=os.path.join(dst_train,'NIFTI_MR_256x256_png_256grey_lvl\t1dual_inphase\setup')

gen_fold_text_folder(file_setup_path,'fold2.txt',val_list_files)
gen_fold_text_folder(file_setup_path,'test.txt',test_list_files)
int_conv_dict={0:0,63:1,126:2,189:3,252:4}

src_dir=dst_dir=r'F:\Biomedical images\Train\sub sample sets\NIFTI_MR_256x256_50_train_segcaps\train'

conv_grnd_img_sing_label(src_dir,dst_dir,int_conv_dict)<jupyter_output><empty_output><jupyter_text>Sub sampling patients general functions and setup<jupyter_code>dst_directory_src,dst_directory_grnd,file_tuple_set=pat_subsampler(r'F:\Biomedical images\Train\NIFTI_MR_256x256_seg_rename',
                                                                   r'F:\Biomedical images\Train\sub sample sets\NIFTI_MR_256x256_250train',
                                                                   file_list_nm_train,250)

write_subsmpl_file(dst_directory_src,dst_directory_grnd,file_tuple_set)
def pat_subsampler(data_src_path,data_dst_path,pat_key_word:list,
     No_imgs:int,Pat_weights={},
                   sampling='Uniform',pat_str='pat_id_',subdir_imgs=['masks','images']):
    """The purpose of this method is to subsample from each patient an even number of times to ensure no patient specific
    bias is introduced into the process."""
    #Convert patient numbers to stirng 
    str_pat_no_lst=[str(x) for x in pat_key_word if type(x)==str]
    #Confirm file arguments match requiremens
    assert (sampling.lower() in ['uniform','weighted']), "Correct sampling method argument uniform or weighted not inputed"
    #Getting different sampling method if required. 
    pat_dict=subsample_pat_weight(sampling,No_imgs,
                                  str_pat_no_lst,
                                  Pat_weights)
    #ipdb.set_trace()   
    #Iterate through work directory
    for root,subdir,files in os.walk(data_src_path):
        #If two subdirectories found denoted as containing ground and source proceed to second step for analysis 
        sub_dir_chk=[x for x in subdir if x.lower() in subdir_imgs]
        
        if len(sub_dir_chk)==2:
            
            dst_directory_src,dst_directory_grnd,dst_dir_setup=gen_dst_grnd_src_path(root,data_dst_path)
            
            file_dict=gen_img_fileset_dict(root,subdir,key_word_sp=1)
            keys=list(file_dict.keys())
            
            tmp_file_set_sampl=file_dict[keys[0]]
            
            #Getting patient ids
            final_pat_idxs=subsample_patient_idx(tmp_file_set_sampl,pat_dict,pat_str)
                
            file_grnd_files=[file_dict[keys[0]][i] for i in final_pat_idxs]
            file_img_files=[file_dict[keys[1]][i] for i in final_pat_idxs]
                
            file_tuple_set=list(zip(file_img_files,
                           file_grnd_files))
            
            write_subsmpl_file(dst_directory_src,dst_directory_grnd,file_tuple_set)
            #Writing tupe set to file for analysis
            #Generatng setup file for analysis
            #ipdb.set_trace()
            gen_fold_text_folder(dst_dir_setup,'fold1.txt',file_tuple_set)
            
def write_subsmpl_file(dst_directory_src,dst_directory_grnd,file_tuple_set):
    """Write subsampled files to file"""
    for grnd_img,file_img in file_tuple_set:
        
        img_dst_b_name=os.path.basename(file_img)
        grnd_dst_b_name=os.path.basename(grnd_img)
        #Getting destinationnames for copying files across
        img_dst_f_name=os.path.join(dst_directory_src,img_dst_b_name)
        grnd_dst_f_name=os.path.join(dst_directory_grnd,grnd_dst_b_name)
        #Writing subsampled files to new folder set. 
        copyfile(file_img,img_dst_f_name)
        copyfile(grnd_img,grnd_dst_f_name)
def subsample_pat_weight(sampling_arg,
                         total_no_imgs,
                         pat_id_lst,
                         Pat_weights={}):
    """The purpose of this method is to return the number of samples that are to be
    returned per patient class bsaed on weighted input."""
    if sampling_arg.lower()=='uniform':
        img_per_pat=int(round(total_no_imgs/len(pat_id_lst),0))
        pat_dict={k:img_per_pat for k in pat_id_lst}
    else:
        pat_dict={k:int(round(v*total_no_imgs,0)) for k,v in Pat_weights}
        
    return pat_dict
def subsample_patient_idx(tmp_file_set_sampl:list,pat_dict:dict,pat_str='pat_id_'):
    """The purpose is to subsample patient ids indexes from dataset"""
    final_pat_idxs=[]
    #ipdb.set_trace()
    for pat_id,sampl_no in pat_dict.items():
        tmp_pat_str=pat_str+pat_id+'_'
        #Getting indices of specific patient files in list
        pat_indices = [i for i, s in enumerate(tmp_file_set_sampl) if tmp_pat_str in s.lower()]
        #Sampling form indices for analysis
        
        
        try:
    
            sampl_pat_ind=random.sample(pat_indices, sampl_no)
            #Add indices to final list
            final_pat_idxs=final_pat_idxs+sampl_pat_ind
            
        except ValueError as e:
            print(str(e))
            print('Oversampled on patient id:',tmp_pat_str)
            print('samples available:',len(pat_indices))
            print('samples required:',sampl_no)
            print('Max sub sampling performed instead')
            
            #Add indices to final list
            final_pat_idxs=final_pat_idxs+pat_indices
            
        
    return final_pat_idxs

def gen_dst_grnd_src_path(orig_root_dir,dst_path,file_strs={'masks':0,'images':0,'setup':0},
                          dir_depth=-2):
    """The purpose of this method is to generate source and ground destination paths directories
    for the subsampled files"""
    #ipdb.set_trace()
    #Generating directory 
    sub_dir=orig_root_dir.split('\\')[dir_depth:]
    #file_strs
    for img_types,img_paths in file_strs.items():
        
    #Generating source directories for file analysis
        tmp_directory=os.path.join(dst_path,*sub_dir,img_types)

        if not os.path.exists(tmp_directory):
            os.makedirs(tmp_directory)
            
        file_strs[img_types]=tmp_directory
        
    dir_grnd=file_strs['masks']
    dir_img=file_strs['images']
    dir_setup=file_strs['setup']
    
    return (dir_img,dir_grnd,dir_setup)
            
        
img_per_pat=20
path_key_word=[1,2,3,4,5,6]

pat_dict={k:img_per_pat for k in path_key_word}
pat_dict
tr_path=os.path.join('C://trial_path',*root.split('\\')[-2:],'trl')
tr_path
trl_lst=[]

for root,subdir,files in os.walk(os.getcwd()):
    
    if len(subdir)==0:
        #print(root.split('\\')[-2:])
        print(root)
        tmp_file_set=[os.path.join(root,file) for file in files]
    
        trl_lst=trl_lst+tmp_file_set<jupyter_output><empty_output><jupyter_text>Grey level intensity methods for grey level setting<jupyter_code>def bytescaling(data, cmin=None, cmax=None, high=255, low=0):
    """
    Converting the input image to uint8 dtype and scaling
    the range to ``(low, high)`` (default 0-255). If the input image already has 
    dtype uint8, no scaling is done.
    :param data: 16-bit image data array
    :param cmin: bias scaling of small values (def: data.min())
    :param cmax: bias scaling of large values (def: data.max())
    :param high: scale max value to high. (def: 255)
    :param low: scale min value to low. (def: 0)
    :return: 8-bit image data array
    """
    if data.dtype == np.uint8:
        return data

    if high > 255:
        high = 255
    if low < 0:
        low = 0
    if high < low:
        raise ValueError("`high` should be greater than or equal to `low`.")

    if cmin is None:
        cmin = data.min()
    if cmax is None:
        cmax = data.max()

    cscale = cmax - cmin
    if cscale == 0:
        cscale = 1

    scale = float(high - low) / cscale
    bytedata = (data - cmin) * scale + low
    return (bytedata.clip(low, high) + 0.5).astype(np.uint8)
def quantize_igs(im, levels, qtype='uniform', maxCount=255, displayLevels=None):
    """
    Function to run uniform gray-level and improved gray-scale Quantization.
    This takes in an image, and buckets the gray values depending on the params.
    Args:
        im (array): image to be quantized as an array of values from 0 to 255
        levels (int): number of levels to quantize to.
            This should be a positive integer, and smaller than the maxCount.
        qtype (optional[string]): the type of quantization to perform.
            Can be either 'uniform' or 'igs'; Defaults to 'uniform'.
        maxCount (optional[int]): the maximum value for a digital count
        displayLevels (optional[int]): the number of gray levels to expand to.
            By default this value is None and will shrink the range of greys.
            This value should be a positive integer when provided.
    Return:
        the quantized image
    """
    # default value if we need to return early
    returnImage = im

    # get int type
    dtype = im.dtype

    if (displayLevels == None):
        # by default don't re-expand the image
        displayCount = levels
    elif displayLevels > 0:
        displayCount = displayLevels-1
    else:
        print("displayLevels is an invalid value")
        return returnImage

    # we're getting one more level than we should be, so minus 1
    if ((levels > 0) and (levels < maxCount)):
        levels = levels - 1
    else:
        print("levels needs to be a positive value, and smaller than the maxCount")
        return returnImage

    if (qtype == 'uniform'):
        # uniform method from lecture
        returnImage = np.floor((im/((maxCount+1)/float(levels))))*(displayCount/levels)

    elif (qtype == 'igs'):
        # error diffusion method from lecture

        # default error as 0 for the first pixel
        error = 0

        # the list of rows that will be turned into an image
        returnList = []
        for i in range(len(im)):
            returnRow = []
            for j in range(len(im[i])):
                # get a new digital count with the error
                errDC = im[i][j] + error
                # save the error for the next pixel
                error = errDC % (maxCount/levels)

                # calculate the new digital count, and append it to the row
                newDC = np.floor((errDC)/(maxCount/levels))
                returnRow.append(newDC*(displayCount/levels))
            # append the row to the final image
            returnList.append(np.array(returnRow))

        returnImage = np.array(returnList, dtype)

    else:
        # invalid qtype
        print('qtype is an invalid value, please use "uniform", or "igs"')

   
    return np.array(returnImage, dtype)<jupyter_output><empty_output><jupyter_text>Determine sub sampling based on class balance<jupyter_code>file_class_dict={}
for file in files_list:
    #
    tmp_key_set={0:0,126:0,252:0,63:0,184:0}
    #Assess file name and type
    tmp_file=imageio.imread(file)
    uniq_val_class_bal=np.unique(tmp_file)
    tuple_set_vals=list(zip(list(uniq_val_class_bal[0]),
                      list(uniq_val_class_bal[1])))
    for int_val,cnt in tuple_set_vals:
        if int_val in tmp_key_set.keys():
            tmp_key_set[int_val]=cnt
    #Ordering dictionary based on values
    ord_dict = OrderedDict(sorted(trl_dict.items()))
    arr_val=np.array(list(od.values()))
    arr_val_norm=arr_val/arr_val.sum()
    #Creating unique key value for class balance to file name. 
    file_class_dict[arr_val_norm]=file
    
    


if 1 in trl_dict.keys():
    print('yes')
from collections import OrderedDict
import numpy as np
trl_dict={1:2,3:4,66:4}
od = OrderedDict(sorted(trl_dict.items()))
arr_val=np.array(list(od.values()))
arr_val_norm=arr_val/arr_val.sum()
arr_val_norm<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09699245879161823, 'fraction_numerical': 0.015857402605144715, 'mean_word_length': 2.9517078916372204, 'pattern_counts': {'":': 0, '<': 76, '<?xml version=': 0, '>': 84, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '580002', 'license': 'MIT', 'n_tokens_mistral': 11666, 'n_tokens_neox': 11132, 'n_words': 2247, 'path': 'file_infrastructure_boiler_plate_mk3.ipynb', 'repo_name': 'Larkinnjm1/Master_thesis_infrastructure', 'size': '33549'}