<jupyter_start><jupyter_text>Read png file pathes and make txt files for MV_CNN code<jupyter_code>import sys
#import numpy as np
import glob
#import re
#from PIL import Image
import random
import pandas as pd
import os


RootFolder = '/Shared/CTmechanics_COPDGene/Amin/MV_CNN_views'
subjects   = os.listdir(RootFolder)

airwayPhenotypes = pd.read_csv('Data/airway_phenotypes_for_fgf10_replication.csv',index_col=0)
cleanup_nums = {"Lower_lobe_segmental_status":     {"Standard": 0, "Acc. B*": 1, "Abs. RB7": 1}}
airwayPhenotypes.replace(cleanup_nums, inplace=True)
#print(airwayPhenotypes.head(2))
#sid = intersection(subjects,airwayPhenotypes.index.tolist())
#print(len(sid))
print(airwayPhenotypes.shape)

images = glob.glob(RootFolder+'/**/*.png')
airwayPhenotypes = airwayPhenotypes.loc[airwayPhenotypes.index.intersection(subjects)]
subjects = airwayPhenotypes.index.tolist()

print(airwayPhenotypes.shape)
print(len(subjects))
print(len(images))<jupyter_output>135984<jupyter_text>Bundle png images in txt file for use in MV-CNN<jupyter_code>#order = range(48)#[0,1,4,5,6,7,8,9,10,11,2,3]
df = airwayPhenotypes.loc[:,['Lower_lobe_segmental_status']]
try:
    os.mkdir(os.path.join(RootFolder,'txtImages'))
except:
    print('txtImages already exists')
try:
    os.mkdir(os.path.join(RootFolder,'sets'))
except:
    print('sets already exists')

    #df.rename(index=str, columns={"A": "a"
for subject in subjects:
    folder = os.path.join(RootFolder,subject)
    images = glob.glob(folder+'/*.png')
    images.sort()
    pre = [1, len(images)]
    pre = [airwayPhenotypes.loc[subject,'Lower_lobe_segmental_status'], len(images)]
    filename   = subject+'_Cat_{}.txt'.format(pre[0])
    thefile = open(os.path.join(RootFolder,'txtImages',filename), 'w')
    images = pre+images#[ images[i] for i in order]
    for item in images:
        thefile.write("%s\n" % item)
    thefile.close()
    df.loc[subject,'txtImages'] = os.path.join(RootFolder,'txtImages',filename)
    #print(subject)
print(df.head(2)) 
random.seed(0)
#random.shuffle(images)
images = df.sample(frac=1,random_state=1)
print(images.head(2))<jupyter_output>Lower_lobe_segmental_status  \
UA240677                            0   
LA190608                            0   

                                                  txtImages  
UA240677  /Shared/CTmechanics_COPDGene/Amin/MV_CNN_views...  
LA190608  /Shared/CTmechanics_COPDGene/Amin/MV_CNN_views...<jupyter_text>Make list of images for Train, Test and Validation<jupyter_code>n = len(images)
print(n)

sets = {'train':images[:int(n*.6)],
        'validation':images[int(n*.6):int(n*.8)],
        'test':images[int(n*.8):]}

#print(images.Lower_lobe_segmental_status.values)

c=1
for setName, imageList in sets.items():
    thefile = open(os.path.join(RootFolder,'sets',setName+'.txt'), 'w')
    #print(setName,imageList.shape)
    
    for sid, item in imageList.iterrows():
        line = '{} {}\n'.format(item.txtImages, 1)
        line = '{} {}\n'.format(item.txtImages, item.Lower_lobe_segmental_status)
        #print(line)
        thefile.write(line)
    thefile.close()
   
print(sid)
size = (128, 128)
print(size)
sys.getdefaultencoding()
import numpy as np
def stable_softmax(X):
    exps = np.exp(X - np.max(X))
    return exps / np.sum(exps)
sm = (stable_softmax([947,621]))
logsm = -np.log(sm)
print(sm,logsm)
print(sum(logsm*[934, 511])/1445)<jupyter_output>[1.00000000e+000 2.63026133e-142] [ -0. 326.]
115.28442906574395
{'fraction_non_alphanumeric': 0.15209235209235208, 'fraction_numerical': 0.03924963924963925, 'mean_word_length': 4.357032457496136, 'pattern_counts': {'":': 5, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '258790', 'license': 'MIT', 'n_tokens_mistral': 1388, 'n_tokens_neox': 1279, 'n_words': 211, 'path': 'Make_MVCNN_txt_Files.ipynb', 'repo_name': 'mamdamin/MVCNN-TensorFlow', 'size': '3465'}