<jupyter_start><jupyter_text>--- Resample data to achieve more consistent training series- assuming that data has been organized into folders as ./base/xxxxx_class/imgxxx.jpg---<jupyter_code>import os
import sys
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.utils import resample
import matplotlib.pyplot as plt
import shutil as sh

def mean(vec):
    return sum(vec)/len(vec) if len(vec) != 0 else None
"""def resample(X, y, sample_type=None, sample_size=None, class_weights=None, seed=None):

    # Nothing to do if sample_type is 'abs' or not set.  sample_size should then be int
    # If sample type is 'min' or 'max' then sample_size should be float
    if sample_type == 'min':
        sample_size_ = np.round(sample_size * y.value_counts().min()).astype(int)
    elif sample_type == 'max':
        sample_size_ = np.round(sample_size * y.value_counts().max()).astype(int)
    else:
        sample_size_ = max(int(sample_size), 1)

    if seed is not None:
        np.random.seed(seed)

    if class_weights is None:
        class_weights = dict()

    X_resampled = pd.DataFrame()

    for yi in y.unique():
        size = np.round(sample_size_ * class_weights.get(yi, 1.)).astype(int)

        X_yi = X[y == yi]
        sample_index = np.random.choice(X_yi.index, size=size)
        X_resampled = X_resampled.append(X_yi.reindex(sample_index))

    return X_resampled
"""<jupyter_output><empty_output><jupyter_text>---<jupyter_code>#sourceDir = '../dataCropSQ/Img/'
sourceDir = '../data/Img/'
#trainTargetDir = '../dataCropSQRS/train/upper'
#testTargetDir = '../dataCropSQRS/test/upper'
#sourceDir = '/Volumes/SanDiskSSD/BFH_backup/CAS_PML/Project/DeepFashion/dataCropSQ/Img/'
#trainTargetDir = '/Volumes/SanDiskSSD/BFH_backup/CAS_PML/Project/DeepFashion/fdata_CropSQRS/train'
#testTargetDir = '/Volumes/SanDiskSSD/BFH_backup/CAS_PML/Project/DeepFashion/fdata_CropSQRS/test'
with open('../data/Anno/list_category_cloth.txt','r') as fp:
    lines = [ l.rstrip().replace('1','upper') \
                        .replace('2','lower') \
                        .replace('3','full').split() for l in fp][2:]  
    mainCatagories = { l[0]:l[1] for l in lines}
classDict = {}
for d,_,files in os.walk(sourceDir,topdown=False):
    cl_name = d.split('_')[-1]
    if mainCatagories.get(cl_name) is None: continue
    if classDict.get(cl_name) is None: classDict[cl_name] = []
    classDict[cl_name].extend([ d + '/' + f for f in files \
                                   if not f.startswith('.') or not f.endswith('.jpg') ])

# clean out some old classes which were ignored
bunkKeys = []
for cl_name in mainCatagories.keys():
    if classDict.get(cl_name) is None: bunkKeys.append(cl_name)
        
# can't do it in one step, because you're modifing the object you're working on
for b in bunkKeys:
    mainCatagories.pop(b,None)
classCount = { k:len(v) for k,v in classDict.items()}
print('Total image files',sum([v for v in classCount.values()]))
print('UpperClass:',sum([classCount[cl] for cl,ca in mainCatagories.items() if ca == 'upper' ]))
print('LowerClass:',sum([classCount[cl] for cl,ca in mainCatagories.items() if ca == 'lower' ]))
print('FullClass:',sum([classCount[cl] for cl,ca in mainCatagories.items() if ca == 'full' ]))
names = sorted(classCount.items(),key=lambda kv: kv[1])
namesSorted = [ v[0] for v in names]
valuesSorted = [ v[1] for v in names]
labelsSorted = [ mainCatagories[n] for n in namesSorted]
colorSorted = []
for n in namesSorted:
    color =      'red'  if mainCatagories[n] == 'upper' \
            else 'blue' if mainCatagories[n] == 'lower' \
            else 'green'
    colorSorted.append(color)

plt.figure(figsize=(15,5))
plt.bar(range(len(valuesSorted)),valuesSorted,color=colorSorted)
plt.xticks(range(len(valuesSorted)),namesSorted,rotation='vertical')
plt.title('Class distribution of dataset',fontsize=16)
plt.show()
def plotClassCount(clCountDict,title='',color='blue'):
    names = sorted(clCountDict.items(),key=lambda kv: kv[1])
    namesSorted = [ v[0] for v in names]
    valuesSorted = [ v[1] for v in names]
    plt.bar(range(len(valuesSorted)),valuesSorted,color=color)
    plt.xticks(range(len(valuesSorted)),namesSorted,rotation='vertical')
    plt.grid(color='grey',axis='y',linestyle='--', linewidth=0.5)
    plt.title(title,fontsize=16,fontweight='bold')
    plt.show()
    
def summarizeClassCount(clCountDict):
    print(' min=',min(clCountDict.values()))
    print(' avg=',mean(clCountDict.values()))
    print(' max=',max(clCountDict.values()))
catagories = ['upper','lower','full']
ctClDict = { ct:{ k:v for k,v in classDict.items() if mainCatagories[k] == ct} for ct in catagories }
ctClCount = { ct:{ k:len(v) for k,v in ctClDict[ct].items()} for ct in catagories }
print('Summary:  Upper Class')
summarizeClassCount(ctClCount['upper'])
plotClassCount(ctClCount['upper'],title='Upper Category distribution',color='red')
print('Summary:  Lower Class')
summarizeClassCount(ctClCount['lower'])
plotClassCount(ctClCount['lower'],title='Lower class distribution',color='blue')
ctClCount['lower']['Sweatpants']
print('Summary: Full Class')
summarizeClassCount(ctClCount['full'])
plotClassCount(ctClCount['full'],title='Full class distribution',color='green')
print('Robe=',ctClCount['full']['Robe'])
print('Coat=',ctClCount['full']['Coat'])
threshhold = 2000
selectedClasses = {ct:[k for k,v in ctClCount[ct].items() if v >= threshhold] for ct in catagories}

selectedClasses
testSplit = 0.15
trainClDict = {}
testClDict = {}
for ct,cls in selectedClasses.items():
    trainClDict[ct] = {}
    testClDict[ct] = {}
    for cl in cls:
        trainClDict[ct][cl], testClDict[ct][cl] = train_test_split(ctClDict[ct][cl],test_size = testSplit)
print('train[Hoodie]=',len(trainClDict['upper']['Hoodie']))
print(' test[Hoodie]=',len(testClDict['upper']['Hoodie']))
print(' percentage Check=',len(testClDict['upper']['Hoodie'])/len(trainClDict['upper']['Hoodie']))
trainClDictRS = {}
testClDictRS = {}
nsamples = 5000
nsamples_t = int(nsamples*testSplit)

for ct in trainClDict.keys():
    trainClDictRS[ct] = {}
    testClDictRS[ct] = {}
    for cl in trainClDict[ct].keys():
        useReplace_train = not len(trainClDict[ct][cl]) > nsamples
        useReplace_test = not len(testClDict[ct][cl]) > nsamples_t
        trainClDictRS[ct][cl] = resample(trainClDict[ct][cl],n_samples=nsamples,replace=useReplace_train)
        testClDictRS[ct][cl] = resample(testClDict[ct][cl],n_samples=nsamples_t,replace=useReplace_test)
len(trainClDictRS['upper']['Hoodie'])
len(testClDictRS['upper']['Hoodie'])
plt.bar(range(len(list(trainClDictRS['upper'].keys()))),
        [len(v) for k,v in trainClDictRS['upper'].items()],
        color='red')
plt.xticks(range(len(list(trainClDictRS['upper'].keys()))), trainClDictRS['upper'].keys(),rotation='vertical')
plt.title('Upper category, after Re-sampling', fontsize=16, fontweight='bold')
plt.show()
plt.bar(range(len(list(trainClDictRS['lower'].keys()))),
        [len(v) for k,v in trainClDictRS['lower'].items()],
        color='blue')
plt.xticks(range(len(list(trainClDictRS['lower'].keys()))), trainClDictRS['lower'].keys(),rotation='vertical')
plt.title('Lower category, after Re-sampling', fontsize=16, fontweight='bold')
plt.show()
plt.bar(range(len(list(trainClDictRS['full'].keys()))),
        [len(v) for k,v in trainClDictRS['full'].items()],
        color='green')
plt.xticks(range(len(list(trainClDictRS['full'].keys()))), trainClDictRS['full'].keys(),rotation='vertical')
plt.title('Full category, after Re-sampling', fontsize=16, fontweight='bold')
plt.show()<jupyter_output><empty_output><jupyter_text>--- Copy files to new training directory- resampling of redundant files will be handled by the augmenter---<jupyter_code>def copyAndRenameFile(sourceFile,classTargetDir):
    # if file exists, renames the file to unique name
    labelDir,imgFileOrig = sourceFile.split('/')[-2:]
    fullTargetDir = os.path.join(classTargetDir,labelDir)
    if not os.path.exists(fullTargetDir): os.mkdir(fullTargetDir)
        
    ind = 0
    imgFile = imgFileOrig
    #print(imgFile)
    while os.path.exists(os.path.join(fullTargetDir,imgFile)):
        name,ext = imgFileOrig.split('.')
        imgFile = name + f"R{ind:03d}." + ext
        ind += 1
    
    sh.copy(sourceFile,fullTargetDir + '/' + imgFile)
print('sourceDir=',sourceDir)
assert os.path.exists(sourceDir),'Failed to find sourceDir'
print('trainTargetDir=',trainTargetDir)
assert os.path.exists(trainTargetDir),'Failed to find trainTargetDir'
print('testTargetDir=',testTargetDir)
assert os.path.exists(testTargetDir),'Failed to find testTargetDir'
for ct,cls in trainClDictRS.items():
    ctDir = os.path.join(trainTargetDir,ct)
    if not os.path.exists(ctDir): os.mkdir(ctDir)
    for cl,files in cls.items():
        clDir = os.path.join(ctDir,cl)
        if not os.path.exists(clDir): os.mkdir(clDir)
        for f in files:
            copyAndRenameFile(f,clDir)
            break  # remove<jupyter_output>img_00000049.jpg
img_00000042.jpg
img_00000082.jpg
img_00000015.jpg
img_00000005.jpg
img_00000021.jpg
img_00000001.jpg
img_00000049.jpg
img_00000141.jpg
img_00000074.jpg
img_00000013.jpg
img_00000037.jpg
img_00000031.jpg
img_00000023.jpg
img_00000007.jpg
img_00000017.jpg
img_00000015.jpg
img_00000054.jpg
img_00000011.jpg
img_00000061.jpg<jupyter_text>Visualize the validation class distribution<jupyter_code>valDir = '../fashion_data_class/val/''<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15676359039190899, 'fraction_numerical': 0.021807838179519597, 'mean_word_length': 4.936835522201376, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 3, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '362781', 'license': 'MIT', 'n_tokens_mistral': 3474, 'n_tokens_neox': 3264, 'n_words': 638, 'path': 'DeepFashionAnalysis/classDetect/ResampleDataset.ipynb', 'repo_name': 'RexBarker/DeepF', 'size': '9492'}