<jupyter_start><jupyter_text>1. Inputs<jupyter_code>Zoom = 2

#these 2 values are for x5 cross-fold validation:
cfv_const= -2101
last_train_label = 7500
last_valid_label = 9300

ap = argparse.ArgumentParser()
ap.add_argument('--cfv', help="cross fold validation number", dest = 'cfv', default = 1,\
                type=int, choices=choices_)
ap.add_argument('--fromf', help="from folder name", dest = 'fromf',\
                type=str)
                #("--cnn", type=str, help="training cnn name", dest = 'train_cnn')
args = vars(ap.parse_args())
#data-from folder:
new_proc_folder = args["fromf"]
cfv = args["cfv"]
# test:
# cfv_const = -400
# new_proc_folder = cancer_s2
# cfv = 2
# input_df = test_cnn_labels
# wrapping inputs:
new_proc_folder = os.path.join(base_path, new_proc_folder)<jupyter_output><empty_output><jupyter_text>2. Standard functions   5. Distribute processed files among training and validation folders<jupyter_code>try:
    shutil.rmtree(train_cnn)
    shutil.rmtree(valid_cnn)
except OSError as ee:
    print ("Error: %s: " % ee)
os.mkdir(train_cnn)
os.mkdir(valid_cnn)
for score in gs_scores:
    os.mkdir(os.path.join(train_cnn, f'{score}'))
    os.mkdir(os.path.join(valid_cnn, f'{score}'))
# 
#cross-fold validation:
labels_perm = mask_labels.sample(frac = 1, random_state=npseed)
if cfv == 0: #take data without test:    
    train_cnn_labels = labels_perm[:last_train_label]
    valid_cnn_labels = labels_perm[last_train_label:last_valid_label]
elif cfv == 1:
    train_cnn_labels = labels_perm[:cfv_const]
    valid_cnn_labels = labels_perm[cfv_const:]
else:
    train_cnn_labels = pd.concat([labels_perm[:cfv_const*cfv], labels_perm[cfv_const*(cfv-1):]], sort=False)
    valid_cnn_labels = labels_perm[cfv_const*cfv:cfv_const*(cfv-1)]
#split train files among ISUP categories
with tqdm(total=train_cnn_labels.shape[0]) as pbar:
    for j, row in train_cnn_labels.iterrows():
        try:
            filename = row['image_id'] + '.png'
            suffix = row['gleason_score'].replace('+', '_') if row['gleason_score'] != '5+3' else '3_5'            
            path_to_train_class = os.path.join(train_cnn, suffix, filename)
            path_to_copy_from = os.path.join(new_proc_folder, filename)
            bashCommand = "cp " + path_to_copy_from + ' ' + path_to_train_class
            process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
            output, error = process.communicate()
            if error: print(str(error))
        except Exception as ee:
            print(f'Image number {filename}, error:\n{ee}')
        pbar.update(1)
#split validation files among ISUP categories
with tqdm(total=valid_cnn_labels.shape[0]) as pbar:
    for j, row in valid_cnn_labels.iterrows():
        try:
            filename = row['image_id'] + '.png'
            suffix = row['gleason_score'].replace('+', '_') if row['gleason_score'] != '5+3' else '3_5'     
            path_to_valid_class = os.path.join(valid_cnn, suffix, filename)    
            path_to_copy_from = os.path.join(new_proc_folder, filename)
            bashCommand = "cp " + path_to_copy_from + ' ' + path_to_valid_class    
            process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
            output, error = process.communicate()
            if error: print(str(error))
        except Exception as ee:
            print(f'Image number {filename}, error:\n{ee}')
        pbar.update(1)
import os
module_name = 't2_split_gs_score'

os.system('jupyter nbconvert --to python ' + module_name + '.ipynb')
with open(module_name + '.py', 'r') as f:
    lines = f.readlines()
with open(module_name + '.py', 'w') as f:
    for line in lines:
        if 'nbconvert --to python' in line:
            break
        else:
            f.write(line)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13003663003663005, 'fraction_numerical': 0.01098901098901099, 'mean_word_length': 3.3942528735632185, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '72811', 'license': 'Apache-2.0', 'n_tokens_mistral': 1364, 'n_tokens_neox': 1326, 'n_words': 275, 'path': 'cloud/t2_split_gs_score.ipynb', 'repo_name': 'Belobragin/PCA-Kaggle', 'size': '3822'}