<jupyter_start><jupyter_text>Specify the pipeline parameters<jupyter_code>parameter_fn = 'parameters.json'
db_filename = "path/to/database/cosmos.sqlite"
workflow_name = 'CARE_test'

input_dir = "path/to/the/directory/with/data" # folder with the prepared input data
output_dir = "path/to/the/directory/to/save/results/" + workflow_name

data_dir = 'CARE_data'
npz_dir = 'datagen'
model_dir = 'CARE_models'
accuracy_dir = 'accuracy'
accuracy_fn = 'accuracy.csv'
n_jobs = 30
name_high='high_deconvolved'
name_low='low_raw'
name_train='train'
name_validation='validation'
name_test='test'
axes = 'ZYX'
save_training_history = False

patch_size = [(16, 32, 32),
              (32, 64, 64)]
n_patches_per_image = [20]

train_epochs = 10
train_steps_per_epoch = [10]
train_batch_size = [8]

parameter_fn = os.path.abspath(parameter_fn)
args = vars().copy()
keys = [key for key in args.keys() 
        if not (key.startswith('_') or key in sysparams or key == 'sysparams')]
params = {key: args[key] for key in keys}
os.makedirs(os.path.dirname(parameter_fn), exist_ok=True)
with open(parameter_fn, 'w') as f:
    json.dump(params, f, indent=4)
params<jupyter_output><empty_output><jupyter_text>Running the workflowTo run locally, run `python cosmos_workflow.py -p parameters.json`To generate a bsub command to submit to an lsf cluster, specify the following parameters:<jupyter_code>n_cores = 30
n_gpus = 2
queue = 'dgx'
max_mem_gb = 300
python_path = "'/path/to/your/python/environment/bin/:$PATH'"
parameter_fn = os.path.abspath(parameter_fn)
code_path = os.path.abspath('cosmos_workflow.py')
command = rf'python {code_path} -p {parameter_fn} -g {n_gpus} -c {n_cores}'

command = f'bsub -P CARE -J CARE -q {queue} -n {n_cores} -gpu "num={n_gpus}:mode=exclusive_process"'\
          f' -R "rusage[mem={int(max_mem_gb/n_cores)}G]" "export PATH={python_path}; {command}"' 
print(command)<jupyter_output>bsub -P CARE -J CARE -q dgx -n 30 -gpu "num=2:mode=exclusive_process" -R "rusage[mem=10G]" "export PATH='/path/to/your/python/environment/bin/:$PATH'; python /research/sharedresources/cbi/common/Anna/codes/COSMOS-CARE/cosmos_workflow.py -p /research/sharedresources/cbi/common/Anna/codes/COSMOS-CARE/parameters.json -g 2 -c 30"
{'fraction_non_alphanumeric': 0.1417004048582996, 'fraction_numerical': 0.016194331983805668, 'mean_word_length': 5.409221902017291, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '250771', 'license': 'Apache-2.0', 'n_tokens_mistral': 895, 'n_tokens_neox': 849, 'n_words': 180, 'path': 'set_parameters.ipynb', 'repo_name': 'amedyukhina/COSMOS-CARE', 'size': '2223'}