<jupyter_start><jupyter_text>SNGP new exps  CoNLL-2003<jupyter_code>def generate_train_str_config(dpp_config):
    config_list = []
    config_list.append('ue\=sngp do_ue_estimate\=False')
    
    config_list.append('ue.use_selective\={}'.format(dpp_config['use_selective']))
    config_list.append('data.subsample_perc_val\={}'.format(dpp_config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(dpp_config['subsample_perc']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                new_task += f' script=run_conll2003.py seeds=[23419,705525,4837,10671619,1084218,43]'
                subsample_perc = conf['subsample_perc']
                args = ' '.join(generate_func[i](conf))
                
                with open(f'../../workdir/hp_search/electra_raw_sngp/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'                
                
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                new_task+=f' output_dir=../workdir/run_train_models/electra_raw_sngp/{task_name}/{subsample_perc}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open ('../../scripts/ner_scripts/run_train_models_sngp_ner_new.sh', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

train_configs = {
    'use_selective': ['False'],
    'validation_subsample': [0.1],
    'subsample_perc': [0.1],
}
    
cuda_devices = '[0,1]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config])
import os 

def generate_eval_sngp_str_config(mc_config):
    config_list = []
    
    config_list.append('ue\={}'.format(mc_config['ue']))
    config_list.append('do_ue_estimate\=True')
    
    config_list.append('data.subsample_perc_val\={}'.format(mc_config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(mc_config['subsample_perc']))
    
    config_list.append('training.per_device_eval_batch_size\={}'.format(mc_config['per_device_eval_batch_size']))
    
    config_list.append('training\=electra_base')
    config_list.append('ue.committee_size\={}'.format(mc_config['committee_size']))
    
    return config_list


def generate_bash_sngp(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f' script=run_conll2003.py'
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                
                subsample_perc = conf['subsample_perc']
                base_path=f' output_dir=../workdir/run_tasks_for_model_series/electra_raw_sngp/{task_name}/sngp'
                new_task+=base_path
                
                model_series_dir = f'../workdir/run_train_models/electra_raw_sngp/{task_name}/{subsample_perc}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                calc_output_dir = f'../workdir/run_calc_ues_metrics/electra_raw_sngp/{task_name}/sngp'
                calc_command = f'HYDRA_CONFIG_PATH=../configs/run_calc_ues_metrics.yaml python ./run_calc_ues_metrics_ner.py runs_dir=../workdir/run_tasks_for_model_series/electra_raw_sngp/{task_name}/sngp extract_config=False output_dir={calc_output_dir};'

                full_config += '\n' + new_task if len(full_config) else new_task
                full_config += '\n' + calc_command
            
    with open (f'../../scripts/ner_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

eval_configs = {
    'ue': ['sngp'],
    'use_selective': ['False'],
    'validation_subsample': [0.1],
    'subsample_perc': [0.1],
    'per_device_eval_batch_size': [128],
    'committee_size': [10]
}
    
cuda_devices = '[0,1]'

generate_bash_sngp([eval_configs], cuda_devices, tasks, [generate_eval_sngp_str_config], file_name='run_eval_sngp_ner_new.sh')
full_test = 'bash run_train_models_sngp_ner_new.sh\n'
full_test += 'bash run_eval_sngp_ner_new.sh'

with open (f'../../scripts/ner_scripts/run_electra_sngp_ner_new.sh', 'w') as rsh:
    rsh.write(full_test)<jupyter_output><empty_output><jupyter_text>GLUE<jupyter_code>def generate_train_str_config(dpp_config):
    config_list = []
    config_list.append('ue\=sngp do_ue_estimate\=False')
    
    config_list.append('ue.use_selective\={}'.format(dpp_config['use_selective']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                new_task += f' script=run_glue.py seeds=[23419,705525,4837,10671619,1084218,43]'
                args = ' '.join(generate_func[i](conf))
                
                with open(f'../../workdir/hp_search/electra_raw_sngp/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'                
                
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                new_task+=f' output_dir=../workdir/run_train_models/electra_raw_sngp/{task_name}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open ('../../scripts/miscl_scripts/run_train_models_sngp_new.sh', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

train_configs = {
    'use_selective': ['False'],
}
    
cuda_devices = '[0,1]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config])
import os 

def generate_eval_sngp_str_config(mc_config):
    config_list = []
    
    config_list.append('ue\={}'.format(mc_config['ue']))
    config_list.append('do_ue_estimate\=True')
    
    config_list.append('training.per_device_eval_batch_size\={}'.format(mc_config['per_device_eval_batch_size']))
    
    config_list.append('training\=electra_base')
    config_list.append('ue.committee_size\={}'.format(mc_config['committee_size']))
    
    return config_list


def generate_bash_sngp(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f' script=run_glue.py'
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                
                base_path=f' output_dir=../workdir/run_tasks_for_model_series/electra_raw_sngp/{task_name}/sngp'
                new_task+=base_path
                
                model_series_dir = f'../workdir/run_train_models/electra_raw_sngp/{task_name}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                calc_output_dir = f'../workdir/run_calc_ues_metrics/electra_raw_sngp/{task_name}/sngp'
                calc_command = f'HYDRA_CONFIG_PATH=../configs/run_calc_ues_metrics.yaml python ./run_calc_ues_metrics.py runs_dir=../workdir/run_tasks_for_model_series/electra_raw_sngp/{task_name}/sngp extract_config=False output_dir={calc_output_dir};'

                full_config += '\n' + new_task if len(full_config) else new_task
                full_config += '\n' + calc_command
            
    with open (f'../../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

eval_configs = {
    'ue': ['sngp'],
    'use_selective': ['False'],
    'per_device_eval_batch_size': [128],
    'committee_size': [10]
}
    
cuda_devices = '[0,1]'

generate_bash_sngp([eval_configs], cuda_devices, tasks, [generate_eval_sngp_str_config], file_name='run_eval_sngp_new.sh')
full_test = 'bash run_train_models_sngp_new.sh\n'
full_test += 'bash run_eval_sngp_new.sh'

with open (f'../../scripts/miscl_scripts/run_electra_sngp_new.sh', 'w') as rsh:
    rsh.write(full_test)<jupyter_output><empty_output><jupyter_text>SNGP training HP search<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=sngp do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    j = 0
    n_devices = len(cuda_devices)
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                base_arg = f'CUDA_VISIBLE_DEVICES={cuda_devices[j%n_devices]} HYDRA_CONFIG_PATH=../configs/{task_name}.yaml python ./run_glue_with_hyp_search.py'

                new_task = copy.deepcopy(base_arg)
                args = ' '.join(generate_func[i](conf))
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']

                new_task += f' {args}'
                new_task += f' hydra.run.dir=../workdir/hp_search/electra_raw_sngp/{task_name}'
                new_task += " model.model_name_or_path='google/electra-base-discriminator'"
                if (j+1)%n_devices!=0: 
                    new_task += ' &'
                else:
                    new_task += '\nwait'
                full_config += '\n' + new_task if len(full_config) else new_task
                j+=1
                
    with open (f'../../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

train_configs = {
    'reg_type': ['raw'],
}
    
cuda_devices = [0, 1]

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_hp_search_sngp.sh')
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    j = 0
    n_devices = len(cuda_devices)
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                base_arg = f'CUDA_VISIBLE_DEVICES={cuda_devices[j%n_devices]} HYDRA_CONFIG_PATH=../configs/{task_name}.yaml python ./run_conll2003_with_hyp_search.py'

                new_task = copy.deepcopy(base_arg)
                args = ' '.join(generate_func[i](conf))
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']

                new_task += f' {args}'
                new_task += f' hydra.run.dir=../workdir/hp_search/electra_raw_sngp/{task_name}'
                new_task += " model.model_name_or_path='google/electra-base-discriminator'"
                if (j+1)%n_devices!=0: 
                    new_task += ' &'
                else:
                    new_task += '\nwait'
                full_config += '\n' + new_task if len(full_config) else new_task
                j+=1
                
    with open (f'../../scripts/ner_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

train_configs = {
    'reg_type': ['raw'],
}
    
cuda_devices = [0]

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_hp_search_sngp.sh')<jupyter_output><empty_output><jupyter_text>OOD with noise train<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    #config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('+data.noise_perc\={}'.format(config['noise_perc']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                try:
                    task_configs = f'{task_name}.yaml'

                    new_task = copy.deepcopy(base_arg)
                    new_task += f' cuda_devices={cuda_devices}'
                    script = conf['script']
                    new_task += f' script={script}'
                    args = ' '.join(generate_func[i](conf))

                    reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                    use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'

                    with open(f'../../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                        params = yaml.load(f, Loader=yaml.FullLoader)

                    num_train_epochs = params['num_train_epochs']
                    args += f' training.num_train_epochs\={num_train_epochs}'
                    learning_rate = params['learning_rate']
                    args += f' training.learning_rate\={learning_rate}'
                    per_device_train_batch_size = params['per_device_train_batch_size']
                    args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                    weight_decay = params['weight_decay']
                    args += f' +training.weight_decay\={weight_decay}'

                    margin = params['margin']
                    lamb_intra = params['lamb_intra']
                    lamb = params['lamb']
                    args += f' +ue.margin\={margin}'
                    args += f' +ue.lamb_intra\={lamb_intra}'
                    args += f' ue.lamb\={lamb}'

                    new_task += f" args='{args}'"
                    new_task += f' task_configs={task_configs}'
                    noise_perc = conf['noise_perc']

                    new_task+=f' output_dir=../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/noise_perc_{noise_perc}'

                    full_config += '\n' + new_task if len(full_config) else new_task
                except:
                    pass
    with open (f'../../scripts/ood_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['clinc', 'rostd']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['raw'],#['raw', 'metric', 'reg-curr'], 
    'use_spectralnorm': ['False'],# 'True'],
    'script': ['run_ood.py'],
    'noise_perc': [0.01, 0.02, 0.05, 0.1, 0.2, 0.3],
}
    
cuda_devices = '[0,1,2]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_noise.sh')
def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    
    if config['ue'] == 'mc_mahalanobis':
        config_list.append('ue.use_cache\=False')
    else:
        config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    config_list.append('+data.noise_perc\={}'.format(config['noise_perc']))
    
    if config['ue'] == 'nuq' or config['ue'] == 'l_nuq':
        config_list.append('ue.nuq.n_neighbors\=60')
        config_list.append('ue.nuq.log_pN\=0')
        config_list.append('+ue.nuq.n_points\=50')
        config_list.append('+ue.nuq.n_folds\=20')
        config_list.append('+ue.nuq.n_samples\=5')
    
    config_list.append('training\=electra_base')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                try:
                    task_configs = f'{task_name}.yaml'

                    new_task = copy.deepcopy(base_arg)
                    new_task += f' cuda_devices={cuda_devices}'
                    script = conf['script']
                    new_task += f' script={script}'

                    args = ' '.join(generate_func[i](conf))

                    reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                    use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'

                    with open(f'../../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                        params = yaml.load(f, Loader=yaml.FullLoader)

                    num_train_epochs = params['num_train_epochs']
                    args += f' training.num_train_epochs\={num_train_epochs}'
                    learning_rate = params['learning_rate']
                    args += f' training.learning_rate\={learning_rate}'
                    per_device_train_batch_size = params['per_device_train_batch_size']
                    args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                    weight_decay = params['weight_decay']
                    args += f' +training.weight_decay\={weight_decay}'

                    margin = params['margin']
                    lamb_intra = params['lamb_intra']
                    lamb = params['lamb']
                    args += f' ue.margin\={margin}'
                    args += f' ue.lamb_intra\={lamb_intra}'
                    args += f' ue.lamb\={lamb}'

                    args += f' training.per_device_eval_batch_size\=64'

                    new_task += f" args='{args}'"
                    new_task += f' config_path=../configs/{task_configs}'

                    reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                    use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                    noise_perc = conf['noise_perc']

                    base_path=f' output_dir=../workdir/run_tasks_for_model_series/electra_{reg}_{use_spectralnorm}/{task_name}/noise_perc_{noise_perc}'

                    ue_type = conf['ue']
                    base_path+= f'/{ue_type}'
                    new_task+=base_path

                    model_series_dir=f'../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/noise_perc_{noise_perc}/models/{task_name}'
                    new_task+=f' model_series_dir={model_series_dir}'

                    full_config += '\n' + new_task if len(full_config) else new_task
                except:
                    pass
    with open (f'../../scripts/ood_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['clinc', 'rostd']

ue_configs = {
    'ue': ['nuq', 'mahalanobis', 'l_nuq', 'l_mahalanobis'],
    'use_spectralnorm': ['False'],
    'reg_type': ['raw'],
    'script': ['run_ood.py'],
    'noise_perc': [0.01, 0.02, 0.05, 0.1, 0.2, 0.3],
}
    
cuda_devices = '[0,1,2]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_noise.sh')
full_test = 'bash run_train_electra_models_noise.sh\n'
full_test += 'bash run_eval_electra_noise.sh'

with open (f'../../scripts/ood_scripts/run_exps_noise.sh', 'w') as rsh:
    rsh.write(full_test)<jupyter_output><empty_output><jupyter_text>Run exps for model with SN for residuals<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('spectralnorm_layer\={}'.format(config['spectralnorm_layer']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}_ood.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                args = ' '.join(generate_func[i](conf))

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                spectralnorm_layer = conf['spectralnorm_layer']
                if 'resid' in spectralnorm_layer and use_spectralnorm == 'sn':
                    use_spectralnorm+='_all'
                    
                path = f'../../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml'
                with open(path, 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'

                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'

                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'

                new_task+=f' output_dir=../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/'

                full_config += '\n' + new_task if len(full_config) else new_task

    with open (f'../../scripts/ood_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['20newsgroups', 'sst2']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['raw'], 
    'use_spectralnorm': ['True'],
    'script': ['run_ood.py'],
    'spectralnorm_layer': ['resid+last']
}
    
cuda_devices = '[0,1,2]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_new_benchmark_sn_all.sh')<jupyter_output><empty_output><jupyter_text>Eval determenistic methods<jupyter_code>def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('spectralnorm_layer\={}'.format(config['spectralnorm_layer']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    
    if config['ue'] == 'mc_mahalanobis':
        config_list.append('ue.use_cache\=False')
    else:
        config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    config_list.append('data.ood_data\={}'.format(config['ood_data']))
    
    if config['ue'] == 'nuq' or config['ue'] == 'l_nuq':
        config_list.append('ue.nuq.n_neighbors\=60')
        config_list.append('ue.nuq.log_pN\=0')
        config_list.append('+ue.nuq.n_points\=50')
        config_list.append('+ue.nuq.n_folds\=20')
        config_list.append('+ue.nuq.n_samples\=5')
    
    config_list.append('training\=electra_base')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}_ood.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'

                args = ' '.join(generate_func[i](conf))

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                spectralnorm_layer = conf['spectralnorm_layer']
                if 'resid' in spectralnorm_layer and use_spectralnorm == 'sn':
                    use_spectralnorm+='_all'

                path = f'../../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml'
                with open(path, 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'

                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' ue.margin\={margin}'
                args += f' ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'

                args += f' training.per_device_eval_batch_size\=256'

                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                ood_data = conf['ood_data']
                if task_name == ood_data:
                    continue
                elif ood_data in task_name:
                    continue

                base_path=f' output_dir=../workdir/run_tasks_for_model_series/electra_{reg}_{use_spectralnorm}/{task_name}/{ood_data}'

                ue_type = conf['ue']
                base_path+= f'/{ue_type}'
                new_task+=base_path

                model_series_dir=f'../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/models/{task_name}_ood'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
 
    with open (f'../../scripts/ood_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['sst2', '20newsgroups']

ue_configs = {
    'ue': ['l_nuq', 'l_mahalanobis', 'decomposing_md', 'nuq', 'mahalanobis', 'ddu'],
    'use_spectralnorm': ['True'],
    'reg_type': ['raw'],
    'script': ['run_ood.py'],
    'ood_data': ['imdb', 'trec', 'wmt16', 'amazon', 'mnli', 'newsgroup', 'rte', 'sst2'],
    'spectralnorm_layer': ['resid+last']
}
    
cuda_devices = '[0,1,2]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_new_benchmark_sn_all.sh')
full_test = 'bash run_train_electra_models_new_benchmark_sn_all.sh\n'
full_test += 'bash run_eval_electra_new_benchmark_sn_all.sh'

with open (f'../../scripts/ood_scripts/run_exps_new_benchmark_sn_all.sh', 'w') as rsh:
    rsh.write(full_test)<jupyter_output><empty_output><jupyter_text>HP Search for SN for Resid<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('spectralnorm_layer\={}'.format(config['spectralnorm_layer']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    j = 0
    n_devices = len(cuda_devices)
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                base_arg = f'CUDA_VISIBLE_DEVICES={cuda_devices[j%n_devices]} HYDRA_CONFIG_PATH=../configs/{task_name}_ood.yaml python ./run_ood_with_hyp_search.py'

                new_task = copy.deepcopy(base_arg)
                args = ' '.join(generate_func[i](conf))
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                if reg == 'metric' and task_name=='clinc':
                    continue
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                new_task += f' {args}'
                new_task += f' hydra.run.dir=../workdir/hp_search/electra_{reg}_{use_spectralnorm}_all/{task_name}'
                new_task += " model.model_name_or_path='google/electra-base-discriminator'"
                if (j+1)%n_devices!=0: 
                    new_task += ' &'
                else:
                    new_task += '\nwait'
                full_config += '\n' + new_task if len(full_config) else new_task
                j+=1
                
    with open (f'../../scripts/ood_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['sst2', '20newsgroups']

train_configs = {
    'reg_type': ['reg-curr', 'metric', 'raw'],
    'use_spectralnorm': ['True'],
    'spectralnorm_layer': ['resid+last']
}
    
cuda_devices = [0, 1, 2]

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_hp_search_all_sn.sh')<jupyter_output><empty_output><jupyter_text>OOD exps with new benchmark<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                try:
                    task_configs = f'{task_name}_ood.yaml'

                    new_task = copy.deepcopy(base_arg)
                    new_task += f' cuda_devices={cuda_devices}'
                    script = conf['script']
                    new_task += f' script={script}'
                    args = ' '.join(generate_func[i](conf))

                    reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                    use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                    subsample_perc = conf['subsample_perc']
                    
                    if task_name == '20newsgroups':
                        path = f'../../../workdir/optimal_hyperparams/{task_name}_electra_{reg}_{use_spectralnorm}.yaml'
                    else:
                        path = f'../../../hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml'
                    with open(path, 'r') as f:
                        params = yaml.load(f, Loader=yaml.FullLoader)

                    num_train_epochs = params['num_train_epochs']
                    args += f' training.num_train_epochs\={num_train_epochs}'
                    learning_rate = params['learning_rate']
                    args += f' training.learning_rate\={learning_rate}'
                    per_device_train_batch_size = params['per_device_train_batch_size']
                    args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                    weight_decay = params['weight_decay']
                    args += f' +training.weight_decay\={weight_decay}'

                    margin = params['margin']
                    lamb_intra = params['lamb_intra']
                    lamb = params['lamb']
                    args += f' +ue.margin\={margin}'
                    args += f' +ue.lamb_intra\={lamb_intra}'
                    args += f' ue.lamb\={lamb}'

                    new_task += f" args='{args}'"
                    new_task += f' task_configs={task_configs}'

                    new_task+=f' output_dir=../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{subsample_perc}'

                    full_config += '\n' + new_task if len(full_config) else new_task
                except:
                    pass
    with open (f'../../scripts/ood_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['20newsgroups', 'sst2']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['raw'], 
    'use_spectralnorm': ['False', 'True'],
    'script': ['run_ood.py'],
    'subsample_perc': [0.005, 0.01, 0.015, 0.02, 0.03, 0.04, 0.05, 0.1, 0.15, 0.2, 0.3],
}
    
cuda_devices = '[0,1,2]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_new_benchmark_subsamples.sh')
def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    
    if config['ue'] == 'mc_mahalanobis':
        config_list.append('ue.use_cache\=False')
    else:
        config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    config_list.append('data.ood_data\={}'.format(config['ood_data']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    
    if config['ue'] == 'nuq' or config['ue'] == 'l_nuq':
        config_list.append('ue.nuq.n_neighbors\=60')
        config_list.append('ue.nuq.log_pN\=0')
        config_list.append('+ue.nuq.n_points\=50')
        config_list.append('+ue.nuq.n_folds\=20')
        config_list.append('+ue.nuq.n_samples\=5')
    
    config_list.append('training\=electra_base')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                try:
                    task_configs = f'{task_name}_ood.yaml'

                    new_task = copy.deepcopy(base_arg)
                    new_task += f' cuda_devices={cuda_devices}'
                    script = conf['script']
                    new_task += f' script={script}'

                    args = ' '.join(generate_func[i](conf))

                    reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                    use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                    subsample_perc = conf['subsample_perc']

                    if task_name == '20newsgroups':
                        path = f'../../../workdir/optimal_hyperparams/{task_name}_electra_{reg}_{use_spectralnorm}.yaml'
                    else:
                        path = f'../../../hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml'
                    with open(path, 'r') as f:
                        params = yaml.load(f, Loader=yaml.FullLoader)

                    num_train_epochs = params['num_train_epochs']
                    args += f' training.num_train_epochs\={num_train_epochs}'
                    learning_rate = params['learning_rate']
                    args += f' training.learning_rate\={learning_rate}'
                    per_device_train_batch_size = params['per_device_train_batch_size']
                    args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                    weight_decay = params['weight_decay']
                    args += f' +training.weight_decay\={weight_decay}'

                    margin = params['margin']
                    lamb_intra = params['lamb_intra']
                    lamb = params['lamb']
                    args += f' ue.margin\={margin}'
                    args += f' ue.lamb_intra\={lamb_intra}'
                    args += f' ue.lamb\={lamb}'

                    args += f' training.per_device_eval_batch_size\=256'

                    new_task += f" args='{args}'"
                    new_task += f' config_path=../configs/{task_configs}'

                    reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                    use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                    ood_data = conf['ood_data']
                    if task_name == ood_data:
                        continue
                    elif ood_data in task_name:
                        continue

                    base_path=f' output_dir=../workdir/run_tasks_for_model_series/electra_{reg}_{use_spectralnorm}/{task_name}/{subsample_perc}/{ood_data}'

                    ue_type = conf['ue']
                    base_path+= f'/{ue_type}'
                    new_task+=base_path

                    model_series_dir=f'../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{subsample_perc}/models/{task_name}_ood'
                    new_task+=f' model_series_dir={model_series_dir}'

                    full_config += '\n' + new_task if len(full_config) else new_task
                except:
                    pass
    with open (f'../../scripts/ood_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['sst2', '20newsgroups']

ue_configs = {
    'ue': ['nuq', 'mahalanobis'],#['l_nuq', 'l_mahalanobis', 'decomposing_md', 'nuq', 'mahalanobis', 'ddu', 'sngp'],
    'use_spectralnorm': ['False'],# 'True'],
    'reg_type': ['raw'],
    'script': ['run_ood.py'],
    'ood_data': ['imdb', 'trec', 'wmt16', 'amazon', 'mnli', 'newsgroup', 'rte', 'sst2'],
    'subsample_perc': [0.01, 0.02, 0.05, 0.1, 0.2, 0.3],#[0.005, 0.01, 0.015, 0.02, 0.03, 0.04, 0.05, 0.1, 0.15, 0.2, 0.3],
}
    
cuda_devices = '[0,1,2]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_new_benchmark_subsamples.sh')
full_test = 'bash run_train_electra_models_new_benchmark_subsamples.sh\n'
full_test += 'bash run_eval_electra_new_benchmark_subsamples.sh'

with open (f'../../scripts/ood_scripts/run_exps_new_benchmark_subsamples.sh', 'w') as rsh:
    rsh.write(full_test)<jupyter_output><empty_output><jupyter_text>OOD exps with determenistic methods<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    #config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                try:
                    task_configs = f'{task_name}.yaml'

                    new_task = copy.deepcopy(base_arg)
                    new_task += f' cuda_devices={cuda_devices}'
                    script = conf['script']
                    new_task += f' script={script}'
                    args = ' '.join(generate_func[i](conf))

                    reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                    use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'

                    with open(f'../../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                        params = yaml.load(f, Loader=yaml.FullLoader)

                    num_train_epochs = params['num_train_epochs']
                    args += f' training.num_train_epochs\={num_train_epochs}'
                    learning_rate = params['learning_rate']
                    args += f' training.learning_rate\={learning_rate}'
                    per_device_train_batch_size = params['per_device_train_batch_size']
                    args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                    weight_decay = params['weight_decay']
                    args += f' +training.weight_decay\={weight_decay}'

                    margin = params['margin']
                    lamb_intra = params['lamb_intra']
                    lamb = params['lamb']
                    args += f' +ue.margin\={margin}'
                    args += f' +ue.lamb_intra\={lamb_intra}'
                    args += f' ue.lamb\={lamb}'

                    new_task += f" args='{args}'"
                    new_task += f' task_configs={task_configs}'
                    subsample_perc = conf['subsample_perc']

                    new_task+=f' output_dir=../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{subsample_perc}'

                    full_config += '\n' + new_task if len(full_config) else new_task
                except:
                    pass
    with open (f'../../scripts/ood_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['clinc', 'rostd']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['raw'],#['raw', 'metric', 'reg-curr'], 
    'use_spectralnorm': ['False'],# 'True'],
    'script': ['run_ood.py'],
    'subsample_perc': [0.01, 0.02, 0.05, 0.1, 0.15, 0.2, 0.3],
}
    
cuda_devices = '[0,1,2]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_subsamples.sh')
def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    
    if config['ue'] == 'mc_mahalanobis':
        config_list.append('ue.use_cache\=False')
    else:
        config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    #config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    
    if config['ue'] == 'nuq' or config['ue'] == 'l_nuq':
        config_list.append('ue.nuq.n_neighbors\=60')
        config_list.append('ue.nuq.log_pN\=0')
        config_list.append('+ue.nuq.n_points\=50')
        config_list.append('+ue.nuq.n_folds\=20')
        config_list.append('+ue.nuq.n_samples\=5')
    
    config_list.append('training\=electra_base')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                try:
                    task_configs = f'{task_name}.yaml'

                    new_task = copy.deepcopy(base_arg)
                    new_task += f' cuda_devices={cuda_devices}'
                    script = conf['script']
                    new_task += f' script={script}'

                    args = ' '.join(generate_func[i](conf))

                    reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                    use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'

                    with open(f'../../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                        params = yaml.load(f, Loader=yaml.FullLoader)

                    num_train_epochs = params['num_train_epochs']
                    args += f' training.num_train_epochs\={num_train_epochs}'
                    learning_rate = params['learning_rate']
                    args += f' training.learning_rate\={learning_rate}'
                    per_device_train_batch_size = params['per_device_train_batch_size']
                    args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                    weight_decay = params['weight_decay']
                    args += f' +training.weight_decay\={weight_decay}'

                    margin = params['margin']
                    lamb_intra = params['lamb_intra']
                    lamb = params['lamb']
                    args += f' ue.margin\={margin}'
                    args += f' ue.lamb_intra\={lamb_intra}'
                    args += f' ue.lamb\={lamb}'

                    args += f' training.per_device_eval_batch_size\=64'

                    new_task += f" args='{args}'"
                    new_task += f' config_path=../configs/{task_configs}'

                    reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                    use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                    subsample_perc = conf['subsample_perc']

                    base_path=f' output_dir=../workdir/run_glue_for_model_series/electra_{reg}_{use_spectralnorm}/{task_name}/{subsample_perc}'

                    ue_type = conf['ue']
                    base_path+= f'/{ue_type}'
                    new_task+=base_path

                    model_series_dir=f'../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{subsample_perc}/models/{task_name}'
                    new_task+=f' model_series_dir={model_series_dir}'

                    full_config += '\n' + new_task if len(full_config) else new_task
                except:
                    pass
    with open (f'../../scripts/ood_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['clinc', 'rostd']

ue_configs = {
    #'ue': ['decomposing_md', 'nuq', 'mahalanobis', 'ddu', 'sngp'],
    'ue': ['l_nuq', 'ddu', 'nuq', 'mahalanobis', 'l_mahalanobis'],
    'use_spectralnorm': ['False'],# 'True'],
    'reg_type': ['raw'],# 'metric', 'reg-curr'],
    'script': ['run_ood.py'],
    #'validation_subsample': [0.0],
    'subsample_perc': [0.01, 0.02, 0.05, 0.1, 0.15, 0.2, 0.3],
}
    
cuda_devices = '[0,1,2]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_subsamples.sh')
full_test = 'bash run_train_electra_models.sh\n'
full_test += 'bash run_eval_electra.sh'

with open (f'../../scripts/ood_scripts/run_exps.sh', 'w') as rsh:
    rsh.write(full_test)
full_test = 'bash run_train_electra_models_subsamples.sh\n'
full_test += 'bash run_eval_electra_subsamples.sh'

with open (f'../../scripts/ood_scripts/run_exps_subsamples.sh', 'w') as rsh:
    rsh.write(full_test)<jupyter_output><empty_output><jupyter_text>OOD datasets HP Search<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    j = 0
    n_devices = len(cuda_devices)
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                base_arg = f'CUDA_VISIBLE_DEVICES={cuda_devices[j%n_devices]} HYDRA_CONFIG_PATH=../configs/{task_name}.yaml python ./run_ood_with_hyp_search.py'

                new_task = copy.deepcopy(base_arg)
                args = ' '.join(generate_func[i](conf))
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                if reg == 'metric' and task_name=='clinc':
                    continue
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                new_task += f' {args}'
                new_task += f' hydra.run.dir=../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}'
                new_task += " model.model_name_or_path='google/electra-base-discriminator'"
                if (j+1)%n_devices!=0: 
                    new_task += ' &'
                else:
                    new_task += '\nwait'
                full_config += '\n' + new_task if len(full_config) else new_task
                j+=1
                
    with open (f'../../scripts/ood_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['clinc', 'rostd']#, 'snips']

train_configs = {
    'reg_type': ['reg-curr', 'metric'],#['raw'],
    'use_spectralnorm': ['False', 'True'],
}
    
cuda_devices = [0, 1, 2]

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_hp_search_regs.sh')<jupyter_output><empty_output><jupyter_text>Decomposing for GLUE misclassification<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    #config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
           
                new_task+=f' output_dir=../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2', 'sst5']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
}
    
cuda_devices = '[1,4,7]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models.sh')
tasks = ['sst5']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['raw'],#['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False'],# 'True'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
}
    
cuda_devices = '[0,1]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_sst5.sh')
def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('++ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    
    if (config['ue'] == 'mc_mahalanobis') or (config['ue'] == 'mc'):
        config_list.append('ue.use_cache\=False')
    else:
        config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    #config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    if config['ue'] == 'nuq':
        config_list.append('ue.nuq.n_neighbors\=60')
        config_list.append('ue.nuq.log_pN\=0')
        config_list.append('+ue.nuq.n_points\=50')
        config_list.append('+ue.nuq.n_folds\=20')
        config_list.append('+ue.nuq.n_samples\=5')
    
    config_list.append('training\=electra_base')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' ++ue.margin\={margin}'
                args += f' ++ue.lamb_intra\={lamb_intra}'
                args += f' ++ue.lamb\={lamb}'
                
                args += f' training.per_device_eval_batch_size\=64'
        
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                base_path=f' output_dir=../workdir/run_glue_for_model_series/electra_{reg}_{use_spectralnorm}/{task_name}/'
                
                ue_type = conf['ue']
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                model_series_dir=f'../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2', 'sst5']

ue_configs = {
    'ue': ['decomposing_md', 'nuq', 'mahalanobis'],
    'use_spectralnorm': ['False', 'True'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
    #'subsample_perc': [0.1]
}
    
cuda_devices = '[1,4,7]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_decomposing.sh')
tasks = ['sst5']

ue_configs = {
    'ue': ['mahalanobis', 'mc'],
    'use_spectralnorm': ['False'],# 'True'],
    'reg_type': ['raw'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
}
    
cuda_devices = '[0,1]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_sst5.sh')<jupyter_output><empty_output><jupyter_text>Twitter HSO NUQ and Mahalanobis<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                subsample_perc = conf['subsample_perc']
                new_task+=f' output_dir=../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['twitter_hso']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
    'script': ['run_20newsgroups.py'],
    'validation_subsample': [0.0],
    'subsample_perc': [0.0],
}
    
cuda_devices = '[1,2]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_twitter_hso.sh')
def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    
    if config['ue'] == 'mc_mahalanobis':
        config_list.append('ue.use_cache\=False')
    else:
        config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    if config['ue'] == 'nuq':
        config_list.append('ue.nuq.n_neighbors\=60')
        config_list.append('ue.nuq.log_pN\=0')
        config_list.append('+ue.nuq.n_points\=50')
        config_list.append('+ue.nuq.n_folds\=20')
        config_list.append('+ue.nuq.n_samples\=5')
    
    config_list.append('training\=electra_base')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_glue_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' ue.margin\={margin}'
                args += f' ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
                
                args += f' training.per_device_eval_batch_size\=64'
        
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                validation_subsample = conf['validation_subsample']
                subsample_perc = conf['subsample_perc']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                subsample_perc = conf['subsample_perc']
                base_path=f' output_dir=../workdir/run_glue_for_model_series/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                #ue_type = conf['ue']
                ue_type = conf['ue'] if conf['ue'] != 'nuq' else 'nuq_best1'
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                model_series_dir=f'../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['twitter_hso']

ue_configs = {
    'ue': ['ddu', 'nuq', 'mahalanobis'],
    'use_spectralnorm': ['False', 'True'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'script': ['run_20newsgroups.py'],
    'validation_subsample': [0.0],
    'subsample_perc': [0.0]
}
    
cuda_devices = '[1,2,7]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_models_twitter_hso_det_best.sh')<jupyter_output><empty_output><jupyter_text>Twitter HSO HP search<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    j = 0
    n_devices = len(cuda_devices)
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                base_arg = f'CUDA_VISIBLE_DEVICES={cuda_devices[j%n_devices]} HYDRA_CONFIG_PATH=../configs/{task_name}.yaml python ./run_20newsgroups_with_hyp_search.py'

                new_task = copy.deepcopy(base_arg)
                args = ' '.join(generate_func[i](conf))
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                new_task += f' {args}'
                new_task += f' hydra.run.dir=../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}'
                new_task += " model.model_name_or_path='google/electra-base-discriminator'"
                if (j+1)%n_devices!=0: 
                    new_task += ' &'
                else:
                    new_task += '\nwait'
                full_config += '\n' + new_task if len(full_config) else new_task
                j+=1
                
    with open (f'../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['twitter_hso']

train_configs = {
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
}
    
cuda_devices = [0, 1, 2, 3]

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'electra_hp_search_twitter_hso.sh')<jupyter_output><empty_output><jupyter_text>MNLI NUQ, Mahalanobis<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                subsample_perc = conf['subsample_perc']
                new_task+=f' output_dir=../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mnli']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
    'subsample_perc': [0.1],
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_mnli.sh')
def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    
    if config['ue'] == 'mc_mahalanobis':
        config_list.append('ue.use_cache\=False')
    else:
        config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    if config['ue'] == 'nuq':
        config_list.append('ue.nuq.n_neighbors\=60')
        config_list.append('ue.nuq.log_pN\=0')
        config_list.append('+ue.nuq.n_points\=50')
        config_list.append('+ue.nuq.n_folds\=20')
        config_list.append('+ue.nuq.n_samples\=5')
    
    config_list.append('training\=electra_base')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_glue_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' ue.margin\={margin}'
                args += f' ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                validation_subsample = conf['validation_subsample']
                subsample_perc = conf['subsample_perc']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                subsample_perc = conf['subsample_perc']
                base_path=f' output_dir=../workdir/run_glue_for_model_series/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                #ue_type = conf['ue']
                ue_type = conf['ue'] if conf['ue'] != 'nuq' else 'nuq_best1'
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                model_series_dir=f'../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mnli']

ue_configs = {
    'ue': ['ddu', 'nuq', 'mahalanobis'],
    'use_spectralnorm': ['False', 'True'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
    'subsample_perc': [0.1]
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_models_mnli_det_best.sh')<jupyter_output><empty_output><jupyter_text>MNLI HP SEARCH<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    j = 0
    n_devices = len(cuda_devices)
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                base_arg = f'CUDA_VISIBLE_DEVICES={cuda_devices[j%n_devices]} HYDRA_CONFIG_PATH=../configs/{task_name}.yaml python ./run_glue_with_hyp_search.py'

                new_task = copy.deepcopy(base_arg)
                args = ' '.join(generate_func[i](conf))
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                new_task += f' {args}'
                new_task += f' hydra.run.dir=../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}'
                new_task += " model.model_name_or_path='google/electra-base-discriminator'"
                if (j+1)%n_devices!=0: 
                    new_task += ' &'
                else:
                    new_task += '\nwait'
                full_config += '\n' + new_task if len(full_config) else new_task
                j+=1
                
    with open (f'../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mnli']

train_configs = {
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
}
    
cuda_devices = [0, 1, 2, 3, 4, 5]

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'electra_hp_search_mnli.sh')<jupyter_output><empty_output><jupyter_text>Train SST-2 with different size of dataset<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                subsample_perc = conf['subsample_perc']
                new_task+=f' output_dir=../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['sst2']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['raw'],
    'use_spectralnorm': ['False', 'True'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
    'subsample_perc': [0.005, 0.01, 0.015, 0.02, 0.025, 0.03, 0.04, 0.05],
}
    
cuda_devices = '[0,1,2]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_sst2_dif_size.sh')
tasks = ['mrpc']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['raw'],
    'use_spectralnorm': ['False', 'True'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
    'subsample_perc': [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.15, 0.2, 0.25, 0.3],
}
    
cuda_devices = '[0,1,2]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_mrpc_dif_size.sh')
def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    
    if config['ue'] == 'mc_mahalanobis':
        config_list.append('ue.use_cache\=False')
    else:
        config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    if config['ue'] == 'nuq' or config['ue'] == 'l_nuq':
        config_list.append('ue.nuq.n_neighbors\=60')
        config_list.append('ue.nuq.log_pN\=-40')
        config_list.append('+ue.nuq.n_points\=50')
        config_list.append('+ue.nuq.n_folds\=20')
        config_list.append('+ue.nuq.n_samples\=5')
    
    config_list.append('training\=electra_base')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' ue.margin\={margin}'
                args += f' ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                validation_subsample = conf['validation_subsample']
                subsample_perc = conf['subsample_perc']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                subsample_perc = conf['subsample_perc']
                base_path=f' output_dir=../workdir/run_glue_for_model_series/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}'
                
                #ue_type = conf['ue']
                ue_type = conf['ue'] if conf['ue'] != 'nuq' else 'nuq_best1'
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                model_series_dir=f'../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['sst2']

ue_configs = {
    'ue': ['l_mahalanobis', 'nuq', 'mahalanobis', 'ddu', 'l_nuq'],
    'use_spectralnorm': ['False', 'True'],
    'reg_type': ['raw'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
    'subsample_perc': [0.005, 0.01, 0.015, 0.02, 0.025, 0.03, 0.04, 0.05],
}
    
cuda_devices = '[0,1,2]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_models_sst2_det_best_dif_size.sh')
tasks = ['mrpc']

ue_configs = {
    'ue': ['nuq', 'mahalanobis', 'ddu', 'l_mahalanobis', 'l_nuq'],
    'use_spectralnorm': ['False', 'True'],
    'reg_type': ['raw'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
    'subsample_perc': [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.15, 0.2, 0.25, 0.3],
}
    
cuda_devices = '[0,1,2]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_models_mrpc_det_best_dif_size.sh')
full_test = 'bash run_train_electra_models_sst2_dif_size.sh\n'
full_test += 'bash run_train_electra_models_mrpc_dif_size.sh\n'
full_test += 'bash run_eval_electra_models_sst2_det_best_dif_size.sh\n'
full_test += 'bash run_eval_electra_models_mrpc_det_best_dif_size.sh'

with open (f'../../scripts/miscl_scripts/run_exps_diff_size.sh', 'w') as rsh:
    rsh.write(full_test)<jupyter_output><empty_output><jupyter_text>NUQ, Mahalanobis for SST-2<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                subsample_perc = conf['subsample_perc']
                new_task+=f' output_dir=../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['sst2']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
    'subsample_perc': [0.1],
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_sst2.sh')
def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    
    if config['ue'] == 'mc_mahalanobis':
        config_list.append('ue.use_cache\=False')
    else:
        config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    if config['ue'] == 'nuq':
        config_list.append('ue.nuq.n_neighbors\=60')
        config_list.append('ue.nuq.log_pN\=-40')
        config_list.append('+ue.nuq.n_points\=50')
        config_list.append('+ue.nuq.n_folds\=20')
        config_list.append('+ue.nuq.n_samples\=5')
    
    config_list.append('training\=electra_base')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_glue_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' ue.margin\={margin}'
                args += f' ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                validation_subsample = conf['validation_subsample']
                subsample_perc = conf['subsample_perc']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                subsample_perc = conf['subsample_perc']
                base_path=f' output_dir=../workdir/run_glue_for_model_series/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                #ue_type = conf['ue']
                ue_type = conf['ue'] if conf['ue'] != 'nuq' else 'nuq_best1'
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                model_series_dir=f'../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['sst2']

ue_configs = {
    'ue': ['ddu', 'nuq', 'mahalanobis'],
    'use_spectralnorm': ['False', 'True'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
    'subsample_perc': [0.1]
}
    
cuda_devices = '[0,1]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_models_sst2_det_best.sh')<jupyter_output><empty_output><jupyter_text>With Noise<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    config_list.append('+data.noise_perc\={}'.format(config['noise_perc']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                subsample_perc = conf['subsample_perc']
                noise_perc = conf['noise_perc']
                new_task+=f' output_dir=../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}/noise_{noise_perc}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['sst2']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['raw'],
    'use_spectralnorm': ['False', 'True'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
    'noise_perc': [0.01, 0.05, 0.1, 0.15, 0.2, 0.25],
    'subsample_perc': [0.1]
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_sst2_noise.sh')
def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    
    if config['ue'] == 'mc_mahalanobis':
        config_list.append('ue.use_cache\=False')
    else:
        config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    config_list.append('+data.noise_perc\={}'.format(config['noise_perc']))
    
    config_list.append('training\=electra_base')
    
    if config['ue'] == 'nuq':
        config_list.append('ue.nuq.n_neighbors\=60')
        config_list.append('+ue.nuq.n_points\=50')
        config_list.append('+ue.nuq.n_folds\=20')
        config_list.append('+ue.nuq.n_samples\=5')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_glue_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' ue.margin\={margin}'
                args += f' ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                validation_subsample = conf['validation_subsample']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                subsample_perc = conf['subsample_perc']
                noise_perc = conf['noise_perc']
                base_path=f' output_dir=../workdir/run_glue_for_model_series/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}/noise_{noise_perc}'
                
                #ue_type = conf['ue']
                ue_type = conf['ue'] if conf['ue'] != 'nuq' else 'nuq_best1'
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                
                model_series_dir=f'../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}/noise_{noise_perc}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['sst2']

ue_configs = {
    'ue': ['nuq', 'mahalanobis'],
    'use_spectralnorm': ['False', 'True'],
    'reg_type': ['raw'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
    'noise_perc': [0.01, 0.05, 0.1, 0.15, 0.2, 0.25],
    'subsample_perc': [0.1]
}
    
cuda_devices = '[0,1,2,3,4]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_models_sst2_det_noise.sh')<jupyter_output><empty_output><jupyter_text>Electra train SST5<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                subsample_perc = conf['subsample_perc']
                new_task+=f' output_dir=../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['sst5']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
    'script': ['run_20newsgroups.py'],
    'validation_subsample': [0.0],
    'subsample_perc': [0.0],
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_sst5.sh')<jupyter_output><empty_output><jupyter_text>Nuq, Mahalanobis<jupyter_code>def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    
    if config['ue'] == 'mc_mahalanobis':
        config_list.append('ue.use_cache\=False')
    else:
        config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    if config['ue'] == 'nuq':
        config_list.append('ue.nuq.n_neighbors\=60')
        config_list.append('+ue.nuq.n_points\=50')
        config_list.append('+ue.nuq.n_folds\=20')
        config_list.append('+ue.nuq.n_samples\=5')

    config_list.append('training\=electra_base')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_glue_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' ue.margin\={margin}'
                args += f' ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                validation_subsample = conf['validation_subsample']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                subsample_perc = conf['subsample_perc']
                base_path=f' output_dir=../workdir/run_glue_for_model_series/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}'
                
                ue_type = conf['ue'] if conf['ue'] != 'nuq' else 'nuq_best1'
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                model_series_dir=f'../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['sst5']

ue_configs = {
    'ue': ['nuq', 'mahalanobis'],
    'use_spectralnorm': ['False', 'True'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'script': ['run_20newsgroups.py'],
    'validation_subsample': [0.0],
    'subsample_perc': [0.0]
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_models_sst5_det_best.sh')<jupyter_output><empty_output><jupyter_text>Electra HP search for SST5<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    j = 0
    n_devices = len(cuda_devices)
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                base_arg = f'CUDA_VISIBLE_DEVICES={cuda_devices[j%n_devices]} HYDRA_CONFIG_PATH=../configs/{task_name}.yaml python ./run_20newsgroups_with_hyp_search.py'

                new_task = copy.deepcopy(base_arg)
                args = ' '.join(generate_func[i](conf))
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                new_task += f' {args}'
                new_task += f' hydra.run.dir=../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}'
                new_task += " model.model_name_or_path='google/electra-base-discriminator'"
                if (j+1)%n_devices!=0: 
                    new_task += ' &'
                else:
                    new_task += '\nwait'
                full_config += '\n' + new_task if len(full_config) else new_task
                j+=1
                
    with open (f'../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['sst5']

train_configs = {
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
}
    
cuda_devices = [0, 1, 2, 3]

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'electra_hp_search_sst5.sh')<jupyter_output><empty_output><jupyter_text>Electra train QQP and Amazon with noise<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    config_list.append('+data.noise_perc\={}'.format(config['noise_perc']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                subsample_perc = conf['subsample_perc']
                noise_perc = conf['noise_perc']
                new_task+=f' output_dir=../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}/noise_{noise_perc}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['qqp']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
    'noise_perc': [0.01, 0.05, 0.1, 0.15, 0.2, 0.25],
    'subsample_perc': [0.1]
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_qqp_noise.sh')<jupyter_output><empty_output><jupyter_text>NUQ, Mahalanobis with noise<jupyter_code>def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    
    if config['ue'] == 'mc_mahalanobis':
        config_list.append('ue.use_cache\=False')
    else:
        config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    config_list.append('+data.noise_perc\={}'.format(config['noise_perc']))
    
    config_list.append('training\=electra_base')
    
    if config['ue'] == 'nuq':
        config_list.append('ue.nuq.n_neighbors\=40')
        config_list.append('+ue.nuq.n_points\=50')
        config_list.append('+ue.nuq.n_folds\=10')
        config_list.append('+ue.nuq.n_samples\=3')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_glue_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' ue.margin\={margin}'
                args += f' ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                validation_subsample = conf['validation_subsample']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                subsample_perc = conf['subsample_perc']
                noise_perc = conf['noise_perc']
                base_path=f' output_dir=../workdir/run_glue_for_model_series/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}/noise_{noise_perc}'
                
                #ue_type = conf['ue']
                ue_type = conf['ue'] if conf['ue'] != 'nuq' else 'nuq_best1'
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                
                model_series_dir=f'../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}/noise_{noise_perc}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['qqp']

ue_configs = {
    'ue': ['nuq', 'mahalanobis'],
    'use_spectralnorm': ['False', 'True'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
    'noise_perc': [0.01, 0.05, 0.1, 0.15, 0.2, 0.25],
    'subsample_perc': [0.1]
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_models_qqp_det_noise.sh')<jupyter_output><empty_output><jupyter_text>Electra train QQP and Amazon<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                subsample_perc = conf['subsample_perc']
                new_task+=f' output_dir=../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}_{subsample_perc}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['qqp']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0]
}
    
cuda_devices = '[0,1]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_qqp.sh')
tasks = ['amazon']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
    'script': ['run_20newsgroups.py'],
    'validation_subsample': [0.0],
    'subsample_perc': [0.01]
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_amazon.sh')<jupyter_output><empty_output><jupyter_text>NUQ, Mahalanobis<jupyter_code>def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    
    if config['ue'] == 'mc_mahalanobis':
        config_list.append('ue.use_cache\=False')
    else:
        config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    if config['ue'] == 'nuq':
        config_list.append('ue.nuq.n_neighbors\=40')
        config_list.append('+ue.nuq.n_points\=50')
        config_list.append('+ue.nuq.n_folds\=10')
        config_list.append('+ue.nuq.n_samples\=3')
    
    config_list.append('training\=electra_base')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_glue_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                new_task += f' script={script}'
                
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' ue.margin\={margin}'
                args += f' ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                validation_subsample = conf['validation_subsample']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                subsample_perc = conf['subsample_perc']
                base_path=f' output_dir=../workdir/run_glue_for_model_series/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                #ue_type = conf['ue']
                ue_type = conf['ue'] if conf['ue'] != 'nuq' else 'nuq_best1'
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                model_series_dir=f'../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['qqp']

ue_configs = {
    'ue': ['nuq', 'mahalanobis'],
    'use_spectralnorm': ['False', 'True'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'script': ['run_glue.py'],
    'validation_subsample': [0.0],
    'subsample_perc': [0.1]
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_models_qqp_det_best.sh')
tasks = ['amazon']

ue_configs = {
    'ue': ['nuq', 'mahalanobis'],
    'use_spectralnorm': ['False', 'True'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'script': ['run_20newsgroups.py'],
    'validation_subsample': [0.0],
    'subsample_perc': [0.1]
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_models_amazon_det_best.sh')<jupyter_output><empty_output><jupyter_text>MC dropout<jupyter_code>def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))


    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    
    if config['ue'] == 'mc':
        config_list.append('ue.use_cache\=False')
    else:
        config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    
    config_list.append('training\=electra_base')
    
    config_list.append('ue.dropout_subs\={}'.format(config['dropout_subs']))

    config_list.append('ue.committee_size\={}'.format(config['committee_size']))
    config_list.append('ue.inference_prob\={}'.format(config['inference_prob']))
    
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_glue_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                script = conf['script']
                if task_name=='qqp':
                    script = 'run_glue.py'
                else:
                    script = 'run_20newsgroups.py'
                new_task += f' script={script}'
                
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                validation_subsample = conf['validation_subsample']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                subsample_perc = conf['subsample_perc']
                base_path=f' output_dir=../workdir/run_glue_for_model_series/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                ue_type = conf['ue']
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                model_series_dir=f'../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['qqp', 'amazon']

ue_configs = {
    'ue': ['mc'],
    'use_spectralnorm': ['False'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'script': [''],
    'validation_subsample': [0.0],
    'subsample_perc': [0.1],
    'calibrate': ['False'],
    'dropout_subs': ['all'],
    'committee_size': [20],
    'inference_prob': [0.1]
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_electra_models_qqp_amazon_mc_best.sh')<jupyter_output><empty_output><jupyter_text>MC-DPP<jupyter_code>import os 

def generate_eval_dpp_str_config(dpp_config, ds, ddpp):
    config_list = []
    config_list.append('ue\={}'.format(dpp_config['ue']))
    config_list.append('do_ue_estimate\=True')
    
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('data.validation_subsample\={}'.format(dpp_config['validation_subsample']))
    
    use_selective = 'False' if dpp_config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\=raw')
    config_list.append('training\=electra_base')
        
    config_list.append('ue.dropout.is_reused_mask\={}'.format(dpp_config['is_reused_mask']))
    config_list.append('ue.dropout.committee_size\={}'.format(dpp_config['dropout_committee_size']))
    
    ds_to_mf = {'amazon': {'ddpp_dpp': 0.4, 'ddpp_ood': 0.6},
                'qqp': {'ddpp_dpp': 0.6, 'ddpp_ood': 0.5}
    }
    max_frac = ds_to_mf[ds][ddpp]
    config_list.append('ue.dropout.max_frac\={}'.format(max_frac))    
    
    config_list.append('ue.committee_size\={}'.format(dpp_config['committee_size']))
    config_list.append('ue.dropout.use_ood_sampling\={}'.format(dpp_config['use_ood_sampling']))
    
    return config_list

def generate_bash_mcdpp(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_glue_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'
                
                use_ood_sampling = conf['use_ood_sampling']
                dpp_type = 'ood' if use_ood_sampling=='True' else 'dpp'
                ddpp = f'ddpp_{dpp_type}'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                if task_name=='qqp':
                    script = 'run_glue.py'
                else:
                    script = 'run_20newsgroups.py'
                new_task += f' script={script}'
                
                args = ' '.join(generate_func[i](conf, task_name, ddpp))
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'no_sn'
                
                with open(f'../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                
                validation_subsample = conf['validation_subsample']
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'no_sn'
                base_path=f' output_dir=../workdir/run_glue_for_model_series/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                base_path+= f'/ddpp_{dpp_type}/'
                new_task+=base_path
                
                model_series_dir = f'../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['qqp', 'amazon']

dpp_configs = {
    'ue': ['mc-dpp'],
    "committee_size": [100],
    'calibrate': ['True'],
    'reg_type': ['raw', 'reg-curr', 'metric'],
    'use_ood_sampling': ['False', 'True'],
    'dropout_committee_size': [50],
    'validation_subsample': [0.0],
    'is_reused_mask': ['True'],
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash_mcdpp([dpp_configs], cuda_devices, tasks, [generate_eval_dpp_str_config], file_name='run_eval_qqp_amazon_models_dpp.sh')<jupyter_output><empty_output><jupyter_text>Electra HP search for Amazon<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    j = 0
    n_devices = len(cuda_devices)
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                base_arg = f'CUDA_VISIBLE_DEVICES={cuda_devices[j%n_devices]} HYDRA_CONFIG_PATH=../configs/{task_name}.yaml python ./run_20newsgroups_with_hyp_search.py'

                new_task = copy.deepcopy(base_arg)
                args = ' '.join(generate_func[i](conf))
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                new_task += f' {args}'
                new_task += f' hydra.run.dir=../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}'
                new_task += " model.model_name_or_path='google/electra-base-discriminator'"
                if (j+1)%n_devices!=0: 
                    new_task += ' &'
                else:
                    new_task += '\nwait'
                full_config += '\n' + new_task if len(full_config) else new_task
                j+=1
                
    with open (f'../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['amazon']

train_configs = {
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
}
    
cuda_devices = [0, 1, 2, 3, 4, 5]

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'electra_hp_search_amazon.sh')<jupyter_output><empty_output><jupyter_text>Electra HP search NEW<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    j = 0
    n_devices = len(cuda_devices)
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                base_arg = f'CUDA_VISIBLE_DEVICES={cuda_devices[j%n_devices]} HYDRA_CONFIG_PATH=../configs/{task_name}.yaml python ./run_glue_with_hyp_search.py'

                new_task = copy.deepcopy(base_arg)
                args = ' '.join(generate_func[i](conf))
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                new_task += f' {args}'
                new_task += f' hydra.run.dir=../workdir/hp_search/electra_{reg}_{use_spectralnorm}/{task_name}'
                new_task += " model.model_name_or_path='google/electra-base-discriminator'"
                if (j+1)%n_devices!=0: 
                    new_task += ' &'
                else:
                    new_task += '\nwait'
                full_config += '\n' + new_task if len(full_config) else new_task
                j+=1
                
    with open (f'../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

train_configs = {
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
}
    
cuda_devices = [0, 1, 2]#'[0,1]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'electra_hp_search.sh')
tasks = ['qqp']

train_configs = {
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
}
    
cuda_devices = [0, 1]

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'electra_hp_search_qqp.sh')<jupyter_output><empty_output><jupyter_text>DeBERTA HP Search<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    j = 0
    n_devices = len(cuda_devices)
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                base_arg = f'CUDA_VISIBLE_DEVICES={cuda_devices[j%n_devices]} HYDRA_CONFIG_PATH=../configs/{task_name}.yaml python ./run_glue_with_hyp_search.py'

                new_task = copy.deepcopy(base_arg)
                args = ' '.join(generate_func[i](conf))
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                new_task += f' {args}'
                new_task += f' hydra.run.dir=../workdir/hp_search/deberta_{reg}_{use_spectralnorm}/{task_name}'
                new_task += " model.model_name_or_path='microsoft/deberta-base'"
                if (j+1)%n_devices!=0: 
                    new_task += ' &'
                else:
                    new_task += '\nwait'
                full_config += '\n' + new_task if len(full_config) else new_task
                j+=1
                
    with open (f'../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

train_configs = {
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
}
    
cuda_devices = [0, 1]#'[0,1]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'deberta_hp_search.sh')
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    j = 0
    n_devices = len(cuda_devices)

    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                base_arg = f'CUDA_VISIBLE_DEVICES={cuda_devices[j%n_devices]} HYDRA_CONFIG_PATH=../configs/{task_name}.yaml python ./run_conll2003_with_hyp_search.py'

                new_task = copy.deepcopy(base_arg)
                args = ' '.join(generate_func[i](conf))
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                new_task += f' {args}'
                new_task += f' hydra.run.dir=../workdir/hp_search/deberta_{reg}_{use_spectralnorm}/{task_name}'
                new_task += " model.model_name_or_path='microsoft/deberta-base'"
                if (j+1)%n_devices!=0: 
                    new_task += ' &'
                else:
                    new_task += '\nwait'
                full_config += '\n' + new_task if len(full_config) else new_task
                j+=1
                
    with open (f'../scripts/ner_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

train_configs = {
    'reg_type': ['metric'],#, 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
}
    
cuda_devices = [2]#'[0,1]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'deberta_hp_search.sh')<jupyter_output><empty_output><jupyter_text>DeBERTA train all CoNLL-2003<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('data.subsample_perc_val\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    
    config_list.append("model.model_name_or_path\=microsoft/deberta-base")
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                new_task += f' script=run_conll2003.py'
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                args = ' '.join(generate_func[i](conf))
                                
                with open(f'../../optimal_hyperparameters/deberta_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                new_task+=f' output_dir=../workdir/run_train_models/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../../scripts/ue_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False'],# 'True'],
    'validation_subsample': [0.1],
    'subsample_perc': [0.1]
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_deberta_models.sh')<jupyter_output><empty_output><jupyter_text>MC-DPP<jupyter_code>import os 

def generate_eval_dpp_str_config(dpp_config, ds, ddpp):
    config_list = []
    config_list.append('ue\={}'.format(dpp_config['ue']))
    config_list.append('do_ue_estimate\=True')
    
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('data.subsample_perc_val\={}'.format(dpp_config['validation_subsample']))
    
    use_selective = 'False' if dpp_config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(dpp_config['reg_type']))
    config_list.append('training\=electra_base')
        
    config_list.append('ue.dropout.is_reused_mask\={}'.format(dpp_config['is_reused_mask']))
    config_list.append('ue.dropout.committee_size\={}'.format(dpp_config['dropout_committee_size']))
    
    ds_to_mf = {
        'conll2003': {'ddpp_ood': 0.45, 'ddpp_dpp': 0.3}      
    }
    max_frac = ds_to_mf[ds][ddpp]
    config_list.append('ue.dropout.max_frac\={}'.format(max_frac))
    config_list.append('ue.dropout.mask_name_for_mask\=rbf')
    
    config_list.append('ue.committee_size\={}'.format(dpp_config['committee_size']))
    config_list.append('ue.dropout.use_ood_sampling\={}'.format(dpp_config['use_ood_sampling']))
    
    return config_list

def generate_bash_mcdpp(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'
                
                use_ood_sampling = conf['use_ood_sampling']
                dpp_type = 'ood' if use_ood_sampling=='True' else 'dpp'
                ddpp = f'ddpp_{dpp_type}'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                new_task += f' script=run_conll2003.py'
                
                args = ' '.join(generate_func[i](conf, task_name, ddpp))
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'no_sn'
                
                with open(f'../../optimal_hyperparameters/deberta_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                
                validation_subsample = conf['validation_subsample']
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'no_sn'
                base_path=f' output_dir=../workdir/run_tasks_for_model_series/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                base_path+= f'/ddpp_{dpp_type}/'
                new_task+=base_path
                
                model_series_dir = f'../workdir/run_train_models/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'
                
                calc_output_dir = f'../workdir/run_calc_ues_metrics/deberta_{reg}_{use_spectralnorm}/{task_name}/ddpp_{dpp_type}'
                calc_command = f'HYDRA_CONFIG_PATH=../configs/run_calc_ues_metrics.yaml python ./run_calc_ues_metrics_ner.py runs_dir=../workdir/run_tasks_for_model_series/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/ddpp_{dpp_type} extract_config=False output_dir={calc_output_dir};'

                full_config += '\n' + new_task if len(full_config) else new_task
                full_config += '\n' + calc_command
            
    with open (f'../../scripts/ue_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

dpp_configs = {
    'ue': ['mc-dpp'],
    #'max_frac' : [0.3, 0.4, 0.5, 0.6],
    "committee_size": [100],
    'calibrate': ['True'],
    'reg_type': ['raw', 'reg-curr', 'metric'],
    'use_ood_sampling': ['False', 'True'],
    'dropout_committee_size': [20],
    'validation_subsample': [0.1],
    'is_reused_mask': ['True'],
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash_mcdpp([dpp_configs], cuda_devices, tasks, [generate_eval_dpp_str_config], file_name='run_eval_deberta_models_dpp_best.sh')<jupyter_output><empty_output><jupyter_text>MC dropout<jupyter_code>def generate_eval_mc_str_config(mc_config):
    config_list = []
    
    config_list.append('ue\={}'.format(mc_config['ue']))
    config_list.append('do_ue_estimate\=True')
    
    config_list.append('ue.calibrate\={}'.format(mc_config['calibrate']))
    config_list.append('data.subsample_perc_val\={}'.format(mc_config['validation_subsample']))
    
    use_selective = 'False' if mc_config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.use_cache\=False')
    config_list.append('ue.reg_type\={}'.format(mc_config['reg_type']))
    config_list.append('training\=electra_base')

    config_list.append('ue.dropout_subs\={}'.format(mc_config['dropout_subs']))

    config_list.append('ue.committee_size\={}'.format(mc_config['committee_size']))
    config_list.append('ue.inference_prob\={}'.format(mc_config['inference_prob']))
    
    return config_list


def generate_bash_mc(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'
                
                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                new_task += f' script=run_conll2003.py'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'no_sn'
                
                with open(f'../../optimal_hyperparameters/deberta_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                
                validation_subsample = conf['validation_subsample']
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'no_sn'
                base_path=f' output_dir=../workdir/run_tasks_for_model_series/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                base_path+= f'/mc_all/'
                new_task+=base_path
                
                model_series_dir = f'../workdir/run_train_models/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'
                
                calc_output_dir = f'../workdir/run_calc_ues_metrics/deberta_{reg}_{use_spectralnorm}/{task_name}/mc_all'
                calc_command = f'HYDRA_CONFIG_PATH=../configs/run_calc_ues_metrics.yaml python ./run_calc_ues_metrics_ner.py runs_dir=../workdir/run_tasks_for_model_series/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/mc_all extract_config=False output_dir={calc_output_dir};'

                full_config += '\n' + new_task if len(full_config) else new_task
                full_config += '\n' + calc_command
            
    with open (f'../../scripts/ue_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
mc_configs = {
    'ue': ['mc'],
    'dropout_subs': ['all'],
    "committee_size": [20],
    'calibrate': ['False'],
    'inference_prob': [0.1],
    'reg_type': ['raw', 'reg-curr', 'metric'],
    'validation_subsample': [0.1],
}

cuda_devices = '[0,1,2,3,4,5]'

generate_bash_mc([mc_configs], cuda_devices, tasks, [generate_eval_mc_str_config], file_name='run_eval_deberta_models_mc_best.sh')<jupyter_output><empty_output><jupyter_text>NUQ, Mahalanobis, MC Mahalanobis<jupyter_code>def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    
    if config['ue'] == 'mc_mahalanobis':
        config_list.append('ue.use_cache\=False')
    else:
        config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    config_list.append('data.subsample_perc_val\={}'.format(config['validation_subsample']))
    
    config_list.append('training\=electra_base')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                new_task += f' script=run_conll2003.py'
                
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../../optimal_hyperparameters/deberta_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' ue.margin\={margin}'
                args += f' ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                validation_subsample = conf['validation_subsample']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                base_path=f' output_dir=../workdir/run_tasks_for_model_series/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                ue_type = conf['ue']
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                model_series_dir=f'../workdir/run_train_models/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                calc_output_dir = f'../workdir/run_calc_ues_metrics/deberta_{reg}_{use_spectralnorm}/{task_name}/{ue_type}'
                calc_command = f'HYDRA_CONFIG_PATH=../configs/run_calc_ues_metrics.yaml python ./run_calc_ues_metrics_ner.py runs_dir=../workdir/run_tasks_for_model_series/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/{ue_type} extract_config=False output_dir={calc_output_dir};'

                full_config += '\n' + new_task if len(full_config) else new_task
                full_config += '\n' + calc_command
            
    with open (f'../../scripts/ue_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

ue_configs = {
    'ue': ['mahalanobis'],#['nuq', 'mahalanobis', 'mc_mahalanobis'],
    'validation_subsample': [0.1],
    'use_spectralnorm': ['False'],# 'True'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
}

cuda_devices = '[0,1,2,3,4,5]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_deberta_models_det_best.sh')<jupyter_output><empty_output><jupyter_text>Deep Ensemble<jupyter_code>def generate_bash(cuda_devices, tasks):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_ensemble_series.yaml python ./run_train_ensemble_series.py'
    for task_name in tasks:
        task_configs = f'{task_name}.yaml'
        subsample_perc = 0.1
        args = f'data.subsample_perc\={subsample_perc} ue.calibrate\=True data.subsample_perc_val\=0.1'
        
        with open(f'../../../workdir/hp_search/deberta_raw_no_sn/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

        num_train_epochs = params['num_train_epochs']
        args += f' training.num_train_epochs\={num_train_epochs}'
        learning_rate = params['learning_rate']
        args += f' training.learning_rate\={learning_rate}'
        per_device_train_batch_size = params['per_device_train_batch_size']
        args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
        weight_decay = params['weight_decay']
        args += f' +training.weight_decay\={weight_decay}'
        
        margin = params['margin']
        lamb_intra = params['lamb_intra']
        lamb = params['lamb']
        args += f' +ue.margin\={margin}'
        args += f' +ue.lamb_intra\={lamb_intra}'
        args += f' ue.lamb\={lamb}'
        args += ' model.model_name_or_path\=microsoft/deberta-base'
        
        output_dir = f'../workdir/run_train_ensemble_series/deberta/{task_name}/'
        
        new_task = copy.deepcopy(base_arg)
        new_task += f" task_configs={task_configs} args='{args}' output_dir='{output_dir}' cuda_devices={cuda_devices}"
        new_task += f' script=run_conll2003.py'
        
        full_config += '\n' + new_task if len(full_config) else new_task
    with open ('../../scripts/ue_scripts/run_train_deberta_ensemble_models_ner.sh', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

cuda_devices = '[0,1,2,3,4,5]'

generate_bash(cuda_devices, tasks)
def generate_bash_eval_ens(cuda_devices, tasks, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_ensemble_series.yaml python run_tasks_for_ensemble_series.py'
    for task_name in tasks:
        task_configs = f'{task_name}.yaml'

        new_task = copy.deepcopy(base_arg)
        new_task += f' cuda_devices={cuda_devices}'
        new_task += f' config_path=../configs/{task_configs}'
        
        
        args = f'data.subsample_perc\=0.1 ue.calibrate\=True data.subsample_perc_val\=0.1'
        with open(f'../../../workdir/hp_search/deberta_raw_no_sn/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
            params = yaml.load(f, Loader=yaml.FullLoader)

        num_train_epochs = params['num_train_epochs']
        args += f' training.num_train_epochs\={num_train_epochs}'
        learning_rate = params['learning_rate']
        args += f' training.learning_rate\={learning_rate}'
        per_device_train_batch_size = params['per_device_train_batch_size']
        args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
        weight_decay = params['weight_decay']
        args += f' +training.weight_decay\={weight_decay}'
        
        margin = params['margin']
        lamb_intra = params['lamb_intra']
        lamb = params['lamb']
        args += f' +ue.margin\={margin}'
        args += f' +ue.lamb_intra\={lamb_intra}'
        args += f' ue.lamb\={lamb}'

        ensemble_series_dir = f'../workdir/run_train_ensemble_series/deberta/{task_name}/ensembles/'
        new_task+=f" args='{args}' ensemble_series_dir={ensemble_series_dir}"
        new_task+=f' output_dir=../workdir/run_tasks_for_ensemble_series/deberta/{task_name}/'
        new_task+=f' script=run_conll2003.py'

        calc_output_dir = f'../workdir/run_calc_ues_metrics/deberta_raw_no_sn/{task_name}/ensemble'
        calc_command = f'HYDRA_CONFIG_PATH=../configs/run_calc_ues_metrics.yaml python ./run_calc_ues_metrics_ner.py runs_dir=../workdir/run_tasks_for_ensemble_series/deberta/{task_name}/final_results extract_config=False output_dir={calc_output_dir};'

        full_config += '\n' + new_task if len(full_config) else new_task
        full_config += '\n' + calc_command

    with open (f'../../scripts/ue_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

cuda_devices = '0'

generate_bash_eval_ens(cuda_devices, tasks, 'run_eval_deberta_ensemble_series_ner.sh')<jupyter_output><empty_output><jupyter_text>DeBERTA train all GLUE<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    
    config_list.append("model.model_name_or_path\=microsoft/deberta-base")
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                args = ' '.join(generate_func[i](conf))
                
                
                with open(f'../../optimal_hyperparameters/deberta_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                new_task+=f' output_dir=../workdir/run_train_models/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../../scripts/ue_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False'],# 'True'],
    'validation_subsample': [0.0]
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_deberta_models_glue.sh')<jupyter_output><empty_output><jupyter_text>MC-DPP<jupyter_code>import os 

def generate_eval_dpp_str_config(dpp_config, ds, ddpp):
    config_list = []
    config_list.append('ue\={}'.format(dpp_config['ue']))
    config_list.append('do_ue_estimate\=True')
    
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('data.validation_subsample\={}'.format(dpp_config['validation_subsample']))
    
    use_selective = 'False' if dpp_config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(dpp_config['reg_type']))
    config_list.append('training\=electra_base')
        
    config_list.append('ue.dropout.is_reused_mask\={}'.format(dpp_config['is_reused_mask']))
    config_list.append('ue.dropout.committee_size\={}'.format(dpp_config['dropout_committee_size']))

    ds_to_mf = {
        'mrpc': {'ddpp_ood': 0.45, 'ddpp_dpp': 0.6},
        'cola': {'ddpp_ood': 0.45, 'ddpp_dpp': 0.6},
        'sst2': {'ddpp_ood': 0.45, 'ddpp_dpp': 0.6}
    }
    
    max_frac = ds_to_mf[ds][ddpp]
    config_list.append('ue.dropout.max_frac\={}'.format(max_frac)) 
    mask_name_for_mask = 'rbf'
    config_list.append('ue.dropout.mask_name_for_mask\={}'.format(mask_name_for_mask)) 
    
    config_list.append('ue.committee_size\={}'.format(dpp_config['committee_size']))
    config_list.append('ue.dropout.use_ood_sampling\={}'.format(dpp_config['use_ood_sampling']))
    
    return config_list

def generate_bash_mcdpp(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'
                
                use_ood_sampling = conf['use_ood_sampling']
                dpp_type = 'ood' if use_ood_sampling=='True' else 'dpp'
                ddpp = f'ddpp_{dpp_type}'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf, task_name, ddpp))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                
                validation_subsample = conf['validation_subsample']
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'no_sn'
                base_path=f' output_dir=../workdir/run_tasks_for_model_series/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                base_path+= f'/ddpp_{dpp_type}/'
                new_task+=base_path
                
                model_series_dir = f'../workdir/run_train_models/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                calc_output_dir = f'../workdir/run_calc_ues_metrics/deberta_{reg}_{use_spectralnorm}/{task_name}/ddpp_{dpp_type}'
                calc_command = f'HYDRA_CONFIG_PATH=../configs/run_calc_ues_metrics.yaml python ./run_calc_ues_metrics.py runs_dir=../workdir/run_tasks_for_model_series/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/ddpp_{dpp_type} extract_config=False output_dir={calc_output_dir};'

                full_config += '\n' + new_task if len(full_config) else new_task
                full_config += '\n' + calc_command
            
    with open (f'../../scripts/ue_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

dpp_configs = {
    'ue': ['mc-dpp'],
    #'max_frac' : [0.3, 0.4, 0.5, 0.6],
    "committee_size": [100],
    'calibrate': ['True'],
    'reg_type': ['raw', 'reg-curr', 'metric'],
    'use_ood_sampling': ['False', 'True'],
    'dropout_committee_size': [50],
    'validation_subsample': [0.0],
    'is_reused_mask': ['True'],
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash_mcdpp([dpp_configs], cuda_devices, tasks, [generate_eval_dpp_str_config], file_name='run_eval_deberta_models_glue_dpp_best.sh')<jupyter_output><empty_output><jupyter_text>MC Dropout<jupyter_code>def generate_eval_mc_str_config(mc_config):
    config_list = []
    
    config_list.append('ue\={}'.format(mc_config['ue']))
    config_list.append('do_ue_estimate\=True')
    
    config_list.append('ue.calibrate\={}'.format(mc_config['calibrate']))
    config_list.append('data.validation_subsample\={}'.format(mc_config['validation_subsample']))
    
    use_selective = 'False' if mc_config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(mc_config['reg_type']))
    config_list.append('training\=electra_base')

    config_list.append('ue.dropout_subs\={}'.format(mc_config['dropout_subs']))

    config_list.append('ue.committee_size\={}'.format(mc_config['committee_size']))
    config_list.append('ue.inference_prob\={}'.format(mc_config['inference_prob']))
    
    return config_list



def generate_bash_mc(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'
                
                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                
                validation_subsample = conf['validation_subsample']
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'no_sn'
                base_path=f' output_dir=../workdir/run_tasks_for_model_series/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                base_path+= f'/mc_all/'
                new_task+=base_path
                
                model_series_dir = f'../workdir/run_train_models/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                calc_output_dir = f'../workdir/run_calc_ues_metrics/deberta_{reg}_{use_spectralnorm}/{task_name}/mc_all'
                calc_command = f'HYDRA_CONFIG_PATH=../configs/run_calc_ues_metrics.yaml python ./run_calc_ues_metrics.py runs_dir=../workdir/run_tasks_for_model_series/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/mc_all extract_config=False output_dir={calc_output_dir};'

                full_config += '\n' + new_task if len(full_config) else new_task
                full_config += '\n' + calc_command
            
    with open (f'../../scripts/ue_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

mc_configs = {
    'ue': ['mc'],
    'dropout_subs': ['all'],
    "committee_size": [20],
    'calibrate': ['False'],
    'inference_prob': [0.1],
    'reg_type': ['raw', 'reg-curr', 'metric'],
    'validation_subsample': [0.0],
}

cuda_devices = '[0,1,2,3,4,5]'

generate_bash_mc([mc_configs], cuda_devices, tasks, [generate_eval_mc_str_config], file_name='run_eval_deberta_models_glue_mc_best.sh')<jupyter_output><empty_output><jupyter_text>Mahalanobis, NUQ, MC Mahalanobis<jupyter_code>def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    
    config_list.append('training.per_device_eval_batch_size\=64') #due to out-of-memory

    config_list.append('do_ue_estimate\=True')
    config_list.append('ue.use_cache\=True')
    config_list.append('ue.calibrate\=False')
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    
    config_list.append('training\=electra_base')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                validation_subsample = conf['validation_subsample']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                base_path=f' output_dir=../workdir/run_tasks_for_model_series/deberta_{reg}_{use_spectralnorm}/{task_name}/0.0'
                
                ue_type = conf['ue']
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                model_series_dir=f'../workdir/run_train_models/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                calc_output_dir = f'../workdir/run_calc_ues_metrics/deberta_{reg}_{use_spectralnorm}/{task_name}/{ue_type}'
                calc_command = f'HYDRA_CONFIG_PATH=../configs/run_calc_ues_metrics.yaml python ./run_calc_ues_metrics.py runs_dir=../workdir/run_tasks_for_model_series/deberta_{reg}_{use_spectralnorm}/{task_name}/0.0/{ue_type} extract_config=False output_dir={calc_output_dir};'

                full_config += '\n' + new_task if len(full_config) else new_task
                full_config += '\n' + calc_command
            
    with open (f'../../scripts/ue_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

ue_configs = {
    'ue': ['mahalanobis'],# 'nuq', 'mc_mahalanobis'],
    'validation_subsample': [0.0],
    'use_spectralnorm': ['False'],# 'True'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
}

cuda_devices = '[0,1,2,3,4,5]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_deberta_models_glue_det_best.sh')<jupyter_output><empty_output><jupyter_text>Deep Ensemble<jupyter_code>def generate_bash(cuda_devices, tasks):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_ensemble_series.yaml python ./run_train_ensemble_series.py'
    for task_name in tasks:
        task_configs = f'{task_name}.yaml'
        subsample_perc = 0.63 if task_name!='sst2' else 0.1
        args = f'data.subsample_perc\={subsample_perc} ue.calibrate\=True data.validation_subsample\=0.0 do_train\=True do_eval\=False do_ue_estimate\=False'
        
        with open(f'../../../workdir/hp_search/deberta_raw_no_sn/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

        num_train_epochs = params['num_train_epochs']
        args += f' training.num_train_epochs\={num_train_epochs}'
        learning_rate = params['learning_rate']
        args += f' training.learning_rate\={learning_rate}'
        per_device_train_batch_size = params['per_device_train_batch_size']
        args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
        weight_decay = params['weight_decay']
        args += f' +training.weight_decay\={weight_decay}'
        args += ' model.model_name_or_path\=microsoft/deberta-base'
        
        output_dir = f'../workdir/run_train_ensemble_series/deberta/{task_name}/'
        
        new_task = copy.deepcopy(base_arg)
        new_task += f" task_configs={task_configs} args='{args}' output_dir='{output_dir}' cuda_devices={cuda_devices}"
        full_config += '\n' + new_task if len(full_config) else new_task
    with open ('../../scripts/ue_scripts/run_train_deberta_ensemble_models.sh', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

cuda_devices = '[0,1,2,3,4,5]'

generate_bash(cuda_devices, tasks)
def generate_bash_eval_ens(cuda_devices, tasks, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_ensemble_series.yaml python run_tasks_for_ensemble_series.py'
    for task_name in tasks:
        task_configs = f'{task_name}.yaml'

        new_task = copy.deepcopy(base_arg)
        new_task += f' cuda_devices={cuda_devices}'
        new_task += f' config_path=../configs/{task_configs}'

        ensemble_series_dir = f'../workdir/run_train_ensemble_series/deberta/{task_name}/ensembles/'
        new_task+=f' ensemble_series_dir={ensemble_series_dir}'
        new_task+=f' output_dir=../workdir/run_tasks_for_ensemble_series/deberta/{task_name}/'

        calc_output_dir = f'../workdir/run_calc_ues_metrics/deberta_raw_no_sn/{task_name}/ensemble'
        calc_command = f'HYDRA_CONFIG_PATH=../configs/run_calc_ues_metrics.yaml python ./run_calc_ues_metrics.py runs_dir=../workdir/run_tasks_for_ensemble_series/deberta/{task_name}/final_results extract_config=False output_dir={calc_output_dir};'

        full_config += '\n' + new_task if len(full_config) else new_task
        full_config += '\n' + calc_command

    with open (f'../../scripts/ue_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

cuda_devices = '0'

generate_bash_eval_ens(cuda_devices, tasks, 'run_eval_deberta_ensemble_series.sh')<jupyter_output><empty_output><jupyter_text>Deberta MD SN search<jupyter_code>def generate_train_str_config(dpp_config):
    config_list = []
    config_list.append('ue\=mc do_ue_estimate\=False')
    
    use_selective = 'False' if dpp_config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(dpp_config['reg_type']))
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('+ue.use_spectralnorm\={}'.format(dpp_config['use_spectralnorm_train']))
    config_list.append('+ue.sn_value\={}'.format(dpp_config['sn_value']))
    config_list.append('data.validation_subsample\={}'.format(dpp_config['validation_subsample']))
    config_list.append('+data.eval_subsample\={}'.format(dpp_config['eval_subsample']))
    config_list.append('training\=electra_base')
    config_list.append("model.model_name_or_path\=microsoft/deberta-base")
    
    return config_list

def generate_ue_eval_methods_str_config(mc_config):
    config_list = []
    
    config_list.append('ue\={}'.format(mc_config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(mc_config['use_spectralnorm_inference']))

    use_selective = 'False' if mc_config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('do_ue_estimate\=True')
    config_list.append('ue.use_cache\=True')
    config_list.append('ue.calibrate\=False')
    
    config_list.append('+ue.sn_value\={}'.format(mc_config['sn_value']))
    config_list.append('data.validation_subsample\={}'.format(mc_config['validation_subsample']))
    config_list.append('+data.eval_subsample\={}'.format(mc_config['eval_subsample']))
    
    config_list.append('training\=electra_base')
    
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                try:
                    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
                    task_configs = f'{task_name}.yaml'

                    new_task = copy.deepcopy(base_arg)
                    new_task += f' cuda_devices={cuda_devices}'
                    new_task += f' script=run_glue_dpp_hp.py'

                    args = ' '.join(generate_func[0](conf))

                    reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                    validation_subsample = conf['validation_subsample']
                    eval_subsample = conf['eval_subsample']
                    use_spectralnorm = 'sn' if conf['use_spectralnorm_train'] == 'True' else 'no_sn'
                    
                    with open(f'../../../workdir/hp_search/deberta_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                        params = yaml.load(f, Loader=yaml.FullLoader)

                    num_train_epochs = params['num_train_epochs']
                    args += f' training.num_train_epochs\={num_train_epochs}'
                    learning_rate = params['learning_rate']
                    args += f' training.learning_rate\={learning_rate}'
                    per_device_train_batch_size = params['per_device_train_batch_size']
                    args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                    weight_decay = params['weight_decay']
                    args += f' +training.weight_decay\={weight_decay}'

                    try:
                        margin = params['margin']
                        lamb_intra = params['lamb_intra']
                    except:
                        margin = 0
                        lamb_intra = 0

                    args += f' +ue.margin\={margin}'
                    args += f' +ue.lamb_intra\={lamb_intra}'
                    lamb = params['lamb']
                    args += f' ue.lamb\={lamb}'

                    new_task += f" args='{args}'"
                    new_task += f' task_configs={task_configs}'

                    sn_value = conf['sn_value']
                    new_task+=f' output_dir=../workdir/run_train_models_sn_params/deberta_{reg}_{use_spectralnorm}/{task_name}/{eval_subsample}/{sn_value}'

                    full_config += '\n' + new_task if len(full_config) else new_task



                    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
                    task_configs = f'{task_name}.yaml'

                    new_task = copy.deepcopy(base_arg)
                    new_task += f' cuda_devices={cuda_devices}'
                    args = ' '.join(generate_func[1](conf))
                    new_task += f" args='{args}'"
                    new_task += f' config_path=../configs/{task_configs}'
                    new_task += f' script=run_glue_dpp_hp.py'

                    reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                    validation_subsample = conf['validation_subsample']
                    eval_subsample = conf['eval_subsample']
                    use_spectralnorm = 'sn' if conf['use_spectralnorm_train'] == 'True' else 'no_sn'
                    sn_value = conf['sn_value']

                    base_path=f' output_dir=../workdir/run_tasks_for_model_series_sn_params/deberta_{reg}_{use_spectralnorm}/{task_name}/{eval_subsample}/{sn_value}'

                    ue_type = conf['ue']
                    base_path+= f'/{ue_type}'
                    new_task+=base_path

                    model_series_dir=f'../workdir/run_train_models_sn_params/deberta_{reg}_{use_spectralnorm}/{task_name}/{eval_subsample}/{sn_value}/models/{task_name}/'
                    new_task+=f' model_series_dir={model_series_dir}'

                    calc_output_dir = f'../workdir/run_calc_ues_metrics_sn_params/deberta_{reg}_{use_spectralnorm}/{task_name}/{sn_value}/{ue_type}'
                    calc_command = f'HYDRA_CONFIG_PATH=../configs/run_calc_ues_metrics.yaml python ./run_calc_ues_metrics.py runs_dir=../workdir/run_tasks_for_model_series_sn_params/deberta_{reg}_{use_spectralnorm}/{task_name}/{eval_subsample}/{sn_value}/{ue_type} extract_config=False output_dir={calc_output_dir};'

                    full_config += '\n' + new_task if len(full_config) else new_task
                    full_config += '\n' + calc_command


                    full_config += f'\nrm -rf {model_series_dir}'
                except:
                    pass
    with open (f'../../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

ue_configs = {
    'calibrate': ['False'],
    'ue': ['mahalanobis'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'validation_subsample': [0.0],
    'eval_subsample': [0.2],
    'sn_value': [0.4, 0.6, 0.8, 1, 2, 3],
    'use_spectralnorm_inference': ['False'],
    'use_spectralnorm_train': ['True'],
}
    
cuda_devices = '[0,1,2,3,4,5]'


generate_bash([ue_configs], cuda_devices, tasks, 
              [generate_train_str_config, generate_ue_eval_methods_str_config], filename='run_train_eval_det_models_with_new_sn_params_deberta.sh')
def generate_train_str_config(dpp_config):
    config_list = []
    config_list.append('ue\=mc do_ue_estimate\=False')
    
    use_selective = 'False' if dpp_config['reg_type']=='raw' else 'True'
    
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(dpp_config['reg_type']))
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('+ue.use_spectralnorm\={}'.format(dpp_config['use_spectralnorm_train']))
    config_list.append('+ue.sn_value\={}'.format(dpp_config['sn_value']))
    config_list.append('data.subsample_perc_val\={}'.format(dpp_config['subsample_perc_val']))
    config_list.append('data.subsample_perc\={}'.format(dpp_config['subsample_perc']))
    config_list.append('ue.use_cache\=False')
    config_list.append('training\=electra_base')
    config_list.append("model.model_name_or_path\=microsoft/deberta-base")
    
    return config_list

def generate_ue_eval_methods_str_config(mc_config):
    config_list = []
    
    config_list.append('ue\={}'.format(mc_config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(mc_config['use_spectralnorm_inference']))

    use_selective = 'False' if mc_config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('do_ue_estimate\=True')
    config_list.append('ue.use_cache\=False')
    config_list.append('ue.calibrate\=False')
    
    config_list.append('+ue.sn_value\={}'.format(mc_config['sn_value']))
    config_list.append('data.subsample_perc_val\={}'.format(mc_config['subsample_perc_val']))
    config_list.append('data.subsample_perc\={}'.format(mc_config['subsample_perc']))
    
    config_list.append('training\=electra_base')
    
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                new_task += f' script=run_conll2003_dpp_hp.py'

                args = ' '.join(generate_func[0](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                eval_subsample = conf['subsample_perc_val']
                use_spectralnorm = 'sn' if conf['use_spectralnorm_train'] == 'True' else 'no_sn'
                
                with open(f'../../../workdir/hp_search/deberta_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                                
                try:
                    margin = params['margin']
                    lamb_intra = params['lamb_intra']
                except:
                    margin = 0
                    lamb_intra = 0
                    
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                lamb = params['lamb']
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                sn_value = conf['sn_value']
                new_task+=f' output_dir=../workdir/run_train_models_sn_params/deberta_{reg}_{use_spectralnorm}/{task_name}/{eval_subsample}/{sn_value}'

                full_config += '\n' + new_task if len(full_config) else new_task
                
                
                
                base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[1](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                new_task += f' script=run_conll2003_dpp_hp.py'
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                eval_subsample = conf['subsample_perc_val']
                use_spectralnorm = 'sn' if conf['use_spectralnorm_train'] == 'True' else 'no_sn'
                sn_value = conf['sn_value']
                
                base_path=f' output_dir=../workdir/run_tasks_for_model_series_sn_params/deberta_{reg}_{use_spectralnorm}/{task_name}/{eval_subsample}/{sn_value}'
                
                ue_type = conf['ue']
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                model_series_dir=f'../workdir/run_train_models_sn_params/deberta_{reg}_{use_spectralnorm}/{task_name}/{eval_subsample}/{sn_value}/models/{task_name}/'
                new_task+=f' model_series_dir={model_series_dir}'

                calc_output_dir = f'../workdir/run_calc_ues_metrics_sn_params/deberta_{reg}_{use_spectralnorm}/{task_name}/{sn_value}/{ue_type}'
                calc_command = f'HYDRA_CONFIG_PATH=../configs/run_calc_ues_metrics.yaml python ./run_calc_ues_metrics_ner.py runs_dir=../workdir/run_tasks_for_model_series_sn_params/deberta_{reg}_{use_spectralnorm}/{task_name}/{eval_subsample}/{sn_value}/{ue_type} extract_config=False output_dir={calc_output_dir};'

                full_config += '\n' + new_task if len(full_config) else new_task
                full_config += '\n' + calc_command
                
                
                full_config += f'\nrm -rf {model_series_dir}'
    with open (f'../../scripts/ner_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

ue_configs = {
    'calibrate': ['False'],
    'ue': ['mahalanobis'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'subsample_perc_val': [0.1],
    'subsample_perc': [0.1],
    'sn_value': [0.4, 0.6, 0.8, 1, 2, 3],
    'use_spectralnorm_inference': ['False'],
    'use_spectralnorm_train': ['True'],
}

cuda_devices = '[0,1,2,3,4,5]'


generate_bash([ue_configs], cuda_devices, tasks, 
              [generate_train_str_config, generate_ue_eval_methods_str_config], filename='run_train_eval_det_models_with_new_sn_params_deberta.sh')<jupyter_output><empty_output><jupyter_text>Deberta optimal MDSN<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('+ue.sn_value\={}'.format(config['sn_value']))
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    
    config_list.append("model.model_name_or_path\=microsoft/deberta-base")
    config_list.append('training\=electra_base')
    return config_list

deberta_sn_values = {
    'cola': {
        'metric': 0.8,
        'reg': 0.4,
        'raw': 0.4,
    },
    'mrpc': {
        'metric': 1,
        'reg': 0.6,
        'raw': 0.8,
    },
    'sst2': {
        'metric': 0.6,
        'reg': 0.6,
        'raw': 0.6,
    },
    'conll2003': {
        'metric': 2,
        'reg': 1,
        'raw': 1,
    },
}
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                conf['sn_value'] = deberta_sn_values[task_name][reg]
                args = ' '.join(generate_func[i](conf))
                
                
                with open(f'../../optimal_hyperparameters/deberta_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                new_task+=f' output_dir=../workdir/run_train_models/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../../scripts/ue_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['True'],
    'validation_subsample': [0.0]
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_deberta_models_glue_sn.sh')
def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    
    config_list.append('training.per_device_eval_batch_size\=64') #due to out-of-memory

    config_list.append('do_ue_estimate\=True')
    config_list.append('ue.use_cache\=True')
    config_list.append('ue.calibrate\=False')
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    
    config_list.append('training\=electra_base')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                validation_subsample = conf['validation_subsample']
                use_spectralnorm = 'sn' if conf['use_spectralnorm_model'] == 'True' else 'no_sn'
                
                base_path=f' output_dir=../workdir/run_tasks_for_model_series/deberta_{reg}_{use_spectralnorm}/{task_name}/0.0'
                
                ue_type = conf['ue']
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                model_series_dir=f'../workdir/run_train_models/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                calc_output_dir = f'../workdir/run_calc_ues_metrics/deberta_{reg}_{use_spectralnorm}/{task_name}/{ue_type}'
                calc_command = f'HYDRA_CONFIG_PATH=../configs/run_calc_ues_metrics.yaml python ./run_calc_ues_metrics.py runs_dir=../workdir/run_tasks_for_model_series/deberta_{reg}_{use_spectralnorm}/{task_name}/0.0/{ue_type} extract_config=False output_dir={calc_output_dir};'

                full_config += '\n' + new_task if len(full_config) else new_task
                full_config += '\n' + calc_command
            
    with open (f'../../scripts/ue_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

ue_configs = {
    'ue': ['mahalanobis'],
    'validation_subsample': [0.0],
    'use_spectralnorm': ['False'],
    'use_spectralnorm_model': ['True'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
}

cuda_devices = '[0,1,2,3,4,5]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_deberta_models_glue_det_best_sn.sh')
full_test = 'bash run_train_deberta_models_glue_sn.sh\n'
full_test += 'bash run_eval_deberta_models_glue_det_best_sn.sh'

with open (f'../../scripts/ue_scripts/run_deberta_new_sn_params.sh', 'w') as rsh:
    rsh.write(full_test)
def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('+ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))
    config_list.append('data.subsample_perc_val\={}'.format(config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    config_list.append('+ue.sn_value\={}'.format(config['sn_value']))
    
    config_list.append("model.model_name_or_path\=microsoft/deberta-base")
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                conf['sn_value'] = deberta_sn_values[task_name][reg]
                
                new_task += f' script=run_conll2003.py'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../../optimal_hyperparameters/deberta_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                new_task+=f' output_dir=../workdir/run_train_models/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../../scripts/ue_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['True'],
    'validation_subsample': [0.1],
    'subsample_perc': [0.1]
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_deberta_models_sn.sh')
def generate_ue_eval_methods_str_config(config):
    config_list = []
    
    config_list.append('ue\={}'.format(config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(config['use_spectralnorm']))

    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    
    if config['ue'] == 'mc_mahalanobis':
        config_list.append('ue.use_cache\=False')
    else:
        config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    config_list.append('data.subsample_perc_val\={}'.format(config['validation_subsample']))
    
    config_list.append('training\=electra_base')
    
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                new_task += f' script=run_conll2003.py'
                
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm_model'] == 'True' else 'no_sn'
                
                with open(f'../../optimal_hyperparameters/deberta_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' ue.margin\={margin}'
                args += f' ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                validation_subsample = conf['validation_subsample']
                
                base_path=f' output_dir=../workdir/run_tasks_for_model_series/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                ue_type = conf['ue']
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                model_series_dir=f'../workdir/run_train_models/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                calc_output_dir = f'../workdir/run_calc_ues_metrics/deberta_{reg}_{use_spectralnorm}/{task_name}/{ue_type}'
                calc_command = f'HYDRA_CONFIG_PATH=../configs/run_calc_ues_metrics.yaml python ./run_calc_ues_metrics_ner.py runs_dir=../workdir/run_tasks_for_model_series/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/{ue_type} extract_config=False output_dir={calc_output_dir};'

                full_config += '\n' + new_task if len(full_config) else new_task
                full_config += '\n' + calc_command
            
    with open (f'../../scripts/ue_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

ue_configs = {
    'ue': ['mahalanobis'],
    'validation_subsample': [0.1],
    'use_spectralnorm': ['False'],
    'use_spectralnorm_model': ['True'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
}

cuda_devices = '[0,1,2,3,4,5]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_deberta_models_det_best_sn.sh')
full_test = 'bash run_train_deberta_models_sn.sh\n'
full_test += 'bash run_eval_deberta_models_det_best_sn.sh'

with open (f'../../scripts/ue_scripts/run_deberta_ner_new_sn_params.sh', 'w') as rsh:
    rsh.write(full_test)<jupyter_output><empty_output><jupyter_text>SNGP NER<jupyter_code>def generate_train_str_config(dpp_config):
    config_list = []
    config_list.append('ue\=sngp do_ue_estimate\=False')
    
    config_list.append('ue.use_selective\={}'.format(dpp_config['use_selective']))
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('data.subsample_perc_val\={}'.format(dpp_config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(dpp_config['subsample_perc']))
    
    #config_list.append('training.per_device_eval_batch_size\={}'.format(dpp_config['per_device_eval_batch_size']))
    #config_list.append('training.num_train_epochs\={}'.format(dpp_config['num_train_epochs']))
    #config_list.append('training.learning_rate\={}'.format(dpp_config['learning_rate']))

    if dpp_config['use_selective'] == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models_ner.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                subsample_perc = conf['subsample_perc']
                conf['num_train_epochs'] = 80 if  subsample_perc == 0.1 else 8
                args = ' '.join(generate_func[i](conf))
                with open(f'../workdir/sngp/{task_name}/sngp_/0.1/2021-10-11/09-46-55-0/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)
                
                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                if conf['use_selective'] == 'True':
                    new_task+=f' output_dir=../workdir/run_train_models/electra-reg-sngp/{task_name}/{subsample_perc}'
                else:
                    new_task+=f' output_dir=../workdir/run_train_models/electra-raw-sngp/{task_name}/{subsample_perc}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open ('../scripts/ner_scripts/run_train_models_sngp_ner.sh', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

train_configs = {
    'calibrate': ['False'],
    'use_selective': ['False'],
    'validation_subsample': [0.1],
    'subsample_perc': [0.1],
}
    
cuda_devices = '[1,2,3]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config])<jupyter_output><empty_output><jupyter_text>Eval SNGP NER<jupyter_code>import os 

def generate_eval_sngp_str_config(mc_config):
    config_list = []
    
    config_list.append('ue\={}'.format(mc_config['ue']))
    config_list.append('do_ue_estimate\=True')
    
    config_list.append('ue.calibrate\={}'.format(mc_config['calibrate']))
    config_list.append('data.subsample_perc_val\={}'.format(mc_config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(mc_config['subsample_perc']))
    
    config_list.append('training.per_device_eval_batch_size\={}'.format(mc_config['per_device_eval_batch_size']))
    
    if mc_config['use_selective'] == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')

    config_list.append('ue.committee_size\={}'.format(mc_config['committee_size']))
    
    return config_list


def generate_bash_sngp(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_conll2003_for_model_series.yaml python run_conll2003_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                
                subsample_perc = conf['subsample_perc']
                if conf['use_selective'] == 'True':
                    base_path=f' output_dir=../workdir/run_ner_for_model_series/electra-reg-sngp/{task_name}/{subsample_perc}'
                else:
                    base_path=f' output_dir=../workdir/run_ner_for_model_series/electra-raw-sngp/{task_name}/{subsample_perc}'
                new_task+=base_path
                
                if conf['use_selective'] == 'True':
                    model_series_dir = f'../workdir/run_train_models/electra-reg-sngp/{task_name}/{subsample_perc}/models/{task_name}'
                else:
                    model_series_dir = f'../workdir/run_train_models/electra-raw-sngp/{task_name}/{subsample_perc}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/ner_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

eval_configs = {
    'ue': ['sngp'],
    'calibrate': ['False'],
    'use_selective': ['False'],
    'use_spectralnorm': ['False'],
    'validation_subsample': [0.1],
    'subsample_perc': [0.1],
    'per_device_eval_batch_size': [64],
    'committee_size': [10]
}
    
cuda_devices = '[1,2,3]'

generate_bash_sngp([eval_configs], cuda_devices, tasks, [generate_eval_sngp_str_config], file_name='run_eval_sngp_ner.sh')<jupyter_output><empty_output><jupyter_text>Generate train scripts for sngp<jupyter_code>def generate_train_str_config(dpp_config):
    config_list = []
    config_list.append('ue\=sngp do_ue_estimate\=False')
    
    config_list.append('ue.use_selective\={}'.format(dpp_config['use_selective']))
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('ue.use_spectralnorm\={}'.format(dpp_config['use_spectralnorm']))
    config_list.append('data.validation_subsample\={}'.format(dpp_config['validation_subsample']))
    
    config_list.append('ue.sngp.ridge_factor\={}'.format(dpp_config['ridge_factor']))
    config_list.append('ue.sngp.momentum\={}'.format(dpp_config['momentum']))
    
    if dpp_config['use_selective'] == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')
    return config_list
import yaml

def generate_bash(configs, cuda_devices, tasks, generate_func):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                output_dir = f'../workdir/run_train_models/electra-calibrate/{task_name}'
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                size = 0.0 if task_name!='sst2' else 0.1
                with open(f'../workdir/sngp/{task_name}/sngp_/{size}/2021-10-11/07-01-17-0/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)
                
                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                use_spectralnorm = conf['use_spectralnorm']
                if conf['use_selective'] == 'True':
                    new_task+=f' output_dir=../workdir/run_train_models/electra-reg-sngp-correct-hp/{task_name}/{validation_subsample}'
                else:
                    new_task+=f' output_dir=../workdir/run_train_models/electra-raw-sngp-correct-hp/{task_name}/{validation_subsample}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open ('../scripts/miscl_scripts/run_train_models_sngp.sh', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

train_configs = {
    'calibrate': ['False'],
    'use_selective': ['False'],
    'use_spectralnorm': ['False'],
    'validation_subsample': [0.0],
    'ridge_factor': [1e-4],
    'momentum': [0.999],
}
    
cuda_devices = '[0,2]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config])<jupyter_output><empty_output><jupyter_text>SNGP HP search<jupyter_code>def generate_train_str_config(dpp_config):
    config_list = []
    config_list.append('ue\=sngp do_ue_estimate\=False')
    
    config_list.append('ue.use_selective\={}'.format(dpp_config['use_selective']))
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('ue.use_spectralnorm\={}'.format(dpp_config['use_spectralnorm']))
    config_list.append('data.validation_subsample\={}'.format(dpp_config['validation_subsample']))
    config_list.append('+data.eval_subsample\={}'.format(dpp_config['eval_subsample']))
    
    config_list.append('ue.sngp.ridge_factor\={}'.format(dpp_config['ridge_factor']))
    config_list.append('ue.sngp.momentum\={}'.format(dpp_config['momentum']))
    
    
    if dpp_config['use_selective'] == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')
    return config_list
import yaml

def generate_bash(configs, cuda_devices, tasks, generate_func):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                output_dir = f'../workdir/run_train_models/electra-calibrate/{task_name}'
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                size = 0.0 if task_name!='sst2' else 0.1
                with open(f'../workdir/sngp/{task_name}/sngp_/{size}/2021-10-11/07-01-17-0/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)
                
                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                use_spectralnorm = conf['use_spectralnorm']
                
                ridge_factor = conf['ridge_factor']
                momentum = conf['momentum']
                
                if conf['use_selective'] == 'True':
                    new_task+=f' output_dir=../workdir/run_train_models/electra-reg-sngp/{task_name}/{ridge_factor}_{momentum}_{validation_subsample}'
                else:
                    new_task+=f' output_dir=../workdir/run_train_models/electra-raw-sngp/{task_name}/{ridge_factor}_{momentum}_{validation_subsample}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open ('../scripts/miscl_scripts/run_train_models_sngp_hp_search.sh', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

train_configs = {
    'calibrate': ['False'],
    'use_selective': ['False'],
    'use_spectralnorm': ['False'],
    'validation_subsample': [0.0],
    'ridge_factor': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1],
    'momentum': [0.999, 0.99, 0.9],
    'eval_subsample': [0.2]
}
    
cuda_devices = '[1,2,3]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config])
def generate_eval_str_config(dpp_config):
    config_list = []
    config_list.append('ue\=sngp')
    config_list.append('do_ue_estimate\=True')
    
    config_list.append('ue.use_selective\={}'.format(dpp_config['use_selective']))
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('ue.use_spectralnorm\={}'.format(dpp_config['use_spectralnorm']))
    config_list.append('data.validation_subsample\={}'.format(dpp_config['validation_subsample']))
    config_list.append('+data.eval_subsample\={}'.format(dpp_config['eval_subsample']))
    
    config_list.append('ue.sngp.ridge_factor\={}'.format(dpp_config['ridge_factor']))
    config_list.append('ue.sngp.momentum\={}'.format(dpp_config['momentum']))
    
    
    if dpp_config['use_selective'] == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')
    return config_list

def generate_bash_sngp(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_glue_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                
                validation_subsample = conf['validation_subsample']
                ridge_factor = conf['ridge_factor']
                momentum = conf['momentum']
                
                if conf['use_selective'] == 'True':
                    base_path=f' output_dir=../workdir/run_glue_for_model_series/electra-reg-sngp/{task_name}/{ridge_factor}_{momentum}_{validation_subsample}'
                else:
                    base_path=f' output_dir=../workdir/run_glue_for_model_series/electra-raw-sngp/{task_name}/{ridge_factor}_{momentum}_{validation_subsample}'
                new_task+=base_path
                
                if conf['use_selective'] == 'True':
                    model_series_dir = f'../workdir/run_train_models/electra-reg-sngp/{task_name}/{ridge_factor}_{momentum}_{validation_subsample}/models/{task_name}'
                else:
                    model_series_dir = f'../workdir/run_train_models/electra-raw-sngp/{task_name}/{ridge_factor}_{momentum}_{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
        
tasks = ['mrpc', 'cola', 'sst2']

eval_configs = {
    'ue': ['sngp'],
    'calibrate': ['False'],
    'use_selective': ['False'],
    'use_spectralnorm': ['True'],
    'validation_subsample': [0.0],
    'committee_size': [10],
    'ridge_factor': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1],
    'momentum': [0.999, 0.99, 0.9],
    'eval_subsample': [0.2]
}
    
cuda_devices = '[1,2,3]'

generate_bash_sngp([eval_configs], cuda_devices, tasks, [generate_eval_str_config], file_name='run_eval_sngp_hp.sh')<jupyter_output><empty_output><jupyter_text>Generate eval script for sngp<jupyter_code>import os 

def generate_eval_sngp_str_config(mc_config):
    config_list = []
    
    config_list.append('ue\={}'.format(mc_config['ue']))
    config_list.append('do_ue_estimate\=True')
    
    config_list.append('ue.calibrate\={}'.format(mc_config['calibrate']))
    config_list.append('data.validation_subsample\={}'.format(mc_config['validation_subsample']))
    
    if mc_config['use_selective'] == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')

    config_list.append('ue.committee_size\={}'.format(mc_config['committee_size']))
    
    return config_list

def generate_bash_sngp(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_glue_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                
                validation_subsample = conf['validation_subsample']
                if conf['use_selective'] == 'True':
                    base_path=f' output_dir=../workdir/run_glue_for_model_series/electra-reg-sngp-correct-hp/{task_name}/{validation_subsample}'
                else:
                    base_path=f' output_dir=../workdir/run_glue_for_model_series/electra-raw-sngp-correct-hp/{task_name}/{validation_subsample}'
                new_task+=base_path
                
                if conf['use_selective'] == 'True':
                    model_series_dir = f'../workdir/run_train_models/electra-reg-sngp-correct-hp/{task_name}/{validation_subsample}/models/{task_name}'
                else:
                    model_series_dir = f'../workdir/run_train_models/electra-raw-sngp-correct-hp/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

eval_configs = {
    'ue': ['sngp'],
    'calibrate': ['False'],
    'use_selective': ['False'],
    'use_spectralnorm': ['True'],
    'validation_subsample': [0.0],
    'committee_size': [10]
}
    
cuda_devices = '[1,2,3]'

generate_bash_sngp([eval_configs], cuda_devices, tasks, [generate_eval_sngp_str_config], file_name='run_eval_sngp.sh')<jupyter_output><empty_output><jupyter_text>Generate train script for all models<jupyter_code>def generate_train_str_config(dpp_config):
    config_list = []
    config_list.append('ue\=mc do_ue_estimate\=False')
    
    use_selective = 'False' if dpp_config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(dpp_config['reg_type']))
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('+ue.use_spectralnorm\={}'.format(dpp_config['use_spectralnorm']))
    config_list.append('data.validation_subsample\={}'.format(dpp_config['validation_subsample']))
    
    if use_selective == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                output_dir = f'../workdir/run_train_models/electra-calibrate/{task_name}'
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                validation_subsample = conf['validation_subsample']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                with open(f'../../workdir/optimal_hyperparams/{task_name}_electra_{reg}_{use_spectralnorm}_optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                                
                try:
                    margin = params['margin']
                    lamb_intra = params['lamb_intra']
                except:
                    margin = 0
                    lamb_intra = 0
                    
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                lamb = params['lamb']
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                new_task+=f' output_dir=../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'

                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['reg-curr', 'raw'],#['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
    'validation_subsample': [0.0]
}
    
cuda_devices = '[0,1]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_models_electra.sh')<jupyter_output><empty_output><jupyter_text>Generate train script for NER<jupyter_code>def generate_train_str_config(dpp_config):
    config_list = []
    config_list.append('ue\=mc do_ue_estimate\=False')
    
    config_list.append('ue.use_selective\={}'.format(dpp_config['use_selective']))
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('+ue.use_spectralnorm\={}'.format(dpp_config['use_spectralnorm']))
    config_list.append('data.subsample_perc_val\={}'.format(dpp_config['validation_subsample']))
    if dpp_config['use_selective'] == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func):
    full_config = 'cd uncertainty-estimation/src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models_ner.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                output_dir = f'../workdir/run_train_models/electra-calibrate/{task_name}'
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf)) + ' data.subsample_perc\=0.1 ue.calibrate\=True'
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                use_spectralnorm = conf['use_spectralnorm']
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                new_task+=f' output_dir=../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'

                full_config += '\n' + new_task if len(full_config) else new_task
    with open ('../../run_train_models_ner.sh', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

train_configs = {
    'calibrate': ['True'],
    'use_selective': ['True'],
    'reg_type': ['reg-curr', 'raw'],#['metric', 'reg-curr', 'raw'],
    'use_spectralnorm': ['False', 'True'],
    'validation_subsample': [0.1]
}
    
cuda_devices = '[0,1,2,3]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config])<jupyter_output><empty_output><jupyter_text>Generate eval script for DPP and MC-Dropout<jupyter_code>import os 

def generate_eval_dpp_str_config(dpp_config):
    config_list = []
    config_list.append('ue\={}'.format(dpp_config['ue']))
    config_list.append('do_ue_estimate\=True')
    
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('data.validation_subsample\={}'.format(dpp_config['validation_subsample']))
    
    if dpp_config['use_selective'] == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')
        
    config_list.append('ue.dropout.is_reused_mask\={}'.format(dpp_config['is_reused_mask']))
    config_list.append('ue.dropout.max_frac\={}'.format(dpp_config['max_frac']))
    config_list.append('ue.dropout.committee_size\={}'.format(dpp_config['dropout_committee_size']))
    
    config_list.append('ue.committee_size\={}'.format(dpp_config['committee_size']))
    config_list.append('ue.dropout.use_ood_sampling\={}'.format(dpp_config['use_ood_sampling']))
    
    return config_list


def generate_eval_mc_str_config(mc_config):
    config_list = []
    
    config_list.append('ue\={}'.format(mc_config['ue']))
    config_list.append('do_ue_estimate\=True')
    
    config_list.append('ue.calibrate\={}'.format(mc_config['calibrate']))
    config_list.append('data.validation_subsample\={}'.format(mc_config['validation_subsample']))
    
    if mc_config['use_selective'] == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')

    config_list.append('ue.dropout_subs\={}'.format(mc_config['dropout_subs']))

    config_list.append('ue.committee_size\={}'.format(mc_config['committee_size']))
    config_list.append('ue.inference_prob\={}'.format(mc_config['inference_prob']))
    
    return config_list

def generate_bash_dpp(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd uncertainty-estimation/src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_glue_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                
                validation_subsample = conf['validation_subsample']
                if conf['use_selective'] == 'True':
                    base_path=f' output_dir=../workdir/run_glue_for_model_series/electra-reg/{task_name}/{validation_subsample}'
                else:
                    base_path=f' output_dir=../workdir/run_glue_for_model_series/electra-raw/{task_name}/{validation_subsample}'
                
                use_ood_sampling = conf['use_ood_sampling']
                max_frac = conf['max_frac']
                base_path+= f'/raw_dpp/{max_frac}'
                new_task+=base_path
                
                if conf['use_selective'] == 'True':
                    model_series_dir = f'/mnt/users/avazhentsev/uncertainty-estimation/workdir/run_train_models/electra-reg/{task_name}/{validation_subsample}/'
                else:
                    model_series_dir = f'/mnt/users/avazhentsev/uncertainty-estimation/workdir/run_train_models/electra-raw/{task_name}/{validation_subsample}/'
                model_series_dir += os.listdir(model_series_dir)[-1]
                model_series_dir += f'/{os.listdir(model_series_dir)[-1]}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../../{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

dpp_configs = {
    'ue': ['mc-dpp'],
    'max_frac' : [0.3, 0.4, 0.6],
    "committee_size": [100],
    'calibrate': ['True'],
    'use_selective': ['True', 'False'],
    'use_ood_sampling': ['False'],
    'dropout_committee_size': [20],
    'validation_subsample': [0.1, 0.0],
    'is_reused_mask': ['False'],
}
    
cuda_devices = '[2]'

generate_bash_dpp([dpp_configs], cuda_devices, tasks, [generate_eval_dpp_str_config], file_name='run_eval_raw_dpp_models.sh')
def generate_bash_mc(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd uncertainty-estimation/src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_glue_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                if conf['use_selective'] == 'True':
                    base_path=f' output_dir=../workdir/run_glue_for_model_series/electra-reg/{task_name}/0.0'
                else:
                    base_path=f' output_dir=../workdir/run_glue_for_model_series/electra-raw/{task_name}/0.0'
                
                dropout_subs = conf['dropout_subs']
                base_path+= f'/mc/{dropout_subs}'
                new_task+=base_path
                
                model_series_dir=f'../workdir/run_train_models/electra_{reg}_{use_spectralnorm}/{task_name}/0.0'

                if conf['use_selective'] == 'True':
                    model_series_dir = f'/mnt/users/avazhentsev/uncertainty-estimation/workdir/run_train_models/electra-reg/{task_name}/0.0/'
                else:
                    model_series_dir = f'/mnt/users/avazhentsev/uncertainty-estimation/workdir/run_train_models/electra-raw/{task_name}/0.0/'
                
                model_series_dir += os.listdir(model_series_dir)[-1]
                model_series_dir += f'/{os.listdir(model_series_dir)[-1]}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../../{file_name}', 'w') as rsh:
        rsh.write(full_config)
        
mc_configs = {
    'ue': ['mc'],
    'dropout_subs': ['last', 'all'],
    "committee_size": [20],
    'calibrate': ['False'],
    'inference_prob': [0.1],
    'use_selective': ['True', 'False'],
    'validation_subsample': [0.0],
}

cuda_devices = '[3]'

generate_bash_mc([mc_configs], cuda_devices, tasks, [generate_eval_mc_str_config], file_name='run_eval_mc_models.sh')<jupyter_output><empty_output><jupyter_text>Generate eval script for determenistic methods<jupyter_code>def generate_ue_eval_methods_str_config(mc_config):
    config_list = []
    
    config_list.append('ue\={}'.format(mc_config['ue']))
    config_list.append('ue.use_spectralnorm\=False')

    use_selective = 'False' if mc_config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(mc_config['reg_type']))

    config_list.append('do_ue_estimate\=True')
    config_list.append('ue.use_cache\=True')
    
    config_list.append('ue.calibrate\=False')
    config_list.append('data.validation_subsample\={}'.format(mc_config['validation_subsample']))
    
    if use_selective == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')
    
    return config_list
import os

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_glue_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'

                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                validation_subsample = conf['validation_subsample']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                base_path=f' output_dir=../workdir/run_glue_for_model_series_sn/electra_{reg}_{use_spectralnorm}/{task_name}/0.0'
                
                ue_type = conf['ue']
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                model_series_dir=f'../workdir/run_train_models_sn/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)

tasks = ['mrpc', 'cola', 'sst2']

ue_configs = {
    'ue': ['nuq'],#['mahalanobis', 'mc_mahalanobis'],
    'validation_subsample': [0.0],
    'use_spectralnorm': ['False', 'True'],
    'reg_type': ['metric', 'reg-curr', 'raw'],
}

cuda_devices = '[0,1]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_nuq.sh')
import os

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd uncertainty-estimation/src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_glue_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                use_spectralnorm = '-'+conf['use_spectralnorm'] if conf['use_spectralnorm']=='True' else ''
                if conf['use_selective'] == 'True':
                    base_path=f' output_dir=../workdir/run_glue_for_model_series/electra-reg{use_spectralnorm}/{task_name}/0.0'
                else:
                    base_path=f' output_dir=../workdir/run_glue_for_model_series/electra-raw{use_spectralnorm}/{task_name}/0.0'
                
                ue_type = conf['ue']
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                if conf['use_selective'] == 'True':
                    model_series_dir = f'/mnt/users/avazhentsev/uncertainty-estimation/workdir/run_train_models/electra-reg{use_spectralnorm}/{task_name}/0.0/'
                else:
                    model_series_dir = f'/mnt/users/avazhentsev/uncertainty-estimation/workdir/run_train_models/electra-raw{use_spectralnorm}/{task_name}/0.0/'
                
                model_series_dir += os.listdir(model_series_dir)[-1]
                model_series_dir += f'/{os.listdir(model_series_dir)[-1]}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../../{file_name}', 'w') as rsh:
        rsh.write(full_config)

tasks = ['mrpc', 'cola', 'sst2']

ue_configs = {
    'ue': ['mc_mahalanobis'],
    'use_selective': ['False', 'True'],
    'validation_subsample': [0.0],
    'use_spectralnorm': ['False', 'True'],
}

cuda_devices = '[3]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_mc_fp_models.sh')<jupyter_output><empty_output><jupyter_text>Generate eval script for det. methods for NER<jupyter_code>def generate_ue_eval_methods_str_config(mc_config):
    config_list = []
    
    config_list.append('ue\={}'.format(mc_config['ue']))
    config_list.append('ue.use_spectralnorm\={}'.format(mc_config['use_spectralnorm']))
    config_list.append('ue.use_selective\={}'.format(mc_config['use_selective']))
    config_list.append('do_ue_estimate\=True')
    config_list.append('ue.use_cache\=False')
    config_list.append('ue.calibrate\=False')
    config_list.append('data.subsample_perc_val\={}'.format(mc_config['validation_subsample']))
    
    if mc_config['use_selective'] == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')
    
    return config_list
import os

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd uncertainty-estimation/src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_conll2003_for_model_series.yaml python run_conll2003_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                use_spectralnorm = '-'+conf['use_spectralnorm']
                if conf['use_selective'] == 'True':
                    base_path=f' output_dir=../workdir/run_glue_for_model_series/electra-reg{use_spectralnorm}/{task_name}/0.1'
                else:
                    base_path=f' output_dir=../workdir/run_glue_for_model_series/electra-raw{use_spectralnorm}/{task_name}/0.1'
                
                ue_type = conf['ue']
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                if conf['use_selective'] == 'True':
                    model_series_dir = f'/mnt/users/avazhentsev/uncertainty-estimation/workdir/run_train_models/electra-reg{use_spectralnorm}/{task_name}/0.1/models/{task_name}'
                else:
                    model_series_dir = f'/mnt/users/avazhentsev/uncertainty-estimation/workdir/run_train_models/electra-raw{use_spectralnorm}/{task_name}/0.1/models/{task_name}'
            
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../../{file_name}', 'w') as rsh:
        rsh.write(full_config)

tasks = ['conll2003']

ue_configs = {
    'ue': ['mc_mahalanobis'],
    'use_selective': ['False', 'True'],
    'validation_subsample': [0.1],
    'use_spectralnorm': ['False', 'True'],
}

cuda_devices = '[0,1,2,3]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_mc_fp_models_ner.sh')<jupyter_output><empty_output><jupyter_text>Generate script for det. methods (spectral norm)<jupyter_code>import os

def generate_bash(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd uncertainty-estimation/src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_glue_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                use_spectralnorm = '-'+conf['use_spectralnorm'] if conf['use_spectralnorm']=='True' else ''
                if conf['use_selective'] == 'True':
                    base_path=f' output_dir=../workdir/run_glue_for_model_series/electra-reg{use_spectralnorm}/{task_name}/0.0'
                else:
                    base_path=f' output_dir=../workdir/run_glue_for_model_series/electra-raw{use_spectralnorm}/{task_name}/0.0'
                
                ue_type = conf['ue']
                base_path+= f'/{ue_type}'
                new_task+=base_path
                
                if conf['use_selective'] == 'True':
                    model_series_dir = f'/mnt/users/avazhentsev/uncertainty-estimation/workdir/run_train_models/electra-reg{use_spectralnorm}/{task_name}/0.0/'
                else:
                    model_series_dir = f'/mnt/users/avazhentsev/uncertainty-estimation/workdir/run_train_models/electra-raw{use_spectralnorm}/{task_name}/0.0/'
                
                model_series_dir += os.listdir(model_series_dir)[-1]
                model_series_dir += f'/{os.listdir(model_series_dir)[-1]}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../../{file_name}', 'w') as rsh:
        rsh.write(full_config)

tasks = ['mrpc', 'cola', 'sst2']

ue_configs = {
    'ue': ['mc_mahalanobis'],
    'use_selective': ['False', 'True'],
    'validation_subsample': [0.0],
    'use_spectralnorm': ['True'],
}

cuda_devices = '[2]'

generate_bash([ue_configs], cuda_devices, tasks, [generate_ue_eval_methods_str_config], file_name='run_eval_mc_fp_models_sn.sh')<jupyter_output><empty_output><jupyter_text>Generate scripts for train Deep Ensemble<jupyter_code>def generate_bash(cuda_devices, tasks):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_ensemble_series.yaml python ./run_train_ensemble_series.py'
    for task_name in tasks:
        task_configs = f'{task_name}.yaml'
        subsample_perc = 0.63 if task_name!='sst2' else 0.1
        args = f'data.subsample_perc\={subsample_perc} ue.calibrate\=True data.validation_subsample\=0.1'
        
        with open(f'../workdir/optimal_hyperparams/{task_name}_electra_raw_no_sn_optimal_hyperparameters.yaml', 'r') as f:
            params = yaml.load(f, Loader=yaml.FullLoader)

        num_train_epochs = params['num_train_epochs']
        args += f' training.num_train_epochs\={num_train_epochs}'
        learning_rate = params['learning_rate']
        args += f' training.learning_rate\={learning_rate}'
        per_device_train_batch_size = params['per_device_train_batch_size']
        args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
        weight_decay = params['weight_decay']
        args += f' +training.weight_decay\={weight_decay}'
        
        output_dir = f'../workdir/run_train_ensemble_series/{task_name}/'
        
        new_task = copy.deepcopy(base_arg)
        new_task += f" task_configs={task_configs} args='{args}' output_dir='{output_dir}' cuda_devices={cuda_devices}"
        full_config += '\n' + new_task if len(full_config) else new_task
    with open ('../scripts/miscl_scripts/run_train_ensemble_models.sh', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

cuda_devices = '[1,2,3]'

generate_bash(cuda_devices, tasks)<jupyter_output><empty_output><jupyter_text>Generate scripts for eval Deep Ensemble<jupyter_code>def generate_bash_eval_ens(cuda_devices, tasks, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_ensemble_series.yaml python run_glue_for_ensemble_series.py'
    for task_name in tasks:
        task_configs = f'{task_name}.yaml'

        new_task = copy.deepcopy(base_arg)
        new_task += f' cuda_devices={cuda_devices}'
        new_task += f' config_path=../configs/{task_configs}'

        ensemble_series_dir = f'/mnt/users/avazhentsev/uncertainty-estimation/workdir/run_train_ensemble_series/{task_name}/ensembles/'
        new_task+=f' ensemble_series_dir={ensemble_series_dir}'

        full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

cuda_devices = '1'

generate_bash_eval_ens(cuda_devices, tasks, 'run_eval_ensemble_series.sh')<jupyter_output><empty_output><jupyter_text>Generate scripts for train Deep Ensemble NER<jupyter_code>def generate_bash(cuda_devices, tasks):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_ensemble_series_ner.yaml python ./run_train_ensemble_series_ner.py'
    for task_name in tasks:
        task_configs = f'{task_name}.yaml'
        args = 'data.subsample_perc\=0.1 ue.calibrate\=True data.subsample_perc_val\=0.1 ue.use_cache\=True ue.use_selective\=False'
        
        with open(f'../workdir/optimal_hyperparams/{task_name}_electra_raw_no_sn_optimal_hyperparameters.yaml', 'r') as f:
            params = yaml.load(f, Loader=yaml.FullLoader)

        num_train_epochs = params['num_train_epochs']
        args += f' training.num_train_epochs\={num_train_epochs}'
        learning_rate = params['learning_rate']
        args += f' training.learning_rate\={learning_rate}'
        per_device_train_batch_size = params['per_device_train_batch_size']
        args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
        weight_decay = params['weight_decay']
        args += f' +training.weight_decay\={weight_decay}'
        
        output_dir = f'../workdir/run_train_ensemble_series/{task_name}/electra'
        
        new_task = copy.deepcopy(base_arg)
        new_task += f" task_configs={task_configs} args='{args}' output_dir='{output_dir}' cuda_devices={cuda_devices}"
        full_config += '\n' + new_task if len(full_config) else new_task
    with open ('../scripts/ner_scripts/run_train_ensemble_models_ner.sh', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

cuda_devices = '[1,2,3]'

generate_bash(cuda_devices, tasks)<jupyter_output><empty_output><jupyter_text>Generate scripts for eval Deep Ensemble NER<jupyter_code>def generate_bash_eval_ens(cuda_devices, tasks, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_conll2003_for_ensemble_series.yaml python run_conll2003_for_ensemble_series.py'
    for task_name in tasks:
        task_configs = f'{task_name}_selective.yaml'

        new_task = copy.deepcopy(base_arg)
        new_task += f' cuda_devices={cuda_devices}'
        new_task += f' config_path=../configs/{task_configs}'
        
        ensemble_series_dir = f'/mnt/users/avazhentsev/uncertainty-estimation/workdir/run_train_ensemble_series/{task_name}/electra/ensembles/'
        new_task+=f' ensemble_series_dir={ensemble_series_dir} '

        full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/ner_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
import os

tasks = ['conll2003']

cuda_devices = '1'

generate_bash_eval_ens(cuda_devices, tasks, 'run_eval_ensemble_series_ner.sh')
def generate_bash_eval_ens(cuda_devices, tasks, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_conll2003_for_ensemble_series.yaml python run_conll2003_for_mahalanobis_ensemble_series.py'
    for task_name in tasks:
        task_configs = f'{task_name}.yaml'

        new_task = copy.deepcopy(base_arg)
        new_task += f' cuda_devices={cuda_devices}'
        new_task += f' config_path=../configs/{task_configs}'
        
        ensemble_series_dir = f'/mnt/users/avazhentsev/uncertainty-estimation/workdir/run_train_ensemble_series/{task_name}/electra/ensembles/'
        new_task+=f' ensemble_series_dir={ensemble_series_dir} '
        
        new_task+=f' output_dir=../workdir/run_conll2003_for_mahalanobis_ensemble_series'

        full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/ner_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
        
import os

tasks = ['conll2003']

cuda_devices = '1'

generate_bash_eval_ens(cuda_devices, tasks, 'run_eval_mahalanobis_ensemble_series_ner.sh')<jupyter_output><empty_output><jupyter_text>Newsgroups ALL<jupyter_code>def generate_train_str_config(dpp_config):
    config_list = []
    config_list.append('ue\=mc do_ue_estimate\=False')
    
    use_selective = 'True' if dpp_config['reg_type'] != 'raw' else 'False'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(dpp_config['reg_type']))
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('+ue.use_spectralnorm\={}'.format(dpp_config['use_spectralnorm']))
    config_list.append('data.validation_subsample\={}'.format(dpp_config['validation_subsample']))
    
    if use_selective == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')
    return config_list


def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models_20newsgroups.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                output_dir = f'../workdir/run_train_models/electra-calibrate/{task_name}'
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                sn = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                with open(f'../../workdir/optimal_hyperparams/{task_name}_electra_{reg}_{sn}.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                args += f' +ue.margin\={margin}'
                lamb_intra = params['lamb_intra']
                args += f' +ue.lamb_intra\={lamb_intra}'
                lamb = params['lamb']
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                new_task+=f' output_dir=../workdir/run_train_models/electra_{reg}_{sn}/{task_name}/{validation_subsample}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
        
tasks = ['20newsgroups']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['raw', 'reg-curr', 'metric'],
    'use_spectralnorm': ['False', 'True'],
    'validation_subsample': [0.0]
}
    
cuda_devices = '[3,4,5]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_models_newsgroups.sh')<jupyter_output><empty_output><jupyter_text>DPP HP<jupyter_code>def generate_train_str_config(dpp_config):
    config_list = []
    config_list.append('ue\=mc do_ue_estimate\=False')
    
    use_selective = 'True' if dpp_config['reg_type'] != 'raw' else 'False'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(dpp_config['reg_type']))
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('+ue.use_spectralnorm\={}'.format(dpp_config['use_spectralnorm']))
    config_list.append('data.validation_subsample\={}'.format(dpp_config['validation_subsample']))
    config_list.append('+data.eval_subsample\={}'.format(dpp_config['eval_subsample']))
    
    if use_selective == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')
    return config_list

def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models_20newsgroups_dpp_hp.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                output_dir = f'../workdir/run_train_models/electra-calibrate/{task_name}'
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                sn = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                with open(f'../../workdir/optimal_hyperparams/{task_name}_electra_{reg}_{sn}.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                args += f' +ue.margin\={margin}'
                lamb_intra = params['lamb_intra']
                args += f' +ue.lamb_intra\={lamb_intra}'
                lamb = params['lamb']
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                new_task+=f' output_dir=../workdir/run_train_models_dpp_hp/electra_{reg}_{sn}/{task_name}/{validation_subsample}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
        
tasks = ['20newsgroups']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['raw'],
    'use_spectralnorm': ['False'],
    'validation_subsample': [0.0],
    'eval_subsample': [0.2]
}
    
cuda_devices = '[2,7]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_models_dpp_hp.sh')
import os 

def generate_eval_dpp_str_config(dpp_config):
    config_list = []
    config_list.append('ue\={}'.format(dpp_config['ue']))
    config_list.append('do_ue_estimate\=True')
    
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('data.validation_subsample\={}'.format(dpp_config['validation_subsample']))
    config_list.append('+data.eval_subsample\={}'.format(dpp_config['eval_subsample']))

    use_selective = 'True' if dpp_config['reg_type'] != 'raw' else 'False'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(dpp_config['reg_type']))
    
    if use_selective == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')
        
    config_list.append('ue.dropout.is_reused_mask\={}'.format(dpp_config['is_reused_mask']))
    config_list.append('ue.dropout.max_frac\={}'.format(dpp_config['max_frac']))
    config_list.append('ue.dropout.committee_size\={}'.format(dpp_config['dropout_committee_size']))
    
    config_list.append('ue.committee_size\={}'.format(dpp_config['committee_size']))
    config_list.append('ue.dropout.use_ood_sampling\={}'.format(dpp_config['use_ood_sampling']))
    
    return config_list


def generate_bash_dpp(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_20newsgroups_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                
                validation_subsample = conf['validation_subsample']
                base_path=f' output_dir=../workdir/run_glue_for_model_series/electra_raw_no_sn/{task_name}/{validation_subsample}'
                
                use_ood_sampling = conf['use_ood_sampling']
                dpp_type = 'ood' if use_ood_sampling=='True' else 'dpp'
                max_frac = conf['max_frac']
                dropout_committee_size = conf['dropout_committee_size']
                base_path+= f'/ddpp_{dpp_type}_{max_frac}_{dropout_committee_size}/'
                new_task+=base_path
                
                model_series_dir = f'/notebook/uncertainty-estimation/workdir/run_train_models_dpp_hp/electra_raw_no_sn/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['20newsgroups']

dpp_configs = {
    'ue': ['mc-dpp'],
    'max_frac' : [0.3, 0.4, 0.5, 0.6],
    "committee_size": [100],
    'calibrate': ['True'],
    'reg_type': ['raw'],
    'use_ood_sampling': ['False', 'True'],
    'dropout_committee_size': [50],
    'validation_subsample': [0.0],
    'is_reused_mask': ['True'],
    'eval_subsample': [0.2]
}
    
cuda_devices = '[2,7]'

generate_bash_dpp([dpp_configs], cuda_devices, tasks, [generate_eval_dpp_str_config], file_name='run_eval_dpp_models_hp_search_20ng.sh')<jupyter_output><empty_output><jupyter_text>Full<jupyter_code>import os 

def generate_eval_dpp_str_config(dpp_config):
    config_list = []
    config_list.append('ue\={}'.format(dpp_config['ue']))
    config_list.append('do_ue_estimate\=True')
    
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('data.validation_subsample\={}'.format(dpp_config['validation_subsample']))
    
    use_selective = 'True' if dpp_config['reg_type'] != 'raw' else 'False'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(dpp_config['reg_type']))
    
    if use_selective == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')
        
    config_list.append('ue.dropout.is_reused_mask\={}'.format(dpp_config['is_reused_mask']))
    config_list.append('ue.dropout.max_frac\={}'.format(dpp_config['max_frac']))
    config_list.append('ue.dropout.committee_size\={}'.format(dpp_config['dropout_committee_size']))
    
    config_list.append('ue.committee_size\={}'.format(dpp_config['committee_size']))
    config_list.append('ue.dropout.use_ood_sampling\={}'.format(dpp_config['use_ood_sampling']))
    config_list.append('training.per_device_eval_batch_size\=16')
    
    return config_list


def generate_eval_mc_str_config(mc_config):
    config_list = []
    
    config_list.append('ue\={}'.format(mc_config['ue']))
    config_list.append('do_ue_estimate\=True')
    
    config_list.append('ue.calibrate\={}'.format(mc_config['calibrate']))
    config_list.append('data.validation_subsample\={}'.format(mc_config['validation_subsample']))
    
    use_selective = 'True' if mc_config['reg_type'] != 'raw' else 'False'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(mc_config['reg_type']))
    
    if use_selective == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')

    config_list.append('ue.dropout_subs\={}'.format(mc_config['dropout_subs']))

    config_list.append('ue.committee_size\={}'.format(mc_config['committee_size']))
    config_list.append('training.per_device_eval_batch_size\=16')
    #config_list.append('ue.inference_prob\={}'.format(mc_config['inference_prob']))
    
    return config_list

def generate_bash_dpp(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_20newsgroups_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                sn = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                with open(f'../../workdir/optimal_hyperparams/{task_name}_electra_{reg}_{sn}.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                args += f' +ue.margin\={margin}'
                lamb_intra = params['lamb_intra']
                args += f' +ue.lamb_intra\={lamb_intra}'
                lamb = params['lamb']
                args += f' ue.lamb\={lamb}'
                
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                
                validation_subsample = conf['validation_subsample']
                base_path=f' output_dir=../workdir/run_glue_for_model_series/electra_{reg}_{sn}/{task_name}/{validation_subsample}'
                
                use_ood_sampling = conf['use_ood_sampling']
                dpp_type = 'ood' if use_ood_sampling=='True' else 'dpp'
                max_frac = conf['max_frac']
                dropout_committee_size = conf['dropout_committee_size']
                base_path+= f'/ddpp_{dpp_type}_best/'#_{max_frac}_{dropout_committee_size}/'
                new_task+=base_path
                
                model_series_dir = f'../workdir/run_train_models/electra_{reg}_{sn}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['20newsgroups']

dpp_configs = {
    'ue': ['mc-dpp'],
    'max_frac' : [0.6], # best for both ddpp
    "committee_size": [100],
    'calibrate': ['True'],
    'use_spectralnorm': ['False'],
    'reg_type': ['raw', 'reg-curr', 'metric'],
    'use_ood_sampling': ['False', 'True'],
    'dropout_committee_size': [50],
    'validation_subsample': [0.0],
    'is_reused_mask': ['True'],
}
    
cuda_devices = '[4,5,6]'

generate_bash_dpp([dpp_configs], cuda_devices, tasks, [generate_eval_dpp_str_config], file_name='run_eval_20ng_dpp_models.sh')
def generate_bash_mc(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_20newsgroups_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                sn = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                with open(f'../../workdir/optimal_hyperparams/{task_name}_electra_{reg}_{sn}.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                args += f' +ue.margin\={margin}'
                lamb_intra = params['lamb_intra']
                args += f' +ue.lamb_intra\={lamb_intra}'
                lamb = params['lamb']
                args += f' ue.lamb\={lamb}'
                
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                
                validation_subsample = conf['validation_subsample']
                base_path=f' output_dir=../workdir/run_glue_for_model_series/electra_{reg}_{sn}/{task_name}/{validation_subsample}'
                base_path+= f'/mc_all/'
                new_task+=base_path
                
                model_series_dir = f'../workdir/run_train_models/electra_{reg}_{sn}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
        
tasks = ['20newsgroups']

dpp_configs = {
    'ue': ['mc'],
    "committee_size": [20],
    'calibrate': ['False'],
    'use_spectralnorm': ['False'],
    'reg_type': ['raw', 'reg-curr', 'metric'],
    'validation_subsample': [0.0],
    'dropout_subs': ['all']
}   
    
cuda_devices = '[4,5,6]'

generate_bash_mc([dpp_configs], cuda_devices, tasks, [generate_eval_mc_str_config], file_name='run_eval_20ng_mc_models.sh')
import os

def generate_bash_det(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_glue_for_model_series.yaml python run_20newsgroups_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                sn = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                with open(f'../../workdir/optimal_hyperparams/{task_name}_electra_{reg}_{sn}.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                args += f' +ue.margin\={margin}'
                lamb_intra = params['lamb_intra']
                args += f' +ue.lamb_intra\={lamb_intra}'
                lamb = params['lamb']
                args += f' ue.lamb\={lamb}'
                
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                
                validation_subsample = conf['validation_subsample']
                ue = conf['ue']
                base_path=f' output_dir=../workdir/run_glue_for_model_series/electra_{reg}_{sn}/{task_name}/{validation_subsample}'
                base_path+= f'/{ue}/'
                new_task+=base_path
                
                model_series_dir = f'../workdir/run_train_models/electra_{reg}_{sn}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)

        
def generate_eval_ue_str_config(mc_config):
    config_list = []
    
    config_list.append('ue\={}'.format(mc_config['ue']))
    config_list.append('do_ue_estimate\=True')
    
    config_list.append('data.validation_subsample\={}'.format(mc_config['validation_subsample']))
    
    use_selective = 'True' if mc_config['reg_type'] != 'raw' else 'False'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('ue.reg_type\={}'.format(mc_config['reg_type']))
    config_list.append('ue.use_spectralnorm\=False')#.format(mc_config['use_spectralnorm']))
    
    if use_selective == 'True':
        config_list.append('training\=electra_base')
    else:
        config_list.append('training\=electra_base')
    config_list.append('training.per_device_eval_batch_size\=16')

    return config_list

tasks = ['20newsgroups']

ue_configs = {
    'ue': ['mahalanobis'],
    'validation_subsample': [0.0],
    'use_spectralnorm': ['False', 'True'],
    'reg_type': ['raw', 'reg-curr', 'metric'],
}

cuda_devices = '[4,5,6]'

generate_bash_det([ue_configs], cuda_devices, tasks, [generate_eval_ue_str_config], file_name='run_eval_20ng_det_models.sh')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1443416452591119, 'fraction_numerical': 0.005797745285509178, 'mean_word_length': 2.749459860710449, 'pattern_counts': {'":': 10, '<': 241, '<?xml version=': 0, '>': 241, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '60', 'dir': 'jupyter-structured-clean-dedup', 'id': '517922', 'license': 'MIT', 'n_tokens_mistral': 106444, 'n_tokens_neox': 102242, 'n_words': 13893, 'path': 'src/exps_notebooks/generate_series_of_exps.ipynb', 'repo_name': 'AIRI-Institute/uncertainty_transformers', 'size': '310638'}