<jupyter_start><jupyter_text>Electra train all GLUE for DPP HP<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    config_list.append('+ue.use_spectralnorm\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('+data.eval_subsample\={}'.format(config['eval_subsample']))
    
    #config_list.append("model.model_name_or_path\=microsoft/deberta-base")
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                new_task += f' script=run_glue_dpp_hp.py'
                
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                use_spectralnorm_params = '_sn' if conf['use_spectralnorm'] == 'True' else ''
                reg_params = 'cer' if conf['reg_type'] == 'reg-curr' else conf['reg_type']

                with open(f'../../optimal_hyperparameters/electra/{reg_params}{use_spectralnorm_params}_{task_name}_optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                new_task+=f' output_dir=../workdir/run_train_models_dpp_hp/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['raw'],
    'use_spectralnorm': ['False'],
    'validation_subsample': [0.0],
    'eval_subsample': [0.2],
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_glue_dpp_hp.sh')
import os 

def generate_eval_dpp_str_config(dpp_config):
    config_list = []
    config_list.append('ue\={}'.format(dpp_config['ue']))
    config_list.append('do_ue_estimate\=True')
    config_list.append('+ue.use_spectralnorm\=False')
    
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('data.validation_subsample\={}'.format(dpp_config['validation_subsample']))
    config_list.append('+data.eval_subsample\={}'.format(dpp_config['eval_subsample']))
    
    use_selective = 'False' if dpp_config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('training\=electra_base')
        
    config_list.append('ue.dropout.is_reused_mask\={}'.format(dpp_config['is_reused_mask']))
    config_list.append('ue.dropout.max_frac\={}'.format(dpp_config['max_frac']))
    config_list.append('ue.dropout.committee_size\={}'.format(dpp_config['dropout_committee_size']))
    config_list.append('ue.dropout.mask_name_for_mask\={}'.format(dpp_config['mask_name_for_mask']))
    
    config_list.append('ue.committee_size\={}'.format(dpp_config['committee_size']))
    config_list.append('ue.dropout.use_ood_sampling\={}'.format(dpp_config['use_ood_sampling']))
    
    return config_list

def generate_bash_dpp(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                new_task += f' script=run_glue_dpp_hp.py'
                
                validation_subsample = conf['validation_subsample']
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'no_sn'
                
                base_path=f' output_dir=../workdir/run_tasks_for_model_series_dpp_hp/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                use_ood_sampling = conf['use_ood_sampling']
                dpp_type = 'ood' if use_ood_sampling=='True' else 'dpp'
                max_frac = conf['max_frac']
                dropout_committee_size = conf['dropout_committee_size']
                mask_name_for_mask = conf['mask_name_for_mask']
                base_path+= f'/ddpp_{dpp_type}_{mask_name_for_mask}_{max_frac}_{dropout_committee_size}/'
                new_task+=base_path
                
                model_series_dir = f'../workdir/run_train_models_dpp_hp/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

dpp_configs = {
    'ue': ['mc-dpp'],
    'max_frac' : [0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6],
    'mask_name_for_mask': ['rbf'],
    "committee_size": [100],
    'calibrate': ['True'],
    'reg_type': ['raw'],
    'use_ood_sampling': ['False', 'True'],
    'dropout_committee_size': [50],
    'validation_subsample': [0.0],
    'is_reused_mask': ['True'],
    'eval_subsample': [0.2]
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash_dpp([dpp_configs], cuda_devices, tasks, [generate_eval_dpp_str_config], file_name='run_eval_electra_models_glue_dpp_hp.sh')
full_test = 'bash run_train_electra_models_glue_dpp_hp.sh\n'
full_test += 'bash run_eval_electra_models_glue_dpp_hp.sh'

with open (f'../../scripts/miscl_scripts/run_electra_ddpp_hp.sh', 'w') as rsh:
    rsh.write(full_test)<jupyter_output><empty_output><jupyter_text>DPP HP CoNLL-2003<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    config_list.append('+ue.use_spectralnorm\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    config_list.append('data.subsample_perc_val\={}'.format(config['validation_subsample']))
    #config_list.append('+data.eval_subsample\={}'.format(config['eval_subsample']))
    
    #config_list.append("model.model_name_or_path\=microsoft/deberta-base")
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                new_task += f' script=run_conll2003_dpp_hp.py'
                
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
                
                use_spectralnorm_params = '_sn' if conf['use_spectralnorm'] == 'True' else ''
                reg_params = 'cer' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                
                with open(f'../../optimal_hyperparameters/electra/{reg_params}{use_spectralnorm_params}_{task_name}_optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                new_task+=f' output_dir=../workdir/run_train_models_dpp_hp/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../../scripts/ner_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['raw'],
    'use_spectralnorm': ['False'],
    'validation_subsample': [0.1],
    'subsample_perc': [0.1],
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_electra_models_dpp_hp.sh')
import os 

def generate_eval_dpp_str_config(dpp_config):
    config_list = []
    config_list.append('ue\={}'.format(dpp_config['ue']))
    config_list.append('do_ue_estimate\=True')
    config_list.append('+ue.use_spectralnorm\=False')
    
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('data.subsample_perc_val\={}'.format(dpp_config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(dpp_config['subsample_perc']))
    
    use_selective = 'False' if dpp_config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('training\=electra_base')
        
    config_list.append('ue.dropout.is_reused_mask\={}'.format(dpp_config['is_reused_mask']))
    config_list.append('ue.dropout.max_frac\={}'.format(dpp_config['max_frac']))
    config_list.append('ue.dropout.committee_size\={}'.format(dpp_config['dropout_committee_size']))
    config_list.append('ue.dropout.mask_name_for_mask\={}'.format(dpp_config['mask_name_for_mask']))
    
    config_list.append('ue.committee_size\={}'.format(dpp_config['committee_size']))
    config_list.append('ue.dropout.use_ood_sampling\={}'.format(dpp_config['use_ood_sampling']))
    
    return config_list


def generate_bash_dpp(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                new_task += f' script=run_conll2003_dpp_hp.py'
                
                validation_subsample = conf['validation_subsample']
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'no_sn'
                
                base_path=f' output_dir=../workdir/run_tasks_for_model_series_dpp_hp/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                use_ood_sampling = conf['use_ood_sampling']
                dpp_type = 'ood' if use_ood_sampling=='True' else 'dpp'
                max_frac = conf['max_frac']
                dropout_committee_size = conf['dropout_committee_size']
                mask_name_for_mask = conf['mask_name_for_mask']
                base_path+= f'/ddpp_{dpp_type}_{mask_name_for_mask}_{max_frac}_{dropout_committee_size}/'
                new_task+=base_path
                
                model_series_dir = f'../workdir/run_train_models_dpp_hp/electra_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../../scripts/ner_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

dpp_configs = {
    'ue': ['mc-dpp'],
    'max_frac' : [0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6],
    'mask_name_for_mask': ['rbf'],
    "committee_size": [100],
    'calibrate': ['True'],
    'reg_type': ['raw'],
    'use_ood_sampling': ['False', 'True'],
    'dropout_committee_size': [20],
    'validation_subsample': [0.1],
    'is_reused_mask': ['True'],
    'subsample_perc': [0.1]
}

cuda_devices = '[0,1,2,3,4,5]'

generate_bash_dpp([dpp_configs], cuda_devices, tasks, [generate_eval_dpp_str_config], file_name='run_eval_electra_models_dpp_hp.sh')
full_test = 'bash run_train_electra_models_dpp_hp.sh\n'
full_test += 'bash run_eval_electra_models_dpp_hp.sh'

with open (f'../../scripts/ner_scripts/run_electra_ddpp_hp.sh', 'w') as rsh:
    rsh.write(full_test)<jupyter_output><empty_output><jupyter_text>DEBERTA<jupyter_code>def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    config_list.append('+ue.use_spectralnorm\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('data.validation_subsample\={}'.format(config['validation_subsample']))
    config_list.append('+data.eval_subsample\={}'.format(config['eval_subsample']))
    
    config_list.append("model.model_name_or_path\=microsoft/deberta-base")
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                new_task += f' script=run_glue_dpp_hp.py'
                
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'

                with open(f'../../optimal_hyperparameters/deberta_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                new_task+=f' output_dir=../workdir/run_train_models_dpp_hp/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../../scripts/miscl_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['raw'],
    'use_spectralnorm': ['False'],
    'validation_subsample': [0.0],
    'eval_subsample': [0.2],
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_deberta_models_glue_dpp_hp.sh')
import os 

def generate_eval_dpp_str_config(dpp_config):
    config_list = []
    config_list.append('ue\={}'.format(dpp_config['ue']))
    config_list.append('do_ue_estimate\=True')
    config_list.append('+ue.use_spectralnorm\=False')
    
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('data.validation_subsample\={}'.format(dpp_config['validation_subsample']))
    config_list.append('+data.eval_subsample\={}'.format(dpp_config['eval_subsample']))
    
    use_selective = 'False' if dpp_config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('training\=electra_base')
        
    config_list.append('ue.dropout.is_reused_mask\={}'.format(dpp_config['is_reused_mask']))
    config_list.append('ue.dropout.max_frac\={}'.format(dpp_config['max_frac']))
    config_list.append('ue.dropout.committee_size\={}'.format(dpp_config['dropout_committee_size']))
    config_list.append('ue.dropout.mask_name_for_mask\={}'.format(dpp_config['mask_name_for_mask']))
    
    config_list.append('ue.committee_size\={}'.format(dpp_config['committee_size']))
    config_list.append('ue.dropout.use_ood_sampling\={}'.format(dpp_config['use_ood_sampling']))
    
    return config_list

def generate_bash_dpp(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                new_task += f' script=run_glue_dpp_hp.py'
                
                validation_subsample = conf['validation_subsample']
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'no_sn'
                
                base_path=f' output_dir=../workdir/run_tasks_for_model_series_dpp_hp/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                use_ood_sampling = conf['use_ood_sampling']
                dpp_type = 'ood' if use_ood_sampling=='True' else 'dpp'
                max_frac = conf['max_frac']
                dropout_committee_size = conf['dropout_committee_size']
                mask_name_for_mask = conf['mask_name_for_mask']
                base_path+= f'/ddpp_{dpp_type}_{mask_name_for_mask}_{max_frac}_{dropout_committee_size}/'
                new_task+=base_path
                
                model_series_dir = f'../workdir/run_train_models_dpp_hp/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../../scripts/miscl_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['mrpc', 'cola', 'sst2']

dpp_configs = {
    'ue': ['mc-dpp'],
    'max_frac' : [0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6],
    'mask_name_for_mask': ['rbf'],
    "committee_size": [100],
    'calibrate': ['True'],
    'reg_type': ['raw'],
    'use_ood_sampling': ['False', 'True'],
    'dropout_committee_size': [50],
    'validation_subsample': [0.0],
    'is_reused_mask': ['True'],
    'eval_subsample': [0.2]
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash_dpp([dpp_configs], cuda_devices, tasks, [generate_eval_dpp_str_config], file_name='run_eval_deberta_models_glue_dpp_hp.sh')
full_test = 'bash run_train_deberta_models_glue_dpp_hp.sh\n'
full_test += 'bash run_eval_deberta_models_glue_dpp_hp.sh'

with open (f'../../scripts/miscl_scripts/run_deberta_ddpp_hp.sh', 'w') as rsh:
    rsh.write(full_test)
def generate_train_str_config(config):
    config_list = []
    config_list.append('ue\=mc')
    config_list.append('do_ue_estimate\=False')
    config_list.append('+ue.use_spectralnorm\=False')
    
    use_selective = 'False' if config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    
    config_list.append('ue.calibrate\={}'.format(config['calibrate']))
    config_list.append('ue.reg_type\={}'.format(config['reg_type']))
    config_list.append('data.subsample_perc\={}'.format(config['subsample_perc']))
    config_list.append('data.subsample_perc_val\={}'.format(config['validation_subsample']))
    #config_list.append('+data.eval_subsample\={}'.format(config['eval_subsample']))
    
    config_list.append("model.model_name_or_path\=microsoft/deberta-base")
    config_list.append('training\=electra_base')
    return config_list
def generate_bash(configs, cuda_devices, tasks, generate_func, filename=''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_train_models.yaml python ./run_train_models.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                new_task += f' script=run_conll2003_dpp_hp.py'
                
                args = ' '.join(generate_func[i](conf))
                
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'sn' if conf['use_spectralnorm'] == 'True' else 'no_sn'
            
                with open(f'../../optimal_hyperparameters/deberta_{reg}_{use_spectralnorm}/{task_name}/optimal_hyperparameters.yaml', 'r') as f:
                    params = yaml.load(f, Loader=yaml.FullLoader)

                num_train_epochs = params['num_train_epochs']
                args += f' training.num_train_epochs\={num_train_epochs}'
                learning_rate = params['learning_rate']
                args += f' training.learning_rate\={learning_rate}'
                per_device_train_batch_size = params['per_device_train_batch_size']
                args += f' training.per_device_train_batch_size\={per_device_train_batch_size}'
                weight_decay = params['weight_decay']
                args += f' +training.weight_decay\={weight_decay}'
                
                margin = params['margin']
                lamb_intra = params['lamb_intra']
                lamb = params['lamb']
                args += f' +ue.margin\={margin}'
                args += f' +ue.lamb_intra\={lamb_intra}'
                args += f' ue.lamb\={lamb}'
        
                new_task += f" args='{args}'"
                new_task += f' task_configs={task_configs}'
                
                validation_subsample = conf['validation_subsample']
                new_task+=f' output_dir=../workdir/run_train_models_dpp_hp/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                full_config += '\n' + new_task if len(full_config) else new_task
    with open (f'../../scripts/ner_scripts/{filename}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

train_configs = {
    'calibrate': ['False'],
    'reg_type': ['raw'],
    'use_spectralnorm': ['False'],
    'validation_subsample': [0.1],
    'subsample_perc': [0.1],
}
    
cuda_devices = '[0,1,2,3,4,5]'

generate_bash([train_configs], cuda_devices, tasks, [generate_train_str_config], 'run_train_deberta_models_dpp_hp.sh')
import os 

def generate_eval_dpp_str_config(dpp_config):
    config_list = []
    config_list.append('ue\={}'.format(dpp_config['ue']))
    config_list.append('do_ue_estimate\=True')
    config_list.append('+ue.use_spectralnorm\=False')
    
    config_list.append('ue.calibrate\={}'.format(dpp_config['calibrate']))
    config_list.append('data.subsample_perc_val\={}'.format(dpp_config['validation_subsample']))
    config_list.append('data.subsample_perc\={}'.format(dpp_config['subsample_perc']))
    
    use_selective = 'False' if dpp_config['reg_type']=='raw' else 'True'
    config_list.append('ue.use_selective\={}'.format(use_selective))
    config_list.append('training\=electra_base')
        
    config_list.append('ue.dropout.is_reused_mask\={}'.format(dpp_config['is_reused_mask']))
    config_list.append('ue.dropout.max_frac\={}'.format(dpp_config['max_frac']))
    config_list.append('ue.dropout.committee_size\={}'.format(dpp_config['dropout_committee_size']))
    config_list.append('ue.dropout.mask_name_for_mask\={}'.format(dpp_config['mask_name_for_mask']))
    
    config_list.append('ue.committee_size\={}'.format(dpp_config['committee_size']))
    config_list.append('ue.dropout.use_ood_sampling\={}'.format(dpp_config['use_ood_sampling']))
    
    return config_list


def generate_bash_dpp(configs, cuda_devices, tasks, generate_func, file_name = ''):
    full_config = 'cd ../../src'
    base_arg = 'HYDRA_CONFIG_PATH=../configs/run_tasks_for_model_series.yaml python run_tasks_for_model_series.py'
    for task_name in tasks:
        for i, mc_configs in enumerate(configs):
            for conf in expand_config(mc_configs):
                task_configs = f'{task_name}.yaml'

                new_task = copy.deepcopy(base_arg)
                new_task += f' cuda_devices={cuda_devices}'
                args = ' '.join(generate_func[i](conf))
                new_task += f" args='{args}'"
                new_task += f' config_path=../configs/{task_configs}'
                new_task += f' script=run_conll2003_dpp_hp.py'
                
                validation_subsample = conf['validation_subsample']
                reg = 'reg' if conf['reg_type'] == 'reg-curr' else conf['reg_type']
                use_spectralnorm = 'no_sn'
                
                base_path=f' output_dir=../workdir/run_tasks_for_model_series_dpp_hp/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}'
                
                use_ood_sampling = conf['use_ood_sampling']
                dpp_type = 'ood' if use_ood_sampling=='True' else 'dpp'
                max_frac = conf['max_frac']
                dropout_committee_size = conf['dropout_committee_size']
                mask_name_for_mask = conf['mask_name_for_mask']
                base_path+= f'/ddpp_{dpp_type}_{mask_name_for_mask}_{max_frac}_{dropout_committee_size}/'
                new_task+=base_path
                
                model_series_dir = f'../workdir/run_train_models_dpp_hp/deberta_{reg}_{use_spectralnorm}/{task_name}/{validation_subsample}/models/{task_name}'
                new_task+=f' model_series_dir={model_series_dir}'

                full_config += '\n' + new_task if len(full_config) else new_task
            
    with open (f'../../scripts/ner_scripts/{file_name}', 'w') as rsh:
        rsh.write(full_config)
tasks = ['conll2003']

dpp_configs = {
    'ue': ['mc-dpp'],
    'max_frac' : [0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6],
    'mask_name_for_mask': ['rbf'],
    "committee_size": [100],
    'calibrate': ['True'],
    'reg_type': ['raw'],
    'use_ood_sampling': ['False', 'True'],
    'dropout_committee_size': [20],
    'validation_subsample': [0.1],
    'is_reused_mask': ['True'],
    'subsample_perc': [0.1]
}

cuda_devices = '[0,1,2,3,4,5]'

generate_bash_dpp([dpp_configs], cuda_devices, tasks, [generate_eval_dpp_str_config], file_name='run_eval_deberta_models_dpp_hp.sh')
full_test = 'bash run_train_deberta_models_dpp_hp.sh\n'
full_test += 'bash run_eval_deberta_models_dpp_hp.sh'

with open (f'../../scripts/ner_scripts/run_deberta_ddpp_hp.sh', 'w') as rsh:
    rsh.write(full_test)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14691052582130218, 'fraction_numerical': 0.006514045911496665, 'mean_word_length': 3.0191315292636878, 'pattern_counts': {'":': 4, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '267189', 'license': 'MIT', 'n_tokens_mistral': 11173, 'n_tokens_neox': 10726, 'n_words': 1366, 'path': 'src/exps_notebooks/generate_series_of_exps_dpp_hp.ipynb', 'repo_name': 'AIRI-Institute/uncertainty_transformers', 'size': '31931'}