Muennighoff's picture
Add files
4e82575
raw
history blame
127 kB
{"amazon_reviews_multi_en": {"prompt_body_title_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "en", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.5044}, "template_name": "prompt_body_title_to_star"}, "prompt_review_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "en", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.4652}, "template_name": "prompt_review_to_star"}, "prompt_title_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "en", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.3226}, "template_name": "prompt_title_to_star"}}, "amazon_reviews_multi_es": {"prompt_body_title_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "es", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.391}, "template_name": "prompt_body_title_to_star"}, "prompt_review_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "es", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.376}, "template_name": "prompt_review_to_star"}, "prompt_title_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "es", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.2564}, "template_name": "prompt_title_to_star"}}, "amazon_reviews_multi_fr": {"prompt_body_title_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "fr", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.4116}, "template_name": "prompt_body_title_to_star"}, "prompt_review_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "fr", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.3878}, "template_name": "prompt_review_to_star"}, "prompt_title_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "fr", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.274}, "template_name": "prompt_title_to_star"}}, "amazon_reviews_multi_zh": {"prompt_body_title_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "zh", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.3948}, "template_name": "prompt_body_title_to_star"}, "prompt_review_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "zh", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.3754}, "template_name": "prompt_review_to_star"}, "prompt_title_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "zh", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.2858}, "template_name": "prompt_title_to_star"}}, "aqua_rat_raw": {"Answer questions from options": {"arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Answer questions from options', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "raw", "dataset_name": "aqua_rat", "evaluation": {"accuracy": 0.20078740157480315}, "template_name": "Answer questions from options"}, "answer_quiz": {"arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='answer_quiz', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "raw", "dataset_name": "aqua_rat", "evaluation": {"accuracy": 0.19291338582677164}, "template_name": "answer_quiz"}, "select_the_best_option": {"arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='select_the_best_option', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "raw", "dataset_name": "aqua_rat", "evaluation": {"accuracy": 0.16535433070866143}, "template_name": "select_the_best_option"}}, "art_None": {"choose_hypothesis": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "art", "evaluation": {"accuracy": 0.4934725848563969}, "template_name": "choose_hypothesis"}, "choose_hypothesis_believable": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_believable', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "art", "evaluation": {"accuracy": 0.5163185378590078}, "template_name": "choose_hypothesis_believable"}, "choose_hypothesis_desc": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_desc', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "art", "evaluation": {"accuracy": 0.512402088772846}, "template_name": "choose_hypothesis_desc"}, "choose_hypothesis_likely": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_likely', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "art", "evaluation": {"accuracy": 0.5150130548302873}, "template_name": "choose_hypothesis_likely"}, "choose_hypothesis_options": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_options', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "art", "evaluation": {"accuracy": 0.5267624020887729}, "template_name": "choose_hypothesis_options"}}, "banking77_None": {"direct_to_which_department": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='direct_to_which_department', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "banking77", "evaluation": {"accuracy": 0.13214285714285715}, "template_name": "direct_to_which_department"}, "help_page_topic": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='help_page_topic', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "banking77", "evaluation": {"accuracy": 0.14383116883116884}, "template_name": "help_page_topic"}, "rephrase_as_banking_term": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='rephrase_as_banking_term', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "banking77", "evaluation": {"accuracy": 0.16103896103896104}, "template_name": "rephrase_as_banking_term"}}, "blbooksgenre_title_genre_classifiction": {"classify": {"arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='classify', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "title_genre_classifiction", "dataset_name": "blbooksgenre", "evaluation": {"accuracy": 0.2534562211981567}, "template_name": "classify"}, "multi-choice": {"arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='multi-choice', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "title_genre_classifiction", "dataset_name": "blbooksgenre", "evaluation": {"accuracy": 0.4147465437788018}, "template_name": "multi-choice"}, "premise_context_first": {"arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='premise_context_first', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "title_genre_classifiction", "dataset_name": "blbooksgenre", "evaluation": {"accuracy": 0.7430875576036866}, "template_name": "premise_context_first"}}, "blimp_adjunct_island": {"grammatical_between_1_2": {"arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_between_1_2', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "adjunct_island", "dataset_name": "blimp", "evaluation": {"accuracy": 0.501}, "template_name": "grammatical_between_1_2"}, "grammatical_between_A_B": {"arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_between_A_B', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "adjunct_island", "dataset_name": "blimp", "evaluation": {"accuracy": 0.771}, "template_name": "grammatical_between_A_B"}, "grammatical_which_one_1_2": {"arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_which_one_1_2', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "adjunct_island", "dataset_name": "blimp", "evaluation": {"accuracy": 0.516}, "template_name": "grammatical_which_one_1_2"}, "single_sentence_bad_yes_no": {"arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='single_sentence_bad_yes_no', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "adjunct_island", "dataset_name": "blimp", "evaluation": {"accuracy": 0.496}, "template_name": "single_sentence_bad_yes_no"}, "single_sentence_good_yes_no": {"arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='single_sentence_good_yes_no', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "adjunct_island", "dataset_name": "blimp", "evaluation": {"accuracy": 0.524}, "template_name": "single_sentence_good_yes_no"}}, "climate_fever_None": {"claim_and_all_supporting_evidences": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='claim_and_all_supporting_evidences', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "climate_fever", "evaluation": {"accuracy": 0.31791530944625407}, "template_name": "claim_and_all_supporting_evidences"}, "fifth_evidence_and_claim_itemization": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='fifth_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "climate_fever", "evaluation": {"accuracy": 0.24104234527687296}, "template_name": "fifth_evidence_and_claim_itemization"}, "first_evidence_and_claim_itemization": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='first_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "climate_fever", "evaluation": {"accuracy": 0.5570032573289903}, "template_name": "first_evidence_and_claim_itemization"}, "second_evidence_and_claim_itemization": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='second_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "climate_fever", "evaluation": {"accuracy": 0.45342019543973944}, "template_name": "second_evidence_and_claim_itemization"}, "third_evidence_claim_pair": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='third_evidence_claim_pair', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "climate_fever", "evaluation": {"accuracy": 0.6397394136807818}, "template_name": "third_evidence_claim_pair"}}, "codah_codah": {"affirmative_instruction_after_sentence_and_choices": {"arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='affirmative_instruction_after_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "codah", "dataset_name": "codah", "evaluation": {"accuracy": 0.24927953890489912}, "template_name": "affirmative_instruction_after_sentence_and_choices"}, "affirmative_instruction_before_sentence_and_choices": {"arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='affirmative_instruction_before_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "codah", "dataset_name": "codah", "evaluation": {"accuracy": 0.24855907780979827}, "template_name": "affirmative_instruction_before_sentence_and_choices"}, "interrogative_instruction_after_sentence_and_choices": {"arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='interrogative_instruction_after_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "codah", "dataset_name": "codah", "evaluation": {"accuracy": 0.2503602305475504}, "template_name": "interrogative_instruction_after_sentence_and_choices"}}, "commonsense_qa_None": {"answer_given_question_without_options": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='answer_given_question_without_options', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "commonsense_qa", "evaluation": {"accuracy": 0.3931203931203931}, "template_name": "answer_given_question_without_options"}, "most_suitable_answer": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='most_suitable_answer', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "commonsense_qa", "evaluation": {"accuracy": 0.4398034398034398}, "template_name": "most_suitable_answer"}, "question_answering": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='question_answering', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "commonsense_qa", "evaluation": {"accuracy": 0.44471744471744473}, "template_name": "question_answering"}}, "conv_ai_3_None": {"ambiguous": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='ambiguous', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "conv_ai_3", "evaluation": {"accuracy": 0.39040207522697795}, "template_name": "ambiguous"}, "clarification_needed": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='clarification_needed', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "conv_ai_3", "evaluation": {"accuracy": 0.39040207522697795}, "template_name": "clarification_needed"}, "directly_answer": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='directly_answer', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "conv_ai_3", "evaluation": {"accuracy": 0.6095979247730221}, "template_name": "directly_answer"}, "score_give_number": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='score_give_number', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "conv_ai_3", "evaluation": {"accuracy": 0.21444012105490703}, "template_name": "score_give_number"}, "score_how_much": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='score_how_much', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "conv_ai_3", "evaluation": {"accuracy": 0.1733679204496325}, "template_name": "score_how_much"}}, "craigslist_bargains_None": {"best deal": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='craigslist_bargains', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='best deal', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "craigslist_bargains", "evaluation": {"accuracy": 0.25963149078726966}, "template_name": "best deal"}, "good deal for seller": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='craigslist_bargains', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='good deal for seller', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "craigslist_bargains", "evaluation": {"accuracy": 0.5192629815745393}, "template_name": "good deal for seller"}, "good deal for seller no list price": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='craigslist_bargains', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='good deal for seller no list price', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "craigslist_bargains", "evaluation": {"accuracy": 0.7252931323283082}, "template_name": "good deal for seller no list price"}, "good deal for seller no list price implicit": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='craigslist_bargains', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='good deal for seller no list price implicit', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "craigslist_bargains", "evaluation": {"accuracy": 0.23785594639865998}, "template_name": "good deal for seller no list price implicit"}}, "emotion_None": {"answer_question_with_emotion_label": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='emotion', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='answer_question_with_emotion_label', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "emotion", "evaluation": {"accuracy": 0.232}, "template_name": "answer_question_with_emotion_label"}, "answer_with_class_label": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='emotion', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='answer_with_class_label', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "emotion", "evaluation": {"accuracy": 0.2585}, "template_name": "answer_with_class_label"}, "choose_the_best_emotion_label": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='emotion', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='choose_the_best_emotion_label', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "emotion", "evaluation": {"accuracy": 0.467}, "template_name": "choose_the_best_emotion_label"}, "reply_with_emoation_label": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='emotion', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='reply_with_emoation_label', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "emotion", "evaluation": {"accuracy": 0.436}, "template_name": "reply_with_emoation_label"}}, "financial_phrasebank_sentences_allagree": {"bullish_neutral_bearish": {"arguments": "Namespace(config_name=None, dataset_config_name='sentences_allagree', dataset_name='financial_phrasebank', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='bullish_neutral_bearish', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sentences_allagree", "dataset_name": "financial_phrasebank", "evaluation": {"accuracy": 0.1603356890459364}, "template_name": "bullish_neutral_bearish"}, "complementary_industries": {"arguments": "Namespace(config_name=None, dataset_config_name='sentences_allagree', dataset_name='financial_phrasebank', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='complementary_industries', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sentences_allagree", "dataset_name": "financial_phrasebank", "evaluation": {"accuracy": 0.44787985865724383}, "template_name": "complementary_industries"}, "sentiment": {"arguments": "Namespace(config_name=None, dataset_config_name='sentences_allagree', dataset_name='financial_phrasebank', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='sentiment', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sentences_allagree", "dataset_name": "financial_phrasebank", "evaluation": {"accuracy": 0.31095406360424027}, "template_name": "sentiment"}, "share_price_option": {"arguments": "Namespace(config_name=None, dataset_config_name='sentences_allagree', dataset_name='financial_phrasebank', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='share_price_option', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sentences_allagree", "dataset_name": "financial_phrasebank", "evaluation": {"accuracy": 0.3224381625441696}, "template_name": "share_price_option"}, "word_comes_to_mind": {"arguments": "Namespace(config_name=None, dataset_config_name='sentences_allagree', dataset_name='financial_phrasebank', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='word_comes_to_mind', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sentences_allagree", "dataset_name": "financial_phrasebank", "evaluation": {"accuracy": 0.2751766784452297}, "template_name": "word_comes_to_mind"}}, "glue_cola": {"Following sentence acceptable": {"arguments": "Namespace(config_name=None, dataset_config_name='cola', dataset_name='glue', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Following sentence acceptable', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "cola", "dataset_name": "glue", "evaluation": {"accuracy": 0.31639501438159157}, "template_name": "Following sentence acceptable"}, "Make sense yes no": {"arguments": "Namespace(config_name=None, dataset_config_name='cola', dataset_name='glue', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Make sense yes no', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "cola", "dataset_name": "glue", "evaluation": {"accuracy": 0.3087248322147651}, "template_name": "Make sense yes no"}, "Previous sentence acceptable": {"arguments": "Namespace(config_name=None, dataset_config_name='cola', dataset_name='glue', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Previous sentence acceptable', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "cola", "dataset_name": "glue", "evaluation": {"accuracy": 0.31255992329817833}, "template_name": "Previous sentence acceptable"}, "editing": {"arguments": "Namespace(config_name=None, dataset_config_name='cola', dataset_name='glue', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='editing', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "cola", "dataset_name": "glue", "evaluation": {"accuracy": 0.3077660594439118}, "template_name": "editing"}, "is_this_correct": {"arguments": "Namespace(config_name=None, dataset_config_name='cola', dataset_name='glue', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='is_this_correct', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "cola", "dataset_name": "glue", "evaluation": {"accuracy": 0.4170661553211889}, "template_name": "is_this_correct"}}, "glue_sst2": {"following positive negative": {"arguments": "Namespace(config_name=None, dataset_config_name='sst2', dataset_name='glue', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='following positive negative', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sst2", "dataset_name": "glue", "evaluation": {"accuracy": 0.8795871559633027}, "template_name": "following positive negative"}, "happy or mad": {"arguments": "Namespace(config_name=None, dataset_config_name='sst2', dataset_name='glue', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='happy or mad', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sst2", "dataset_name": "glue", "evaluation": {"accuracy": 0.6571100917431193}, "template_name": "happy or mad"}, "positive negative after": {"arguments": "Namespace(config_name=None, dataset_config_name='sst2', dataset_name='glue', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='positive negative after', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sst2", "dataset_name": "glue", "evaluation": {"accuracy": 0.9277522935779816}, "template_name": "positive negative after"}, "review": {"arguments": "Namespace(config_name=None, dataset_config_name='sst2', dataset_name='glue', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='review', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sst2", "dataset_name": "glue", "evaluation": {"accuracy": 0.9288990825688074}, "template_name": "review"}, "said": {"arguments": "Namespace(config_name=None, dataset_config_name='sst2', dataset_name='glue', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='said', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sst2", "dataset_name": "glue", "evaluation": {"accuracy": 0.8325688073394495}, "template_name": "said"}}, "head_qa_en": {"multiple_choice_a_and_q_en": {"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='head_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_a_and_q_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "en", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.2591508052708638}, "template_name": "multiple_choice_a_and_q_en"}, "multiple_choice_a_and_q_with_context_en": {"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='head_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_a_and_q_with_context_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "en", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.2554904831625183}, "template_name": "multiple_choice_a_and_q_with_context_en"}, "multiple_choice_q_and_a_en": {"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='head_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_q_and_a_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "en", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.2759882869692533}, "template_name": "multiple_choice_q_and_a_en"}, "multiple_choice_q_and_a_index_en": {"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='head_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_q_and_a_index_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "en", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.2774524158125915}, "template_name": "multiple_choice_q_and_a_index_en"}, "multiple_choice_q_and_a_index_with_context_en": {"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='head_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_q_and_a_index_with_context_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "en", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.26720351390922403}, "template_name": "multiple_choice_q_and_a_index_with_context_en"}}, "head_qa_es": {"multiple_choice_a_and_q_en": {"arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='head_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_a_and_q_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "es", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.23206442166910687}, "template_name": "multiple_choice_a_and_q_en"}, "multiple_choice_a_and_q_with_context_en": {"arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='head_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_a_and_q_with_context_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "es", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.2342606149341142}, "template_name": "multiple_choice_a_and_q_with_context_en"}, "multiple_choice_q_and_a_en": {"arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='head_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_q_and_a_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "es", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.26939970717423134}, "template_name": "multiple_choice_q_and_a_en"}, "multiple_choice_q_and_a_index_en": {"arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='head_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_q_and_a_index_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "es", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.2774524158125915}, "template_name": "multiple_choice_q_and_a_index_en"}, "multiple_choice_q_and_a_index_with_context_en": {"arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='head_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_q_and_a_index_with_context_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "es", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.28257686676427524}, "template_name": "multiple_choice_q_and_a_index_with_context_en"}}, "health_fact_None": {"claim_explanation_classification": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='health_fact', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='claim_explanation_classification', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "health_fact", "evaluation": {"accuracy": 0.5477551020408163}, "template_name": "claim_explanation_classification"}, "claim_veracity_classification_after_reading_I_believe": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='health_fact', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='claim_veracity_classification_after_reading_I_believe', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "health_fact", "evaluation": {"accuracy": 0.4204081632653061}, "template_name": "claim_veracity_classification_after_reading_I_believe"}, "claim_veracity_classification_tell_me": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='health_fact', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='claim_veracity_classification_tell_me', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "health_fact", "evaluation": {"accuracy": 0.04653061224489796}, "template_name": "claim_veracity_classification_tell_me"}}, "hlgd_None": {"is_same_event_editor_asks": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='hlgd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='is_same_event_editor_asks', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "hlgd", "evaluation": {"accuracy": 0.7385210246495891}, "template_name": "is_same_event_editor_asks"}, "is_same_event_interrogative_talk": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='hlgd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='is_same_event_interrogative_talk', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "hlgd", "evaluation": {"accuracy": 0.6447559207346544}, "template_name": "is_same_event_interrogative_talk"}, "is_same_event_refer": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='hlgd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='is_same_event_refer', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "hlgd", "evaluation": {"accuracy": 0.7491541807636539}, "template_name": "is_same_event_refer"}, "is_same_event_with_time_interrogative_related": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='hlgd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='is_same_event_with_time_interrogative_related', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "hlgd", "evaluation": {"accuracy": 0.7288545190913485}, "template_name": "is_same_event_with_time_interrogative_related"}, "is_same_event_with_time_interrogative_talk": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='hlgd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='is_same_event_with_time_interrogative_talk', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "hlgd", "evaluation": {"accuracy": 0.7153214113098115}, "template_name": "is_same_event_with_time_interrogative_talk"}}, "hyperpartisan_news_detection_byarticle": {"consider_does_it_follow_a_hyperpartisan_argumentation": {"arguments": "Namespace(config_name=None, dataset_config_name='byarticle', dataset_name='hyperpartisan_news_detection', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='consider_does_it_follow_a_hyperpartisan_argumentation', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "byarticle", "dataset_name": "hyperpartisan_news_detection", "evaluation": {"accuracy": 0.6310077519379845}, "template_name": "consider_does_it_follow_a_hyperpartisan_argumentation"}, "consider_it_exhibits_extreme_one_sidedness": {"arguments": "Namespace(config_name=None, dataset_config_name='byarticle', dataset_name='hyperpartisan_news_detection', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='consider_it_exhibits_extreme_one_sidedness', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "byarticle", "dataset_name": "hyperpartisan_news_detection", "evaluation": {"accuracy": 0.6294573643410852}, "template_name": "consider_it_exhibits_extreme_one_sidedness"}, "consume_with_caution": {"arguments": "Namespace(config_name=None, dataset_config_name='byarticle', dataset_name='hyperpartisan_news_detection', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='consume_with_caution', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "byarticle", "dataset_name": "hyperpartisan_news_detection", "evaluation": {"accuracy": 0.6310077519379845}, "template_name": "consume_with_caution"}, "extreme_left_wing_or_right_wing": {"arguments": "Namespace(config_name=None, dataset_config_name='byarticle', dataset_name='hyperpartisan_news_detection', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='extreme_left_wing_or_right_wing', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "byarticle", "dataset_name": "hyperpartisan_news_detection", "evaluation": {"accuracy": 0.6294573643410852}, "template_name": "extreme_left_wing_or_right_wing"}, "follows_hyperpartisan_argumentation": {"arguments": "Namespace(config_name=None, dataset_config_name='byarticle', dataset_name='hyperpartisan_news_detection', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='follows_hyperpartisan_argumentation', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "byarticle", "dataset_name": "hyperpartisan_news_detection", "evaluation": {"accuracy": 0.6263565891472869}, "template_name": "follows_hyperpartisan_argumentation"}}, "liar_None": {"Given statement guess category": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='liar', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Given statement guess category', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "liar", "evaluation": {"accuracy": 0.17601246105919002}, "template_name": "Given statement guess category"}}, "lince_sa_spaeng": {"express sentiment": {"arguments": "Namespace(config_name=None, dataset_config_name='sa_spaeng', dataset_name='lince', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='express sentiment', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sa_spaeng", "dataset_name": "lince", "evaluation": {"accuracy": 0.5804195804195804}, "template_name": "express sentiment"}, "negation template": {"arguments": "Namespace(config_name=None, dataset_config_name='sa_spaeng', dataset_name='lince', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='negation template', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sa_spaeng", "dataset_name": "lince", "evaluation": {"accuracy": 0.164066702528241}, "template_name": "negation template"}, "original poster expressed sentiment": {"arguments": "Namespace(config_name=None, dataset_config_name='sa_spaeng', dataset_name='lince', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='original poster expressed sentiment', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sa_spaeng", "dataset_name": "lince", "evaluation": {"accuracy": 0.5330823023130715}, "template_name": "original poster expressed sentiment"}, "sentiment trying to express": {"arguments": "Namespace(config_name=None, dataset_config_name='sa_spaeng', dataset_name='lince', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='sentiment trying to express', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sa_spaeng", "dataset_name": "lince", "evaluation": {"accuracy": 0.5852608929532006}, "template_name": "sentiment trying to express"}, "the author seem": {"arguments": "Namespace(config_name=None, dataset_config_name='sa_spaeng', dataset_name='lince', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='the author seem', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sa_spaeng", "dataset_name": "lince", "evaluation": {"accuracy": 0.5831091984938139}, "template_name": "the author seem"}}, "math_qa_None": {"choose_correct_og": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='math_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='choose_correct_og', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "math_qa", "evaluation": {"accuracy": 0.1966499162479062}, "template_name": "choose_correct_og"}, "first_choice_then_problem": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='math_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='first_choice_then_problem', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "math_qa", "evaluation": {"accuracy": 0.21239530988274707}, "template_name": "first_choice_then_problem"}, "gre_problem": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='math_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='gre_problem', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "math_qa", "evaluation": {"accuracy": 0.19028475711892798}, "template_name": "gre_problem"}, "pick_the_correct": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='math_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='pick_the_correct', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "math_qa", "evaluation": {"accuracy": 0.18860971524288106}, "template_name": "pick_the_correct"}, "problem_set_type": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='math_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='problem_set_type', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "math_qa", "evaluation": {"accuracy": 0.4556113902847571}, "template_name": "problem_set_type"}}, "mlsum_es": {"layman_summ_es": {"bleu": 3.343300117413671, "bleu_stderr": 0.09571770719260354, "rouge1_fmeasure": 0.2288764030119338, "rouge1_fmeasure_stderr": 0.002161872693599132, "rouge1_precision": 0.2520284100970267, "rouge1_precision_stderr": 0.002624579613126546, "rouge1_recall": 0.25801030343845277, "rouge1_recall_stderr": 0.0029853006756947876, "rouge2_fmeasure": 0.05865599311701669, "rouge2_fmeasure_stderr": 0.0015962367564830823, "rouge2_precision": 0.0620405891134654, "rouge2_precision_stderr": 0.001792104074524257, "rouge2_recall": 0.07046107840903996, "rouge2_recall_stderr": 0.002067824846818046, "rougeL_fmeasure": 0.176509767972313, "rougeL_fmeasure_stderr": 0.0017890354543847523, "rougeL_precision": 0.19551927910705028, "rougeL_precision_stderr": 0.002202337925100599, "rougeL_recall": 0.19887535262571626, "rougeL_recall_stderr": 0.002438033895761126, "rougeLsum_fmeasure": 0.18031629233459825, "rougeLsum_fmeasure_stderr": 0.0018146301526451975, "rougeLsum_precision": 0.1998439483673831, "rougeLsum_precision_stderr": 0.0022520268262764018, "rougeLsum_recall": 0.20409646438222273, "rougeLsum_recall_stderr": 0.0025514751069907436}, "palm_prompt": {"bleu": 3.619325486335941, "bleu_stderr": 0.179919419902639, "rouge1_fmeasure": 0.23676009275524143, "rouge1_fmeasure_stderr": 0.0021301319931734007, "rouge1_precision": 0.24780663444349338, "rouge1_precision_stderr": 0.002507147956018597, "rouge1_recall": 0.27545135338282833, "rouge1_recall_stderr": 0.003003125501891925, "rouge2_fmeasure": 0.06294952611868992, "rouge2_fmeasure_stderr": 0.0016498514297430393, "rouge2_precision": 0.0640710564358735, "rouge2_precision_stderr": 0.0017900238806753247, "rouge2_recall": 0.07767444430106467, "rouge2_recall_stderr": 0.0021965535248787645, "rougeL_fmeasure": 0.18117329279306474, "rougeL_fmeasure_stderr": 0.0017908424050208493, "rougeL_precision": 0.1901311887539964, "rougeL_precision_stderr": 0.002107469317872449, "rougeL_recall": 0.2111905723569327, "rougeL_recall_stderr": 0.002518349475307121, "rougeLsum_fmeasure": 0.18538806816923076, "rougeLsum_fmeasure_stderr": 0.0018064286752298999, "rougeLsum_precision": 0.19470439311305227, "rougeLsum_precision_stderr": 0.002151332052117866, "rougeLsum_recall": 0.21735203759781344, "rougeLsum_recall_stderr": 0.0026235912805025272}, "summarise_this_in_es_few_sentences": {"bleu": 2.2882710041529153, "bleu_stderr": 0.13207303223436284, "rouge1_fmeasure": 0.2189869273902612, "rouge1_fmeasure_stderr": 0.002121244091390277, "rouge1_precision": 0.22651610428388194, "rouge1_precision_stderr": 0.0025544747897463032, "rouge1_recall": 0.27912823578913315, "rouge1_recall_stderr": 0.0030975001090301035, "rouge2_fmeasure": 0.055896502176123325, "rouge2_fmeasure_stderr": 0.001443062190482965, "rouge2_precision": 0.055842105602095085, "rouge2_precision_stderr": 0.0015575440347090582, "rouge2_recall": 0.07587543663337117, "rouge2_recall_stderr": 0.0021035214347602566, "rougeL_fmeasure": 0.16731035227157529, "rougeL_fmeasure_stderr": 0.001707299917111998, "rougeL_precision": 0.1739623288648582, "rougeL_precision_stderr": 0.002091198749309696, "rougeL_recall": 0.21461276288961498, "rougeL_recall_stderr": 0.0025095083723608315, "rougeLsum_fmeasure": 0.17158408384663376, "rougeLsum_fmeasure_stderr": 0.001743083120213333, "rougeLsum_precision": 0.1782157403124354, "rougeLsum_precision_stderr": 0.0021381828458997194, "rougeLsum_recall": 0.22125747931306458, "rougeLsum_recall_stderr": 0.002679305378482259}}, "movie_rationales_None": {"Evidences + review": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='movie_rationales', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Evidences + review', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "movie_rationales", "evaluation": {"accuracy": 0.935}, "template_name": "Evidences + review"}, "Evidences sentiment classification": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='movie_rationales', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Evidences sentiment classification', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "movie_rationales", "evaluation": {"accuracy": 0.985}, "template_name": "Evidences sentiment classification"}, "Standard binary sentiment analysis": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='movie_rationales', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Standard binary sentiment analysis', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "movie_rationales", "evaluation": {"accuracy": 0.875}, "template_name": "Standard binary sentiment analysis"}}, "mwsc_None": {"in-the-sentence": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='mwsc', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='in-the-sentence', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "mwsc", "evaluation": {"accuracy": 0.5365853658536586}, "template_name": "in-the-sentence"}, "in-the-sentence-question-first": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='mwsc', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='in-the-sentence-question-first', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "mwsc", "evaluation": {"accuracy": 0.5609756097560976}, "template_name": "in-the-sentence-question-first"}, "is-correct": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='mwsc', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='is-correct', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "mwsc", "evaluation": {"accuracy": 0.5853658536585366}, "template_name": "is-correct"}, "options-or": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='mwsc', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='options-or', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "mwsc", "evaluation": {"accuracy": 0.5}, "template_name": "options-or"}, "what-think": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='mwsc', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='what-think', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "mwsc", "evaluation": {"accuracy": 0.5121951219512195}, "template_name": "what-think"}}, "onestop_english_None": {"ara_context": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='onestop_english', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='ara_context', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "onestop_english", "evaluation": {"accuracy": 0.3368606701940035}, "template_name": "ara_context"}, "assess": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='onestop_english', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='assess', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "onestop_english", "evaluation": {"accuracy": 0.37566137566137564}, "template_name": "assess"}, "determine_reading_level_from_the_first_three_sentences": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='onestop_english', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='determine_reading_level_from_the_first_three_sentences', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "onestop_english", "evaluation": {"accuracy": 0.5185185185185185}, "template_name": "determine_reading_level_from_the_first_three_sentences"}, "esl_context": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='onestop_english', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='esl_context', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "onestop_english", "evaluation": {"accuracy": 0.43386243386243384}, "template_name": "esl_context"}, "esl_variation": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='onestop_english', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='esl_variation', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "onestop_english", "evaluation": {"accuracy": 0.5555555555555556}, "template_name": "esl_variation"}}, "poem_sentiment_None": {"guess_sentiment_without_options_variation_1": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='poem_sentiment', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='guess_sentiment_without_options_variation_1', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "poem_sentiment", "evaluation": {"accuracy": 0.2}, "template_name": "guess_sentiment_without_options_variation_1"}, "most_appropriate_sentiment": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='poem_sentiment', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='most_appropriate_sentiment', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "poem_sentiment", "evaluation": {"accuracy": 0.22857142857142856}, "template_name": "most_appropriate_sentiment"}, "positive_or_negative_sentiment_variation_1": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='poem_sentiment', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='positive_or_negative_sentiment_variation_1', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "poem_sentiment", "evaluation": {"accuracy": 0.21904761904761905}, "template_name": "positive_or_negative_sentiment_variation_1"}, "positive_or_negative_sentiment_variation_2": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='poem_sentiment', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='positive_or_negative_sentiment_variation_2', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "poem_sentiment", "evaluation": {"accuracy": 0.20952380952380953}, "template_name": "positive_or_negative_sentiment_variation_2"}, "question_answer_format": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='poem_sentiment', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='question_answer_format', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "poem_sentiment", "evaluation": {"accuracy": 0.20952380952380953}, "template_name": "question_answer_format"}}, "pubmed_qa_pqa_labeled": {"Long Answer to Final Decision": {"arguments": "Namespace(config_name=None, dataset_config_name='pqa_labeled', dataset_name='pubmed_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='Long Answer to Final Decision', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "pqa_labeled", "dataset_name": "pubmed_qa", "evaluation": {"accuracy": 0.575}, "template_name": "Long Answer to Final Decision"}, "Question Answering (Short)": {"arguments": "Namespace(config_name=None, dataset_config_name='pqa_labeled', dataset_name='pubmed_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='Question Answering (Short)', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "pqa_labeled", "dataset_name": "pubmed_qa", "evaluation": {"accuracy": 0.54}, "template_name": "Question Answering (Short)"}}, "riddle_sense_None": {"answer_given_question_without_options": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='riddle_sense', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='answer_given_question_without_options', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "riddle_sense", "evaluation": {"accuracy": 0.37414299706170423}, "template_name": "answer_given_question_without_options"}, "most_suitable_answer": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='riddle_sense', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='most_suitable_answer', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "riddle_sense", "evaluation": {"accuracy": 0.23996082272282077}, "template_name": "most_suitable_answer"}, "question_answering": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='riddle_sense', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='question_answering', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "riddle_sense", "evaluation": {"accuracy": 0.21743388834476005}, "template_name": "question_answering"}, "question_to_answer_index": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='riddle_sense', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='question_to_answer_index', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "riddle_sense", "evaluation": {"accuracy": 0.20274240940254654}, "template_name": "question_to_answer_index"}}, "scicite_None": {"Classify intent": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='scicite', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Classify intent', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "scicite", "evaluation": {"accuracy": 0.13427947598253276}, "template_name": "Classify intent"}, "Classify intent (choices first)": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='scicite', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Classify intent (choices first)', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "scicite", "evaluation": {"accuracy": 0.14737991266375547}, "template_name": "Classify intent (choices first)"}, "Classify intent (select choice)": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='scicite', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Classify intent (select choice)', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "scicite", "evaluation": {"accuracy": 0.5141921397379913}, "template_name": "Classify intent (select choice)"}, "Classify intent w/section (select choice)": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='scicite', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Classify intent w/section (select choice)', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "scicite", "evaluation": {"accuracy": 0.5491266375545851}, "template_name": "Classify intent w/section (select choice)"}, "can_describe": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='scicite', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='can_describe', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "scicite", "evaluation": {"accuracy": 0.3307860262008734}, "template_name": "can_describe"}}, "selqa_answer_selection_analysis": {"is-he-talking-about": {"arguments": "Namespace(config_name=None, dataset_config_name='answer_selection_analysis', dataset_name='selqa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='is-he-talking-about', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "answer_selection_analysis", "dataset_name": "selqa", "evaluation": {"accuracy": 0.9006369426751593}, "template_name": "is-he-talking-about"}, "make-sense-rand": {"arguments": "Namespace(config_name=None, dataset_config_name='answer_selection_analysis', dataset_name='selqa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='make-sense-rand', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "answer_selection_analysis", "dataset_name": "selqa", "evaluation": {"accuracy": 0.9235668789808917}, "template_name": "make-sense-rand"}, "which-answer-1st-vs-random": {"arguments": "Namespace(config_name=None, dataset_config_name='answer_selection_analysis', dataset_name='selqa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='which-answer-1st-vs-random', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "answer_selection_analysis", "dataset_name": "selqa", "evaluation": {"accuracy": 0.5885350318471337}, "template_name": "which-answer-1st-vs-random"}, "would-make-sense-qu-rand": {"arguments": "Namespace(config_name=None, dataset_config_name='answer_selection_analysis', dataset_name='selqa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='would-make-sense-qu-rand', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "answer_selection_analysis", "dataset_name": "selqa", "evaluation": {"accuracy": 0.8955414012738854}, "template_name": "would-make-sense-qu-rand"}}, "snips_built_in_intents_None": {"categorize_query": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='snips_built_in_intents', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='categorize_query', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "snips_built_in_intents", "evaluation": {"accuracy": 0.17682926829268292}, "template_name": "categorize_query"}, "categorize_query_brief": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='snips_built_in_intents', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='categorize_query_brief', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "snips_built_in_intents", "evaluation": {"accuracy": 0.17073170731707318}, "template_name": "categorize_query_brief"}, "intent_query": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='snips_built_in_intents', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='intent_query', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "snips_built_in_intents", "evaluation": {"accuracy": 0.27134146341463417}, "template_name": "intent_query"}, "query_intent": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='snips_built_in_intents', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='query_intent', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "snips_built_in_intents", "evaluation": {"accuracy": 0.2896341463414634}, "template_name": "query_intent"}, "voice_intent": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='snips_built_in_intents', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1750/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='voice_intent', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "snips_built_in_intents", "evaluation": {"accuracy": 0.4634146341463415}, "template_name": "voice_intent"}}, "wmt14_fr_en_en-fr": {"a_good_translation-en-fr-source+target": {"bleu": 2.362605149306088, "bleu_stderr": 0.11691133889977687}, "a_good_translation-en-fr-target": {"bleu": 2.1337681990699706, "bleu_stderr": 0.1106089212341899}, "gpt3-en-fr": {"bleu": 0.12001646955350719, "bleu_stderr": 0.015204194100382587}, "version-en-fr-target": {"bleu": 2.2199071642137147, "bleu_stderr": 0.11850494768928473}, "xglm-en-fr-target": {"bleu": 7.032093073230125, "bleu_stderr": 0.17078287053155283}}, "wmt14_fr_en_fr-en": {"a_good_translation-fr-en-source+target": {"bleu": 20.707991062833415, "bleu_stderr": 0.34815559307484384}, "a_good_translation-fr-en-target": {"bleu": 21.24042389381691, "bleu_stderr": 0.274141949124012}, "gpt3-fr-en": {"bleu": 0.8578242652965287, "bleu_stderr": 0.05616877113550148}, "version-fr-en-target": {"bleu": 18.961608535661192, "bleu_stderr": 0.473798410319246}, "xglm-fr-en-target": {"bleu": 13.352998875720225, "bleu_stderr": 0.20248257088049637}}, "wmt14_hi_en_en-hi": {"a_good_translation-en-hi-source+target": {"bleu": 1.8311038199670524, "bleu_stderr": 0.04221270358130002}, "a_good_translation-en-hi-target": {"bleu": 0.40605133742123417, "bleu_stderr": 0.04576652873201806}, "gpt-3-en-hi-target": {"bleu": 2.3016809855913706e-08, "bleu_stderr": 1.893912177723343e-08}, "version-en-hi-target": {"bleu": 0.23958056990365256, "bleu_stderr": 0.05197906740039761}, "xglm-en-hi-target": {"bleu": 0.893152501923088, "bleu_stderr": 0.06350645619846891}}, "wmt14_hi_en_hi-en": {"a_good_translation-hi-en-source+target": {"bleu": 6.966981523891758, "bleu_stderr": 0.24792997020774274}, "a_good_translation-hi-en-target": {"bleu": 9.02389919219589, "bleu_stderr": 0.1909313466483726}, "gpt-3-hi-en-target": {"bleu": 3.7797885719160414e-138, "bleu_stderr": 5.24576626705427e-93}, "version-hi-en-target": {"bleu": 7.812739647665366, "bleu_stderr": 0.17043951205234545}, "xglm-hi-en-target": {"bleu": 6.138487672011171, "bleu_stderr": 0.1415376356874753}}}