<commit_before>import torch
import p3b9 as bmk
import candle

from dataclasses import dataclass, field
from transformers import (
BertTokenizer, BertConfig, BertForMaskedLM, DataCollatorForLanguageModeling, Trainer, TrainingArguments, HfArgumentParser
)

import webdataset as wd

class truncate(object):

    def __init__(self, max_len):
        self.max_len = max_len

    def __call__(self, doc):
        return doc[:self.max_len]


def initialize_parameters():
    """ Initialize the parameters for the P3B5 benchmark """

    p3b9_bench = bmk.BenchmarkP3B9(
        bmk.file_path,
        "default_model.txt",
        "pytorch",
        prog="p3b9",
        desc="BERT",
    )

    gParameters = candle.finalize_parameters(p3b9_bench)
    return gParameters


def run(args):

    # Repair argument overlap (i.e. arguments
    # that have same function but slightly different
    # names between CANDLE and hf parsers)
    args['seed'] = args['rng_seed']
    args['do_train'] = args['train_bool']
    args['do_eval'] = args['eval_bool']

    parser = HfArgumentParser((TrainingArguments))
    training_args = parser.parse_dict(args)[0]<commit_msg>check for existing keys first<commit_after>import torch
import p3b9 as bmk
import candle

from dataclasses import dataclass, field
from transformers import (
BertTokenizer, BertConfig, BertForMaskedLM, DataCollatorForLanguageModeling, Trainer, TrainingArguments, HfArgumentParser
)

import webdataset as wd

class truncate(object):

    def __init__(self, max_len):
        self.max_len = max_len

    def __call__(self, doc):
        return doc[:self.max_len]


def initialize_parameters():
    """ Initialize the parameters for the P3B5 benchmark """

    p3b9_bench = bmk.BenchmarkP3B9(
        bmk.file_path,
        "default_model.txt",
        "pytorch",
        prog="p3b9",
        desc="BERT",
    )

    gParameters = candle.finalize_parameters(p3b9_bench)
    return gParameters


def run(args):

    # Repair argument overlap (i.e. arguments
    # that have same function but slightly different
    # names between CANDLE and hf parsers)
    args['seed'] = args['rng_seed']
    args['do_train'] = args['train_bool']
    if 'eval_bool' in args.keys():
        args['do_eval'] = args['eval_bool']

    parser = HfArgumentParser((TrainingArguments))
    training_args = parser.parse_dict(args)[0]
{'fraction_non_alphanumeric': 0.09681528662420383, 'fraction_numerical': 0.011040339702760084, 'mean_word_length': 3.693227091633466, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '348501', 'n_tokens_mistral': 759, 'n_tokens_neox': 736, 'n_words': 185}