codingutf8 2022 wechatai and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license rocbert model configuration from configurationutils import pretrainedconfig from utils import logging logger logging getloggername rocbertpretrainedconfigarchivemap weiweishirocbertbasezh https huggingface coweiweishirocbertbasezhresolvemainconfig json class rocbertconfigpretrainedconfig r this is the configuration class to store the configuration of a rocbertmodel it is used to instantiate a rocbert model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the rocbert weiweishirocbertbasezhhttps huggingface coweiweishirocbertbasezh architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 30522 vocabulary size of the rocbert model defines the number of different tokens that can be represented by the inputsids passed when calling rocbertmodel hiddensize int optional defaults to 768 dimension of the encoder layers and the pooler layer numhiddenlayers int optional defaults to 12 number of hidden layers in the transformer encoder numattentionheads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediatesize int optional defaults to 3072 dimension of the intermediate i e feedforward layer in the transformer encoder hiddenact str or function optional defaults to gelu the nonlinear activation function function or string in the encoder and pooler if string gelu relu selu and gelunew are supported hiddendropoutprob float optional defaults to 0 1 the dropout probabilitiy for all fully connected layers in the embeddings encoder and pooler attentionprobsdropoutprob float optional defaults to 0 1 the dropout ratio for the attention probabilities maxpositionembeddings int optional defaults to 512 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 typevocabsize int optional defaults to 2 the vocabulary size of the tokentypeids passed when calling rocbertmodel initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices layernormeps float optional defaults to 1e12 the epsilon used by the layer normalization layers isdecoder bool optional defaults to false whether the model is used as a decoder or not if false the model is used as an encoder usecache bool optional defaults to true whether or not the model should return the last keyvalues attentions not used by all models only relevant if config isdecodertrue positionembeddingtype str optional defaults to absolute type of position embedding choose one of absolute relativekey relativekeyquery for positional embeddings use absolute for more information on relativekey please refer to selfattention with relative position representations shaw et al https arxiv orgabs1803 02155 for more information on relativekeyquery please refer to method 4 in improve transformer models with better relative position embeddings huang et al https arxiv orgabs2009 13658 classifierdropout float optional the dropout ratio for the classification head enablepronunciation bool optional defaults to true whether or not the model use pronunciation embed when training enableshape bool optional defaults to true whether or not the model use shape embed when training pronunciationembeddim int optional defaults to 768 dimension of the pronunciationembed pronunciationvocabsize int optional defaults to 910 pronunciation vocabulary size of the rocbert model defines the number of different tokens that can be represented by the inputpronunciationids passed when calling rocbertmodel shapeembeddim int optional defaults to 512 dimension of the shapeembed shapevocabsize int optional defaults to 24858 shape vocabulary size of the rocbert model defines the number of different tokens that can be represented by the inputshapeids passed when calling rocbertmodel concatinput bool optional defaults to true defines the way of merging the shapeembed pronunciationembed and wordembed if the value is true outputembed torch catwordembed shapeembed pronunciationembed 1 else outputembed wordembed shapeembed pronunciationembed 3 example python from transformers import rocbertmodel rocbertconfig initializing a rocbert weiweishirocbertbasezh style configuration configuration rocbertconfig initializing a model from the weiweishirocbertbasezh style configuration model rocbertmodelconfiguration accessing the model configuration configuration model config modeltype rocbert def init self vocabsize30522 hiddensize768 numhiddenlayers12 numattentionheads12 intermediatesize3072 hiddenactgelu hiddendropoutprob0 1 attentionprobsdropoutprob0 1 maxpositionembeddings512 typevocabsize2 initializerrange0 02 layernormeps1e12 usecachetrue padtokenid0 positionembeddingtypeabsolute classifierdropoutnone enablepronunciationtrue enableshapetrue pronunciationembeddim768 pronunciationvocabsize910 shapeembeddim512 shapevocabsize24858 concatinputtrue kwargs self vocabsize vocabsize self maxpositionembeddings maxpositionembeddings self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self initializerrange initializerrange self typevocabsize typevocabsize self layernormeps layernormeps self usecache usecache self enablepronunciation enablepronunciation self enableshape enableshape self pronunciationembeddim pronunciationembeddim self pronunciationvocabsize pronunciationvocabsize self shapeembeddim shapeembeddim self shapevocabsize shapevocabsize self concatinput concatinput self positionembeddingtype positionembeddingtype self classifierdropout classifierdropout super initpadtokenidpadtokenid kwargs coding utf 8 2022 wechatai and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license rocbert model configuration this is the configuration class to store the configuration of a rocbertmodel it is used to instantiate a rocbert model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the rocbert weiweishi roc bert base zh https huggingface co weiweishi roc bert base zh architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 30522 vocabulary size of the rocbert model defines the number of different tokens that can be represented by the inputs_ids passed when calling rocbertmodel hidden_size int optional defaults to 768 dimension of the encoder layers and the pooler layer num_hidden_layers int optional defaults to 12 number of hidden layers in the transformer encoder num_attention_heads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediate_size int optional defaults to 3072 dimension of the intermediate i e feed forward layer in the transformer encoder hidden_act str or function optional defaults to gelu the non linear activation function function or string in the encoder and pooler if string gelu relu selu and gelu_new are supported hidden_dropout_prob float optional defaults to 0 1 the dropout probabilitiy for all fully connected layers in the embeddings encoder and pooler attention_probs_dropout_prob float optional defaults to 0 1 the dropout ratio for the attention probabilities max_position_embeddings int optional defaults to 512 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 type_vocab_size int optional defaults to 2 the vocabulary size of the token_type_ids passed when calling rocbertmodel initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices layer_norm_eps float optional defaults to 1e 12 the epsilon used by the layer normalization layers is_decoder bool optional defaults to false whether the model is used as a decoder or not if false the model is used as an encoder use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models only relevant if config is_decoder true position_embedding_type str optional defaults to absolute type of position embedding choose one of absolute relative_key relative_key_query for positional embeddings use absolute for more information on relative_key please refer to self attention with relative position representations shaw et al https arxiv org abs 1803 02155 for more information on relative_key_query please refer to method 4 in improve transformer models with better relative position embeddings huang et al https arxiv org abs 2009 13658 classifier_dropout float optional the dropout ratio for the classification head enable_pronunciation bool optional defaults to true whether or not the model use pronunciation embed when training enable_shape bool optional defaults to true whether or not the model use shape embed when training pronunciation_embed_dim int optional defaults to 768 dimension of the pronunciation_embed pronunciation_vocab_size int optional defaults to 910 pronunciation vocabulary size of the rocbert model defines the number of different tokens that can be represented by the input_pronunciation_ids passed when calling rocbertmodel shape_embed_dim int optional defaults to 512 dimension of the shape_embed shape_vocab_size int optional defaults to 24858 shape vocabulary size of the rocbert model defines the number of different tokens that can be represented by the input_shape_ids passed when calling rocbertmodel concat_input bool optional defaults to true defines the way of merging the shape_embed pronunciation_embed and word_embed if the value is true output_embed torch cat word_embed shape_embed pronunciation_embed 1 else output_embed word_embed shape_embed pronunciation_embed 3 example python from transformers import rocbertmodel rocbertconfig initializing a rocbert weiweishi roc bert base zh style configuration configuration rocbertconfig initializing a model from the weiweishi roc bert base zh style configuration model rocbertmodel configuration accessing the model configuration configuration model config
from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

ROC_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "weiweishi/roc-bert-base-zh": "https://huggingface.co/weiweishi/roc-bert-base-zh/resolve/main/config.json",
}


class RoCBertConfig(PretrainedConfig):
    r

    model_type = "roc_bert"

    def __init__(
        self,
        vocab_size=30522,
        hidden_size=768,
        num_hidden_layers=12,
        num_attention_heads=12,
        intermediate_size=3072,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        max_position_embeddings=512,
        type_vocab_size=2,
        initializer_range=0.02,
        layer_norm_eps=1e-12,
        use_cache=True,
        pad_token_id=0,
        position_embedding_type="absolute",
        classifier_dropout=None,
        enable_pronunciation=True,
        enable_shape=True,
        pronunciation_embed_dim=768,
        pronunciation_vocab_size=910,
        shape_embed_dim=512,
        shape_vocab_size=24858,
        concat_input=True,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.max_position_embeddings = max_position_embeddings
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.initializer_range = initializer_range
        self.type_vocab_size = type_vocab_size
        self.layer_norm_eps = layer_norm_eps
        self.use_cache = use_cache
        self.enable_pronunciation = enable_pronunciation
        self.enable_shape = enable_shape
        self.pronunciation_embed_dim = pronunciation_embed_dim
        self.pronunciation_vocab_size = pronunciation_vocab_size
        self.shape_embed_dim = shape_embed_dim
        self.shape_vocab_size = shape_vocab_size
        self.concat_input = concat_input
        self.position_embedding_type = position_embedding_type
        self.classifier_dropout = classifier_dropout
        super().__init__(pad_token_id=pad_token_id, **kwargs)