codingutf8 2022 the openbmb team and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license cpmant model configuration from configurationutils import pretrainedconfig from utils import logging logger logging getloggername cpmantpretrainedconfigarchivemap openbmbcpmant10b https huggingface coopenbmbcpmant10bblobmainconfig json see all cpmant models at https huggingface comodels filtercpmant class cpmantconfigpretrainedconfig r this is the configuration class to store the configuration of a cpmantmodel it is used to instantiate an cpmant model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the cpmant openbmbcpmant10bhttps huggingface coopenbmbcpmant10b architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 30720 vocabulary size of the cpmant model defines the number of different tokens that can be represented by the input passed when calling cpmantmodel hiddensize int optional defaults to 4096 dimension of the encoder layers numattentionheads int optional defaults to 32 number of attention heads in the transformer encoder dimhead int optional defaults to 128 dimension of attention heads for each attention layer in the transformer encoder dimff int optional defaults to 10240 dimension of the intermediate i e feedforward layer in the transformer encoder numhiddenlayers int optional defaults to 48 number of layers of the transformer encoder dropoutp float optional defaults to 0 0 the dropout probabilitiy for all fully connected layers in the embeddings encoder positionbiasnumbuckets int optional defaults to 512 the number of positionbias buckets positionbiasmaxdistance int optional defaults to 2048 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 eps float optional defaults to 1e06 the epsilon used by the layer normalization layers initstd float optional defaults to 1 0 initialize parameters with std initstd prompttypes int optional defaults to 32 the type of prompt promptlength int optional defaults to 32 the length of prompt segmenttypes int optional defaults to 32 the type of segment usecache bool optional defaults to true whether to use cache example python from transformers import cpmantmodel cpmantconfig initializing a cpmant cpmant10b style configuration configuration cpmantconfig initializing a model from the cpmant10b style configuration model cpmantmodelconfiguration accessing the model configuration configuration model config modeltype cpmant def init self vocabsize int 30720 hiddensize int 4096 numattentionheads int 32 dimhead int 128 dimff int 10240 numhiddenlayers int 48 dropoutp int 0 0 positionbiasnumbuckets int 512 positionbiasmaxdistance int 2048 eps int 1e6 initstd float 1 0 prompttypes int 32 promptlength int 32 segmenttypes int 32 usecache bool true kwargs super initkwargs self prompttypes prompttypes self promptlength promptlength self segmenttypes segmenttypes self hiddensize hiddensize self numattentionheads numattentionheads self dimhead dimhead self dimff dimff self numhiddenlayers numhiddenlayers self positionbiasnumbuckets positionbiasnumbuckets self positionbiasmaxdistance positionbiasmaxdistance self dropoutp dropoutp self eps eps self usecache usecache self vocabsize vocabsize self initstd initstd coding utf 8 2022 the openbmb team and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license cpmant model configuration see all cpmant models at https huggingface co models filter cpmant this is the configuration class to store the configuration of a cpmantmodel it is used to instantiate an cpmant model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the cpmant openbmb cpm ant 10b https huggingface co openbmb cpm ant 10b architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 30720 vocabulary size of the cpmant model defines the number of different tokens that can be represented by the input passed when calling cpmantmodel hidden_size int optional defaults to 4096 dimension of the encoder layers num_attention_heads int optional defaults to 32 number of attention heads in the transformer encoder dim_head int optional defaults to 128 dimension of attention heads for each attention layer in the transformer encoder dim_ff int optional defaults to 10240 dimension of the intermediate i e feed forward layer in the transformer encoder num_hidden_layers int optional defaults to 48 number of layers of the transformer encoder dropout_p float optional defaults to 0 0 the dropout probabilitiy for all fully connected layers in the embeddings encoder position_bias_num_buckets int optional defaults to 512 the number of position_bias buckets position_bias_max_distance int optional defaults to 2048 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 eps float optional defaults to 1e 06 the epsilon used by the layer normalization layers init_std float optional defaults to 1 0 initialize parameters with std init_std prompt_types int optional defaults to 32 the type of prompt prompt_length int optional defaults to 32 the length of prompt segment_types int optional defaults to 32 the type of segment use_cache bool optional defaults to true whether to use cache example python from transformers import cpmantmodel cpmantconfig initializing a cpmant cpm ant 10b style configuration configuration cpmantconfig initializing a model from the cpm ant 10b style configuration model cpmantmodel configuration accessing the model configuration configuration model config
from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

CPMANT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "openbmb/cpm-ant-10b": "https://huggingface.co/openbmb/cpm-ant-10b/blob/main/config.json"
    
}


class CpmAntConfig(PretrainedConfig):
    r

    model_type = "cpmant"

    def __init__(
        self,
        vocab_size: int = 30720,
        hidden_size: int = 4096,
        num_attention_heads: int = 32,
        dim_head: int = 128,
        dim_ff: int = 10240,
        num_hidden_layers: int = 48,
        dropout_p: int = 0.0,
        position_bias_num_buckets: int = 512,
        position_bias_max_distance: int = 2048,
        eps: int = 1e-6,
        init_std: float = 1.0,
        prompt_types: int = 32,
        prompt_length: int = 32,
        segment_types: int = 32,
        use_cache: bool = True,
        **kwargs,
    ):
        super().__init__(**kwargs)
        self.prompt_types = prompt_types
        self.prompt_length = prompt_length
        self.segment_types = segment_types
        self.hidden_size = hidden_size
        self.num_attention_heads = num_attention_heads
        self.dim_head = dim_head
        self.dim_ff = dim_ff
        self.num_hidden_layers = num_hidden_layers
        self.position_bias_num_buckets = position_bias_num_buckets
        self.position_bias_max_distance = position_bias_max_distance
        self.dropout_p = dropout_p
        self.eps = eps
        self.use_cache = use_cache
        self.vocab_size = vocab_size
        self.init_std = init_std