codingutf8 2023 huggingface inc licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license gptsanjapanese model configuration from configurationutils import pretrainedconfig from utils import logging logger logging getloggername gptsanjapanesepretrainedconfigarchivemap tanreinamagptsan2 8bspoutisuniform https huggingface cotanreinamagptsan2 8bspoutisuniformresolvemainconfig json class gptsanjapaneseconfigpretrainedconfig r this is the configuration class to store the configuration of a gptsanjapanesemodel it is used to instantiate a gptsanjapanese model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the gptsanjapanese tanreigptsanjapanesehttps huggingface cotanreigptsanjapanese architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information arguments vocabsize int optional defaults to 36000 vocabulary size of the gptsanjapanese model defines the number of different tokens that can be represented by the inputsids passed when calling gptsanjapanesemodel maxpositionembeddings int optional defaults to 1280 the maximum sequence length that this model might ever be used with defaults set this to 1280 dmodel int optional defaults to 1024 size of the encoder layers and the pooler layer dff int optional defaults to 8192 size of the intermediate feed forward layer in each switchtransformersblock dext int optional defaults to 4096 size of the intermediate feed forward layer in each extralayers dspout int optional defaults to 128 size of the spout vector numswitchlayers int optional defaults to 10 number of layers in the switch transformer layer numextlayers int optional defaults to 0 number of layers in the extralayers numheads int optional defaults to 16 number of attention heads for each attention layer in the transformer encoder numexperts int optional defaults to 16 number of experts for each switchtransformer layer expertcapacity int optional defaults to 128 number of tokens that can be stored in each expert if set to 1 the model will behave like a regular transformer dropoutrate float optional defaults to 0 0 the ratio for all dropout layers layernormeps float optional defaults to 1e5 the epsilon used by the layer normalization layers routerbias bool optional defaults to false whether to add a bias to the router routerjitternoise float optional defaults to 0 0 amount of noise to add to the router set it to 0 0 during prediction or set small value usually 1e2 during training routerdtype str optional default to float32 the dtype used for the routers it is preferable to keep the dtype to float32 as specified in the selective precision discussion in the paperhttps arxiv orgabs2101 03961 routerignorepaddingtokens bool optional defaults to false whether to ignore padding tokens when routing outputhiddenstates bool optional default to false whether or not to return the hidden states of all layers see hiddenstates under returned tensors for more detail outputattentions bool optional defaults to false whether or not to return the attentions tensors of all attention layers initializerfactor float optional defaults to 0 002 a factor for initializing all weight matrices outputrouterlogits bool optional default to false whether or not to return the router logits of all experts usecache bool optional defaults to true whether or not the model should return the last keyvalues attentions not used by all models coding utf 8 2023 huggingface inc licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license gptsan japanese model configuration this is the configuration class to store the configuration of a gptsanjapanesemodel it is used to instantiate a gptsanjapanese model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the gptsanjapanese tanrei gptsan japanese https huggingface co tanrei gptsan japanese architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information arguments vocab_size int optional defaults to 36000 vocabulary size of the gptsanjapanese model defines the number of different tokens that can be represented by the inputs_ids passed when calling gptsanjapanesemodel max_position_embeddings int optional defaults to 1280 the maximum sequence length that this model might ever be used with defaults set this to 1280 d_model int optional defaults to 1024 size of the encoder layers and the pooler layer d_ff int optional defaults to 8192 size of the intermediate feed forward layer in each switchtransformersblock d_ext int optional defaults to 4096 size of the intermediate feed forward layer in each extra layers d_spout int optional defaults to 128 size of the spout vector num_switch_layers int optional defaults to 10 number of layers in the switch transformer layer num_ext_layers int optional defaults to 0 number of layers in the extra layers num_heads int optional defaults to 16 number of attention heads for each attention layer in the transformer encoder num_experts int optional defaults to 16 number of experts for each switchtransformer layer expert_capacity int optional defaults to 128 number of tokens that can be stored in each expert if set to 1 the model will behave like a regular transformer dropout_rate float optional defaults to 0 0 the ratio for all dropout layers layer_norm_eps float optional defaults to 1e 5 the epsilon used by the layer normalization layers router_bias bool optional defaults to false whether to add a bias to the router router_jitter_noise float optional defaults to 0 0 amount of noise to add to the router set it to 0 0 during prediction or set small value usually 1e 2 during training router_dtype str optional default to float32 the dtype used for the routers it is preferable to keep the dtype to float32 as specified in the selective precision discussion in the paper https arxiv org abs 2101 03961 router_ignore_padding_tokens bool optional defaults to false whether to ignore padding tokens when routing output_hidden_states bool optional default to false whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail output_attentions bool optional defaults to false whether or not to return the attentions tensors of all attention layers initializer_factor float optional defaults to 0 002 a factor for initializing all weight matrices output_router_logits bool optional default to false whether or not to return the router logits of all experts use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models
from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

GPTSAN_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "tanreinama/GPTSAN-2.8B-spout_is_uniform": (
        "https://huggingface.co/tanreinama/GPTSAN-2.8B-spout_is_uniform/resolve/main/config.json"
    ),
}


class GPTSanJapaneseConfig(PretrainedConfig):
    r

    model_type = "gptsan-japanese"
    keys_to_ignore_at_inference = [
        "past_key_values",
    ]
    attribute_map = {
        "hidden_size": "d_model",
        "num_attention_heads": "num_heads",
        "num_hidden_layers": "num_layers",
    }

    def __init__(
        self,
        vocab_size=36000,
        max_position_embeddings=1280,
        d_model=1024,
        d_ff=8192,
        d_ext=4096,
        d_spout=128,
        num_switch_layers=10,
        num_ext_layers=0,
        num_heads=16,
        num_experts=16,
        expert_capacity=128,
        dropout_rate=0.0,
        layer_norm_epsilon=1e-5,
        router_bias=False,
        router_jitter_noise=0.0,
        router_dtype="float32",
        router_ignore_padding_tokens=False,
        output_hidden_states=False,
        output_attentions=False,
        initializer_factor=0.002,
        output_router_logits=False,
        use_cache=True,
        separator_token_id=35998,
        pad_token_id=35995,
        eos_token_id=35999,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.max_position_embeddings = max_position_embeddings
        self.d_model = d_model
        self.d_ff = d_ff
        self.d_ext = d_ext
        self.d_spout = d_spout
        self.num_switch_layers = num_switch_layers
        self.num_ext_layers = num_ext_layers
        self.num_layers = num_switch_layers + num_ext_layers
        self.num_heads = num_heads
        self.num_experts = num_experts
        self.expert_capacity = expert_capacity
        self.dropout_rate = dropout_rate
        self.layer_norm_epsilon = layer_norm_epsilon
        self.router_bias = router_bias
        self.router_jitter_noise = router_jitter_noise
        self.router_dtype = router_dtype
        self.router_ignore_padding_tokens = router_ignore_padding_tokens
        self.output_hidden_states = output_hidden_states
        self.output_attentions = output_attentions
        self.initializer_factor = initializer_factor
        self.output_router_logits = output_router_logits
        self.use_cache = use_cache

        super().__init__(
            separator_token_id=separator_token_id,
            pad_token_id=pad_token_id,
            eos_token_id=eos_token_id,
            **kwargs,
        )