2020 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license using torch hub return autoconfig frompretrainedargs kwargs addstartdocstringsautotokenizer doc def tokenizerargs kwargs r using torch hub import torch tokenizer torch hub load huggingfacetransformers tokenizer bertbaseuncased download vocabulary from huggingface co and cache tokenizer torch hub load huggingfacetransformers tokenizer testbertsavedmodel e g tokenizer was saved using savepretrained testsavedmodel using torch hub loading from a tf checkpoint file instead of a pytorch model slower return automodel frompretrainedargs kwargs addstartdocstringsautomodelforcausallm doc def modelforcausallmargs kwargs r using torch hub import torch model torch hub load huggingfacetransformers modelforcausallm gpt2 download model and configuration from huggingface co and cache model torch hub load huggingfacetransformers modelforcausallm testsavedmodel e g model was saved using savepretrained testsavedmodel model torch hub load huggingfacetransformers modelforcausallm gpt2 outputattentionstrue update configuration during loading assert model config outputattentions true loading from a tf checkpoint file instead of a pytorch model slower config autoconfig frompretrained tfmodelgpttfmodelconfig json model torch hub load huggingfacetransformers modelforcausallm tfmodelgpttfcheckpoint ckpt index fromtftrue configconfig using torch hub loading from a tf checkpoint file instead of a pytorch model slower return automodelformaskedlm frompretrainedargs kwargs addstartdocstringsautomodelforsequenceclassification doc def modelforsequenceclassificationargs kwargs r using torch hub import torch model torch hub load huggingfacetransformers modelforsequenceclassification bertbaseuncased download model and configuration from huggingface co and cache model torch hub load huggingfacetransformers modelforsequenceclassification testbertmodel e g model was saved using savepretrained testsavedmodel model torch hub load huggingfacetransformers modelforsequenceclassification bertbaseuncased outputattentionstrue update configuration during loading assert model config outputattentions true loading from a tf checkpoint file instead of a pytorch model slower config autoconfig frompretrained tfmodelberttfmodelconfig json model torch hub load huggingfacetransformers modelforsequenceclassification tfmodelberttfcheckpoint ckpt index fromtftrue configconfig using torch hub loading from a tf checkpoint file instead of a pytorch model slower return automodelforquestionanswering frompretrainedargs kwargs 2020 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license using torch hub import torch config torch hub load huggingface transformers config bert base uncased download configuration from huggingface co and cache config torch hub load huggingface transformers config test bert_saved_model e g config or model was saved using save_pretrained test saved_model config torch hub load huggingface transformers config test bert_saved_model my_configuration json config torch hub load huggingface transformers config bert base uncased output_attentions true foo false assert config output_attentions true config unused_kwargs torch hub load huggingface transformers config bert base uncased output_attentions true foo false return_unused_kwargs true assert config output_attentions true assert unused_kwargs foo false using torch hub import torch tokenizer torch hub load huggingface transformers tokenizer bert base uncased download vocabulary from huggingface co and cache tokenizer torch hub load huggingface transformers tokenizer test bert_saved_model e g tokenizer was saved using save_pretrained test saved_model using torch hub import torch model torch hub load huggingface transformers model bert base uncased download model and configuration from huggingface co and cache model torch hub load huggingface transformers model test bert_model e g model was saved using save_pretrained test saved_model model torch hub load huggingface transformers model bert base uncased output_attentions true update configuration during loading assert model config output_attentions true loading from a tf checkpoint file instead of a pytorch model slower config autoconfig from_pretrained tf_model bert_tf_model_config json model torch hub load huggingface transformers model tf_model bert_tf_checkpoint ckpt index from_tf true config config using torch hub import torch model torch hub load huggingface transformers modelforcausallm gpt2 download model and configuration from huggingface co and cache model torch hub load huggingface transformers modelforcausallm test saved_model e g model was saved using save_pretrained test saved_model model torch hub load huggingface transformers modelforcausallm gpt2 output_attentions true update configuration during loading assert model config output_attentions true loading from a tf checkpoint file instead of a pytorch model slower config autoconfig from_pretrained tf_model gpt_tf_model_config json model torch hub load huggingface transformers modelforcausallm tf_model gpt_tf_checkpoint ckpt index from_tf true config config using torch hub import torch model torch hub load huggingface transformers modelformaskedlm bert base uncased download model and configuration from huggingface co and cache model torch hub load huggingface transformers modelformaskedlm test bert_model e g model was saved using save_pretrained test saved_model model torch hub load huggingface transformers modelformaskedlm bert base uncased output_attentions true update configuration during loading assert model config output_attentions true loading from a tf checkpoint file instead of a pytorch model slower config autoconfig from_pretrained tf_model bert_tf_model_config json model torch hub load huggingface transformers modelformaskedlm tf_model bert_tf_checkpoint ckpt index from_tf true config config using torch hub import torch model torch hub load huggingface transformers modelforsequenceclassification bert base uncased download model and configuration from huggingface co and cache model torch hub load huggingface transformers modelforsequenceclassification test bert_model e g model was saved using save_pretrained test saved_model model torch hub load huggingface transformers modelforsequenceclassification bert base uncased output_attentions true update configuration during loading assert model config output_attentions true loading from a tf checkpoint file instead of a pytorch model slower config autoconfig from_pretrained tf_model bert_tf_model_config json model torch hub load huggingface transformers modelforsequenceclassification tf_model bert_tf_checkpoint ckpt index from_tf true config config using torch hub import torch model torch hub load huggingface transformers modelforquestionanswering bert base uncased download model and configuration from huggingface co and cache model torch hub load huggingface transformers modelforquestionanswering test bert_model e g model was saved using save_pretrained test saved_model model torch hub load huggingface transformers modelforquestionanswering bert base uncased output_attentions true update configuration during loading assert model config output_attentions true loading from a tf checkpoint file instead of a pytorch model slower config autoconfig from_pretrained tf_model bert_tf_model_config json model torch hub load huggingface transformers modelforquestionanswering tf_model bert_tf_checkpoint ckpt index from_tf true config config
import os
import sys


SRC_DIR = os.path.join(os.path.dirname(__file__), "src")
sys.path.append(SRC_DIR)


from transformers import (
    AutoConfig,
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForMaskedLM,
    AutoModelForQuestionAnswering,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    add_start_docstrings,
)


dependencies = ["torch", "numpy", "tokenizers", "filelock", "requests", "tqdm", "regex", "sentencepiece", "sacremoses", "importlib_metadata", "huggingface_hub"]


@add_start_docstrings(AutoConfig.__doc__)
def config(*args, **kwargs):
    r

    return AutoConfig.from_pretrained(*args, **kwargs)


@add_start_docstrings(AutoTokenizer.__doc__)
def tokenizer(*args, **kwargs):
    r

    return AutoTokenizer.from_pretrained(*args, **kwargs)


@add_start_docstrings(AutoModel.__doc__)
def model(*args, **kwargs):
    r

    return AutoModel.from_pretrained(*args, **kwargs)


@add_start_docstrings(AutoModelForCausalLM.__doc__)
def modelForCausalLM(*args, **kwargs):
    r
    return AutoModelForCausalLM.from_pretrained(*args, **kwargs)


@add_start_docstrings(AutoModelForMaskedLM.__doc__)
def modelForMaskedLM(*args, **kwargs):
    r

    return AutoModelForMaskedLM.from_pretrained(*args, **kwargs)


@add_start_docstrings(AutoModelForSequenceClassification.__doc__)
def modelForSequenceClassification(*args, **kwargs):
    r

    return AutoModelForSequenceClassification.from_pretrained(*args, **kwargs)


@add_start_docstrings(AutoModelForQuestionAnswering.__doc__)
def modelForQuestionAnswering(*args, **kwargs):
    r
    return AutoModelForQuestionAnswering.from_pretrained(*args, **kwargs)