codingutf8 2020 huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license you can also import this e g from testmodelingfunnel import funnelmodeltester def init self parent batchsize13 seqlength7 istrainingtrue useinputmasktrue usetokentypeidstrue uselabelstrue vocabsize99 blocksizes1 1 2 numdecoderlayers1 dmodel32 nhead4 dhead8 dinner37 hiddenactgelunew hiddendropout0 1 attentiondropout0 1 activationdropout0 0 maxpositionembeddings512 typevocabsize3 initializerstd0 02 set to a smaller value so we can keep the small error threshold 1e5 in the test numlabels3 numchoices4 scopenone basefalse self parent parent self batchsize batchsize self seqlength seqlength self istraining istraining self useinputmask useinputmask self usetokentypeids usetokentypeids self uselabels uselabels self vocabsize vocabsize self blocksizes blocksizes self numdecoderlayers numdecoderlayers self dmodel dmodel self nhead nhead self dhead dhead self dinner dinner self hiddenact hiddenact self hiddendropout hiddendropout self attentiondropout attentiondropout self activationdropout activationdropout self maxpositionembeddings maxpositionembeddings self typevocabsize typevocabsize self typesequencelabelsize 2 self numlabels numlabels self numchoices numchoices self scope scope self initializerstd initializerstd used in the tests to check the size of the first attention layer self numattentionheads nhead used in the tests to check the size of the first hidden state self hiddensize self dmodel used in the tests to check the number of output hidden statesattentions self numhiddenlayers sumself blocksizes 0 if base else self numdecoderlayers funnelmodel adds two hidden layers input embeddings and the sum of the upsampled encoder hidden state with the last hidden state of the first block which is the first hidden state of the decoder if not base self expectednumhiddenlayers self numhiddenlayers 2 def prepareconfigandinputsself inputids idstensorself batchsize self seqlength self vocabsize inputmask none if self useinputmask inputmask randomattentionmaskself batchsize self seqlength tokentypeids none if self usetokentypeids tokentypeids idstensorself batchsize self seqlength self typevocabsize sequencelabels none tokenlabels none choicelabels none if self uselabels sequencelabels idstensorself batchsize self typesequencelabelsize tokenlabels idstensorself batchsize self seqlength self numlabels choicelabels idstensorself batchsize self numchoices config funnelconfig vocabsizeself vocabsize blocksizesself blocksizes numdecoderlayersself numdecoderlayers dmodelself dmodel nheadself nhead dheadself dhead dinnerself dinner hiddenactself hiddenact hiddendropoutself hiddendropout attentiondropoutself attentiondropout activationdropoutself activationdropout maxpositionembeddingsself maxpositionembeddings typevocabsizeself typevocabsize initializerstdself initializerstd return config inputids tokentypeids inputmask sequencelabels tokenlabels choicelabels def createandcheckmodel self config inputids tokentypeids inputmask sequencelabels tokenlabels choicelabels model tffunnelmodelconfigconfig inputs inputids inputids attentionmask inputmask tokentypeids tokentypeids result modelinputs inputs inputids inputmask result modelinputs result modelinputids self parent assertequalresult lasthiddenstate shape self batchsize self seqlength self dmodel config truncateseq false model tffunnelmodelconfigconfig result modelinputids self parent assertequalresult lasthiddenstate shape self batchsize self seqlength self dmodel config separatecls false model tffunnelmodelconfigconfig result modelinputids self parent assertequalresult lasthiddenstate shape self batchsize self seqlength self dmodel def createandcheckbasemodel self config inputids tokentypeids inputmask sequencelabels tokenlabels choicelabels model tffunnelbasemodelconfigconfig inputs inputids inputids attentionmask inputmask tokentypeids tokentypeids result modelinputs inputs inputids inputmask result modelinputs result modelinputids self parent assertequalresult lasthiddenstate shape self batchsize 2 self dmodel config truncateseq false model tffunnelbasemodelconfigconfig result modelinputids self parent assertequalresult lasthiddenstate shape self batchsize 3 self dmodel config separatecls false model tffunnelbasemodelconfigconfig result modelinputids self parent assertequalresult lasthiddenstate shape self batchsize 2 self dmodel def createandcheckforpretraining self config inputids tokentypeids inputmask sequencelabels tokenlabels choicelabels model tffunnelforpretrainingconfigconfig inputs inputids inputids attentionmask inputmask tokentypeids tokentypeids result modelinputs self parent assertequalresult logits shape self batchsize self seqlength def createandcheckformaskedlm self config inputids tokentypeids inputmask sequencelabels tokenlabels choicelabels model tffunnelformaskedlmconfigconfig inputs inputids inputids attentionmask inputmask tokentypeids tokentypeids result modelinputs self parent assertequalresult logits shape self batchsize self seqlength self vocabsize def createandcheckforsequenceclassification self config inputids tokentypeids inputmask sequencelabels tokenlabels choicelabels config numlabels self numlabels model tffunnelforsequenceclassificationconfigconfig inputs inputids inputids attentionmask inputmask tokentypeids tokentypeids result modelinputs self parent assertequalresult logits shape self batchsize self numlabels def createandcheckformultiplechoice self config inputids tokentypeids inputmask sequencelabels tokenlabels choicelabels config numchoices self numchoices model tffunnelformultiplechoiceconfigconfig multiplechoiceinputsids tf tiletf expanddimsinputids 1 1 self numchoices 1 multiplechoiceinputmask tf tiletf expanddimsinputmask 1 1 self numchoices 1 multiplechoicetokentypeids tf tiletf expanddimstokentypeids 1 1 self numchoices 1 inputs inputids multiplechoiceinputsids attentionmask multiplechoiceinputmask tokentypeids multiplechoicetokentypeids result modelinputs self parent assertequalresult logits shape self batchsize self numchoices def createandcheckfortokenclassification self config inputids tokentypeids inputmask sequencelabels tokenlabels choicelabels config numlabels self numlabels model tffunnelfortokenclassificationconfigconfig inputs inputids inputids attentionmask inputmask tokentypeids tokentypeids result modelinputs self parent assertequalresult logits shape self batchsize self seqlength self numlabels def createandcheckforquestionanswering self config inputids tokentypeids inputmask sequencelabels tokenlabels choicelabels model tffunnelforquestionansweringconfigconfig inputs inputids inputids attentionmask inputmask tokentypeids tokentypeids result modelinputs self parent assertequalresult startlogits shape self batchsize self seqlength self parent assertequalresult endlogits shape self batchsize self seqlength def prepareconfigandinputsforcommonself configandinputs self prepareconfigandinputs config inputids tokentypeids inputmask sequencelabels tokenlabels choicelabels configandinputs inputsdict inputids inputids tokentypeids tokentypeids attentionmask inputmask return config inputsdict requiretf class tffunnelmodeltesttfmodeltestermixin pipelinetestermixin unittest testcase allmodelclasses tffunnelmodel tffunnelformaskedlm tffunnelforpretraining tffunnelforquestionanswering tffunnelfortokenclassification if istfavailable else pipelinemodelmapping featureextraction tffunnelbasemodel tffunnelmodel fillmask tffunnelformaskedlm questionanswering tffunnelforquestionanswering textclassification tffunnelforsequenceclassification tokenclassification tffunnelfortokenclassification zeroshot tffunnelforsequenceclassification if istfavailable else testheadmasking false testonnx false def setupself self modeltester tffunnelmodeltesterself self configtester configtesterself configclassfunnelconfig def testconfigself self configtester runcommontests def testmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelconfigandinputs def testforpretrainingself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckforpretrainingconfigandinputs def testformaskedlmself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckformaskedlmconfigandinputs def testfortokenclassificationself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckfortokenclassificationconfigandinputs def testforquestionansweringself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckforquestionansweringconfigandinputs requiretf class tffunnelbasemodeltesttfmodeltestermixin unittest testcase allmodelclasses tffunnelbasemodel tffunnelformultiplechoice tffunnelforsequenceclassification if istfavailable else testheadmasking false testonnx false def setupself self modeltester tffunnelmodeltesterself basetrue self configtester configtesterself configclassfunnelconfig def testconfigself self configtester runcommontests def testbasemodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckbasemodelconfigandinputs def testforsequenceclassificationself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckforsequenceclassificationconfigandinputs def testformultiplechoiceself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckformultiplechoiceconfigandinputs coding utf 8 2020 huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license you can also import this e g from test_modeling_funnel import funnelmodeltester set to a smaller value so we can keep the small error threshold 1e 5 in the test used in the tests to check the size of the first attention layer used in the tests to check the size of the first hidden state used in the tests to check the number of output hidden states attentions funnelmodel adds two hidden layers input embeddings and the sum of the upsampled encoder hidden state with the last hidden state of the first block which is the first hidden state of the decoder
from __future__ import annotations

import unittest

from transformers import FunnelConfig, is_tf_available
from transformers.testing_utils import require_tf

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import tensorflow as tf

    from transformers import (
        TFFunnelBaseModel,
        TFFunnelForMaskedLM,
        TFFunnelForMultipleChoice,
        TFFunnelForPreTraining,
        TFFunnelForQuestionAnswering,
        TFFunnelForSequenceClassification,
        TFFunnelForTokenClassification,
        TFFunnelModel,
    )


class TFFunnelModelTester:
    

    def __init__(
        self,
        parent,
        batch_size=13,
        seq_length=7,
        is_training=True,
        use_input_mask=True,
        use_token_type_ids=True,
        use_labels=True,
        vocab_size=99,
        block_sizes=[1, 1, 2],
        num_decoder_layers=1,
        d_model=32,
        n_head=4,
        d_head=8,
        d_inner=37,
        hidden_act="gelu_new",
        hidden_dropout=0.1,
        attention_dropout=0.1,
        activation_dropout=0.0,
        max_position_embeddings=512,
        type_vocab_size=3,
        initializer_std=0.02,  
        num_labels=3,
        num_choices=4,
        scope=None,
        base=False,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.seq_length = seq_length
        self.is_training = is_training
        self.use_input_mask = use_input_mask
        self.use_token_type_ids = use_token_type_ids
        self.use_labels = use_labels
        self.vocab_size = vocab_size
        self.block_sizes = block_sizes
        self.num_decoder_layers = num_decoder_layers
        self.d_model = d_model
        self.n_head = n_head
        self.d_head = d_head
        self.d_inner = d_inner
        self.hidden_act = hidden_act
        self.hidden_dropout = hidden_dropout
        self.attention_dropout = attention_dropout
        self.activation_dropout = activation_dropout
        self.max_position_embeddings = max_position_embeddings
        self.type_vocab_size = type_vocab_size
        self.type_sequence_label_size = 2
        self.num_labels = num_labels
        self.num_choices = num_choices
        self.scope = scope
        self.initializer_std = initializer_std

        
        self.num_attention_heads = n_head
        
        self.hidden_size = self.d_model
        
        self.num_hidden_layers = sum(self.block_sizes) + (0 if base else self.num_decoder_layers)
        
        
        if not base:
            self.expected_num_hidden_layers = self.num_hidden_layers + 2

    def prepare_config_and_inputs(self):
        input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)

        input_mask = None
        if self.use_input_mask:
            input_mask = random_attention_mask([self.batch_size, self.seq_length])

        token_type_ids = None
        if self.use_token_type_ids:
            token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size)

        sequence_labels = None
        token_labels = None
        choice_labels = None
        if self.use_labels:
            sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
            token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
            choice_labels = ids_tensor([self.batch_size], self.num_choices)

        config = FunnelConfig(
            vocab_size=self.vocab_size,
            block_sizes=self.block_sizes,
            num_decoder_layers=self.num_decoder_layers,
            d_model=self.d_model,
            n_head=self.n_head,
            d_head=self.d_head,
            d_inner=self.d_inner,
            hidden_act=self.hidden_act,
            hidden_dropout=self.hidden_dropout,
            attention_dropout=self.attention_dropout,
            activation_dropout=self.activation_dropout,
            max_position_embeddings=self.max_position_embeddings,
            type_vocab_size=self.type_vocab_size,
            initializer_std=self.initializer_std,
        )

        return (
            config,
            input_ids,
            token_type_ids,
            input_mask,
            sequence_labels,
            token_labels,
            choice_labels,
        )

    def create_and_check_model(
        self,
        config,
        input_ids,
        token_type_ids,
        input_mask,
        sequence_labels,
        token_labels,
        choice_labels,
    ):
        model = TFFunnelModel(config=config)
        inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
        result = model(inputs)

        inputs = [input_ids, input_mask]
        result = model(inputs)

        result = model(input_ids)
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.d_model))

        config.truncate_seq = False
        model = TFFunnelModel(config=config)
        result = model(input_ids)
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.d_model))

        config.separate_cls = False
        model = TFFunnelModel(config=config)
        result = model(input_ids)
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.d_model))

    def create_and_check_base_model(
        self,
        config,
        input_ids,
        token_type_ids,
        input_mask,
        sequence_labels,
        token_labels,
        choice_labels,
    ):
        model = TFFunnelBaseModel(config=config)
        inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
        result = model(inputs)

        inputs = [input_ids, input_mask]
        result = model(inputs)

        result = model(input_ids)
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, 2, self.d_model))

        config.truncate_seq = False
        model = TFFunnelBaseModel(config=config)
        result = model(input_ids)
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, 3, self.d_model))

        config.separate_cls = False
        model = TFFunnelBaseModel(config=config)
        result = model(input_ids)
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, 2, self.d_model))

    def create_and_check_for_pretraining(
        self,
        config,
        input_ids,
        token_type_ids,
        input_mask,
        sequence_labels,
        token_labels,
        choice_labels,
    ):
        model = TFFunnelForPreTraining(config=config)
        inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
        result = model(inputs)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length))

    def create_and_check_for_masked_lm(
        self,
        config,
        input_ids,
        token_type_ids,
        input_mask,
        sequence_labels,
        token_labels,
        choice_labels,
    ):
        model = TFFunnelForMaskedLM(config=config)
        inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
        result = model(inputs)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))

    def create_and_check_for_sequence_classification(
        self,
        config,
        input_ids,
        token_type_ids,
        input_mask,
        sequence_labels,
        token_labels,
        choice_labels,
    ):
        config.num_labels = self.num_labels
        model = TFFunnelForSequenceClassification(config=config)
        inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
        result = model(inputs)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels))

    def create_and_check_for_multiple_choice(
        self,
        config,
        input_ids,
        token_type_ids,
        input_mask,
        sequence_labels,
        token_labels,
        choice_labels,
    ):
        config.num_choices = self.num_choices
        model = TFFunnelForMultipleChoice(config=config)
        multiple_choice_inputs_ids = tf.tile(tf.expand_dims(input_ids, 1), (1, self.num_choices, 1))
        multiple_choice_input_mask = tf.tile(tf.expand_dims(input_mask, 1), (1, self.num_choices, 1))
        multiple_choice_token_type_ids = tf.tile(tf.expand_dims(token_type_ids, 1), (1, self.num_choices, 1))
        inputs = {
            "input_ids": multiple_choice_inputs_ids,
            "attention_mask": multiple_choice_input_mask,
            "token_type_ids": multiple_choice_token_type_ids,
        }
        result = model(inputs)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_choices))

    def create_and_check_for_token_classification(
        self,
        config,
        input_ids,
        token_type_ids,
        input_mask,
        sequence_labels,
        token_labels,
        choice_labels,
    ):
        config.num_labels = self.num_labels
        model = TFFunnelForTokenClassification(config=config)
        inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
        result = model(inputs)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels))

    def create_and_check_for_question_answering(
        self,
        config,
        input_ids,
        token_type_ids,
        input_mask,
        sequence_labels,
        token_labels,
        choice_labels,
    ):
        model = TFFunnelForQuestionAnswering(config=config)
        inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
        result = model(inputs)
        self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length))
        self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length))

    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        (
            config,
            input_ids,
            token_type_ids,
            input_mask,
            sequence_labels,
            token_labels,
            choice_labels,
        ) = config_and_inputs
        inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
        return config, inputs_dict


@require_tf
class TFFunnelModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
    all_model_classes = (
        (
            TFFunnelModel,
            TFFunnelForMaskedLM,
            TFFunnelForPreTraining,
            TFFunnelForQuestionAnswering,
            TFFunnelForTokenClassification,
        )
        if is_tf_available()
        else ()
    )
    pipeline_model_mapping = (
        {
            "feature-extraction": (TFFunnelBaseModel, TFFunnelModel),
            "fill-mask": TFFunnelForMaskedLM,
            "question-answering": TFFunnelForQuestionAnswering,
            "text-classification": TFFunnelForSequenceClassification,
            "token-classification": TFFunnelForTokenClassification,
            "zero-shot": TFFunnelForSequenceClassification,
        }
        if is_tf_available()
        else {}
    )
    test_head_masking = False
    test_onnx = False

    def setUp(self):
        self.model_tester = TFFunnelModelTester(self)
        self.config_tester = ConfigTester(self, config_class=FunnelConfig)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*config_and_inputs)

    def test_for_pretraining(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_pretraining(*config_and_inputs)

    def test_for_masked_lm(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_lm(*config_and_inputs)

    def test_for_token_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_token_classification(*config_and_inputs)

    def test_for_question_answering(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_question_answering(*config_and_inputs)


@require_tf
class TFFunnelBaseModelTest(TFModelTesterMixin, unittest.TestCase):
    all_model_classes = (
        (TFFunnelBaseModel, TFFunnelForMultipleChoice, TFFunnelForSequenceClassification) if is_tf_available() else ()
    )
    test_head_masking = False
    test_onnx = False

    def setUp(self):
        self.model_tester = TFFunnelModelTester(self, base=True)
        self.config_tester = ConfigTester(self, config_class=FunnelConfig)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_base_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_base_model(*config_and_inputs)

    def test_for_sequence_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_sequence_classification(*config_and_inputs)

    def test_for_multiple_choice(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_multiple_choice(*config_and_inputs)