



import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_torch, require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_torch_available, is_vision_available


if is_torch_available():
 import torch


if is_vision_available():
 from PIL import Image

 from transformers import MgpstrProcessor, ViTImageProcessor

@require_torch
@require_vision
class     _A       (	unittest.TestCase    ):
       UpperCamelCase__  :    str											=						ViTImageProcessor if is_vision_available() else None



       @property
       def 		_lowerCamelCase			(		self							:     List[Any]):


           '''simple docstring'''

           return self.image_processor_tester.prepare_image_processor_dict()



       def 		_lowerCamelCase			(		self							:     List[Any]):


           '''simple docstring'''

           __a						       =				(3, 32, 128)
           __a						       =				tempfile.mkdtemp()

           # fmt: off
           __a						       =				['''[GO]''', '''[s]''', '''0''', '''1''', '''2''', '''3''', '''4''', '''5''', '''6''', '''7''', '''8''', '''9''', '''a''', '''b''', '''c''', '''d''', '''e''', '''f''', '''g''', '''h''', '''i''', '''j''', '''k''', '''l''', '''m''', '''n''', '''o''', '''p''', '''q''', '''r''', '''s''', '''t''', '''u''', '''v''', '''w''', '''x''', '''y''', '''z''']
           # fmt: on
           __a						       =				dict(zip(__SCREAMING_SNAKE_CASE     ,		range(len(__SCREAMING_SNAKE_CASE))))

           __a						       =				os.path.join(self.tmpdirname     ,		VOCAB_FILES_NAMES['''vocab_file'''])
           with open(self.vocab_file     ,		'''w'''     ,		encoding='''utf-8''') as fp:
               fp.write(json.dumps(__SCREAMING_SNAKE_CASE) + '''\n''')

           __a						       =				{
               '''do_normalize''': False,
               '''do_resize''': True,
               '''image_processor_type''': '''ViTImageProcessor''',
               '''resample''': 3,
               '''size''': {'''height''': 32, '''width''': 128},
           }
           __a						       =				os.path.join(self.tmpdirname     ,		__SCREAMING_SNAKE_CASE)
           with open(self.image_processor_file     ,		'''w'''     ,		encoding='''utf-8''') as fp:
               json.dump(__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase			(		self							:     Optional[Any]     ,		**__SCREAMING_SNAKE_CASE							:     str):


           '''simple docstring'''

           return MgpstrTokenizer.from_pretrained(self.tmpdirname     ,		**__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase			(		self							:     str     ,		**__SCREAMING_SNAKE_CASE							:     Any):


           '''simple docstring'''

           return ViTImageProcessor.from_pretrained(self.tmpdirname     ,		**__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase			(		self							:     str):


           '''simple docstring'''

           shutil.rmtree(self.tmpdirname)



       def 		_lowerCamelCase			(		self							:     List[Any]):


           '''simple docstring'''

           __a						       =				np.random.randint(255     ,		size=(3, 30, 400)     ,		dtype=np.uinta)

           __a						       =				Image.fromarray(np.moveaxis(__SCREAMING_SNAKE_CASE     ,		0     ,		-1))

           return image_input



       def 		_lowerCamelCase			(		self							:     Union[str, Any]):


           '''simple docstring'''

           __a						       =				self.get_tokenizer()
           __a						       =				self.get_image_processor()

           __a						       =				MgpstrProcessor(tokenizer=__SCREAMING_SNAKE_CASE     ,		image_processor=__SCREAMING_SNAKE_CASE)
           processor.save_pretrained(self.tmpdirname)
           __a						       =				MgpstrProcessor.from_pretrained(self.tmpdirname     ,		use_fast=__SCREAMING_SNAKE_CASE)

           self.assertEqual(processor.char_tokenizer.get_vocab()     ,		tokenizer.get_vocab())
           self.assertIsInstance(processor.char_tokenizer     ,		__SCREAMING_SNAKE_CASE)

           self.assertEqual(processor.image_processor.to_json_string()     ,		image_processor.to_json_string())
           self.assertIsInstance(processor.image_processor     ,		__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase			(		self							:     Union[str, Any]):


           '''simple docstring'''

           __a						       =				self.get_tokenizer()
           __a						       =				self.get_image_processor()

           __a						       =				MgpstrProcessor(tokenizer=__SCREAMING_SNAKE_CASE     ,		image_processor=__SCREAMING_SNAKE_CASE)
           processor.save_pretrained(self.tmpdirname)

           __a						       =				self.get_tokenizer(bos_token='''(BOS)'''     ,		eos_token='''(EOS)''')
           __a						       =				self.get_image_processor(do_normalize=__SCREAMING_SNAKE_CASE     ,		padding_value=1.0)

           __a						       =				MgpstrProcessor.from_pretrained(
               self.tmpdirname     ,		bos_token='''(BOS)'''     ,		eos_token='''(EOS)'''     ,		do_normalize=__SCREAMING_SNAKE_CASE     ,		padding_value=1.0)

           self.assertEqual(processor.char_tokenizer.get_vocab()     ,		tokenizer_add_kwargs.get_vocab())
           self.assertIsInstance(processor.char_tokenizer     ,		__SCREAMING_SNAKE_CASE)

           self.assertEqual(processor.image_processor.to_json_string()     ,		image_processor_add_kwargs.to_json_string())
           self.assertIsInstance(processor.image_processor     ,		__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase			(		self							:     int):


           '''simple docstring'''

           __a						       =				self.get_image_processor()
           __a						       =				self.get_tokenizer()

           __a						       =				MgpstrProcessor(tokenizer=__SCREAMING_SNAKE_CASE     ,		image_processor=__SCREAMING_SNAKE_CASE)

           __a						       =				self.prepare_image_inputs()

           __a						       =				image_processor(__SCREAMING_SNAKE_CASE     ,		return_tensors='''np''')
           __a						       =				processor(images=__SCREAMING_SNAKE_CASE     ,		return_tensors='''np''')

           for key in input_image_proc.keys():
               self.assertAlmostEqual(input_image_proc[key].sum()     ,		input_processor[key].sum()     ,		delta=1E-2)



       def 		_lowerCamelCase			(		self							:     Dict):


           '''simple docstring'''

           __a						       =				self.get_image_processor()
           __a						       =				self.get_tokenizer()

           __a						       =				MgpstrProcessor(tokenizer=__SCREAMING_SNAKE_CASE     ,		image_processor=__SCREAMING_SNAKE_CASE)

           __a						       =				'''test'''

           __a						       =				processor(text=__SCREAMING_SNAKE_CASE)

           __a						       =				tokenizer(__SCREAMING_SNAKE_CASE)
           for key in encoded_tok.keys():
               self.assertListEqual(encoded_tok[key]     ,		encoded_processor[key])



       def 		_lowerCamelCase			(		self							:     Union[str, Any]):


           '''simple docstring'''

           __a						       =				self.get_image_processor()
           __a						       =				self.get_tokenizer()

           __a						       =				MgpstrProcessor(tokenizer=__SCREAMING_SNAKE_CASE     ,		image_processor=__SCREAMING_SNAKE_CASE)

           __a						       =				'''test'''
           __a						       =				self.prepare_image_inputs()

           __a						       =				processor(text=__SCREAMING_SNAKE_CASE     ,		images=__SCREAMING_SNAKE_CASE)

           self.assertListEqual(list(inputs.keys())     ,		['''pixel_values''', '''labels'''])

           # test if it raises when no input is passed
           with pytest.raises(__SCREAMING_SNAKE_CASE):
               processor()



       def 		_lowerCamelCase			(		self							:     Optional[Any]):


           '''simple docstring'''

           __a						       =				self.get_image_processor()
           __a						       =				self.get_tokenizer()

           __a						       =				MgpstrProcessor(tokenizer=__SCREAMING_SNAKE_CASE     ,		image_processor=__SCREAMING_SNAKE_CASE)

           __a						       =				[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9], [3, 4, 3, 1, 1, 8, 9]]

           __a						       =				processor.char_decode(__SCREAMING_SNAKE_CASE)
           __a						       =				tokenizer.batch_decode(__SCREAMING_SNAKE_CASE)
           __a						       =				[seq.replace(''' '''     ,		'''''') for seq in decoded_tok]

           self.assertListEqual(__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase			(		self							:     Optional[Any]):


           '''simple docstring'''

           __a						       =				self.get_image_processor()
           __a						       =				self.get_tokenizer()

           __a						       =				MgpstrProcessor(tokenizer=__SCREAMING_SNAKE_CASE     ,		image_processor=__SCREAMING_SNAKE_CASE)

           __a						       =				None
           __a						       =				self.prepare_image_inputs()

           __a						       =				processor(text=__SCREAMING_SNAKE_CASE     ,		images=__SCREAMING_SNAKE_CASE)

           self.assertListEqual(list(inputs.keys())     ,		processor.model_input_names)





       def 		_lowerCamelCase			(		self							:     Union[str, Any]):


           '''simple docstring'''

           __a						       =				self.get_image_processor()
           __a						       =				self.get_tokenizer()

           __a						       =				MgpstrProcessor(tokenizer=__SCREAMING_SNAKE_CASE     ,		image_processor=__SCREAMING_SNAKE_CASE)

           __a						       =				torch.randn(1     ,		27     ,		38)
           __a						       =				torch.randn(1     ,		27     ,		50_257)
           __a						       =				torch.randn(1     ,		27     ,		30_522)

           __a						       =				processor.batch_decode([char_input, bpe_input, wp_input])

           self.assertListEqual(list(results.keys())     ,		['''generated_text''', '''scores''', '''char_preds''', '''bpe_preds''', '''wp_preds'''])









"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__UpperCamelCase						=      logging.get_logger(__name__)

__UpperCamelCase						=      {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__UpperCamelCase						=      [
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

__UpperCamelCase						=      {
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
__UpperCamelCase						=      {f'''funnel-transformer/{name}''': 512 for name in _model_names}
__UpperCamelCase						=      {f'''funnel-transformer/{name}''': {'''do_lower_case''': True} for name in _model_names}
class 					lowerCAmelCase	(					lowerCamelCase_    ):




		'''simple docstring'''







		SCREAMING_SNAKE_CASE_						:     int								=     VOCAB_FILES_NAMES
		SCREAMING_SNAKE_CASE_						:     List[str]								=     PRETRAINED_VOCAB_FILES_MAP
		SCREAMING_SNAKE_CASE_						:     List[Any]								=     PRETRAINED_INIT_CONFIGURATION
		SCREAMING_SNAKE_CASE_						:     str								=     FunnelTokenizer
		SCREAMING_SNAKE_CASE_						:     Dict								=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		SCREAMING_SNAKE_CASE_						:     int								=     2







		def __init__(	self						,       lowerCAmelCase__=None						,       lowerCAmelCase__=None						,       lowerCAmelCase__=True						,       lowerCAmelCase__="<unk>"						,       lowerCAmelCase__="<sep>"						,       lowerCAmelCase__="<pad>"						,       lowerCAmelCase__="<cls>"						,       lowerCAmelCase__="<mask>"						,       lowerCAmelCase__="<s>"						,       lowerCAmelCase__="</s>"						,       lowerCAmelCase__=True						,       lowerCAmelCase__=True						,       lowerCAmelCase__=None						,       lowerCAmelCase__="##"						,       **lowerCAmelCase__						,       )     ->							Tuple:
								super().__init__(
								    lowerCAmelCase__						,       tokenizer_file=lowerCAmelCase__						,       do_lower_case=lowerCAmelCase__						,       unk_token=lowerCAmelCase__						,       sep_token=lowerCAmelCase__						,       pad_token=lowerCAmelCase__						,       cls_token=lowerCAmelCase__						,       mask_token=lowerCAmelCase__						,       bos_token=lowerCAmelCase__						,       eos_token=lowerCAmelCase__						,       clean_text=lowerCAmelCase__						,       tokenize_chinese_chars=lowerCAmelCase__						,       strip_accents=lowerCAmelCase__						,       wordpieces_prefix=lowerCAmelCase__						,       **lowerCAmelCase__						,       )

								SCREAMING_SNAKE_CASE													=					json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
								if (
								    normalizer_state.get('lowercase'						,       lowerCAmelCase__				) != do_lower_case
								    or normalizer_state.get('strip_accents'						,       lowerCAmelCase__				) != strip_accents
								    or normalizer_state.get('handle_chinese_chars'						,       lowerCAmelCase__				) != tokenize_chinese_chars
								):
														SCREAMING_SNAKE_CASE													=					getattr(lowerCAmelCase__						,       normalizer_state.pop('type'				)				)
														SCREAMING_SNAKE_CASE													=					do_lower_case
														SCREAMING_SNAKE_CASE													=					strip_accents
														SCREAMING_SNAKE_CASE													=					tokenize_chinese_chars
														SCREAMING_SNAKE_CASE													=					normalizer_class(**lowerCAmelCase__				)

								SCREAMING_SNAKE_CASE													=					do_lower_case







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__=None				)     ->							Optional[Any]:
								SCREAMING_SNAKE_CASE													=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
														output += token_ids_a + [self.sep_token_id]

								return output







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__ = None				)     ->							List[int]:
								SCREAMING_SNAKE_CASE													=					[self.sep_token_id]
								SCREAMING_SNAKE_CASE													=					[self.cls_token_id]
								if token_ids_a is None:
														return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0]
								return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__ = None				)     ->							Tuple[str]:
								SCREAMING_SNAKE_CASE													=					self._tokenizer.model.save(lowerCAmelCase__						,       name=lowerCAmelCase__				)
								return tuple(lowerCAmelCase__				)


