


"""simple docstring"""




import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 					lowerCAmelCase	(					lowerCamelCase_	,    unittest.TestCase    ):




		'''simple docstring'''







		SCREAMING_SNAKE_CASE_						:     Tuple								=     LEDTokenizer
		SCREAMING_SNAKE_CASE_						:     int								=     LEDTokenizerFast
		SCREAMING_SNAKE_CASE_						:     Any								=     True







		def 							__A						(	self				)     ->							str:
								super().setUp()
								SCREAMING_SNAKE_CASE													=					[
								    'l',
								    'o',
								    'w',
								    'e',
								    'r',
								    's',
								    't',
								    'i',
								    'd',
								    'n',
								    '\u0120',
								    '\u0120l',
								    '\u0120n',
								    '\u0120lo',
								    '\u0120low',
								    'er',
								    '\u0120lowest',
								    '\u0120newer',
								    '\u0120wider',
								    '<unk>',
								]
								SCREAMING_SNAKE_CASE													=					dict(zip(lowerCAmelCase__						,       range(len(lowerCAmelCase__				)				)				)				)
								SCREAMING_SNAKE_CASE													=					['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
								SCREAMING_SNAKE_CASE													=					{'unk_token': '<unk>'}

								SCREAMING_SNAKE_CASE													=					os.path.join(self.tmpdirname						,       VOCAB_FILES_NAMES['vocab_file']				)
								SCREAMING_SNAKE_CASE													=					os.path.join(self.tmpdirname						,       VOCAB_FILES_NAMES['merges_file']				)
								with open(self.vocab_file						,       'w'						,       encoding='utf-8'				) as fp:
														fp.write(json.dumps(lowerCAmelCase__				) + '\n'				)
								with open(self.merges_file						,       'w'						,       encoding='utf-8'				) as fp:
														fp.write('\n'.join(lowerCAmelCase__				)				)







		def 							__A						(	self						,       **lowerCAmelCase__				)     ->							Optional[int]:
								kwargs.update(self.special_tokens_map				)
								return self.tokenizer_class.from_pretrained(self.tmpdirname						,       **lowerCAmelCase__				)







		def 							__A						(	self						,       **lowerCAmelCase__				)     ->							List[str]:
								kwargs.update(self.special_tokens_map				)
								return self.rust_tokenizer_class.from_pretrained(self.tmpdirname						,       **lowerCAmelCase__				)







		def 							__A						(	self						,       lowerCAmelCase__				)     ->							Optional[int]:
								return "lower newer", "lower newer"







		@cached_property
		def 							__A						(	self				)     ->							Optional[int]:
								return LEDTokenizer.from_pretrained('allenai/led-base-16384'				)







		@cached_property
		def 							__A						(	self				)     ->							List[str]:
								return LEDTokenizerFast.from_pretrained('allenai/led-base-16384'				)







		@require_torch
		def 							__A						(	self				)     ->							Union[str, Any]:
								SCREAMING_SNAKE_CASE													=					['A long paragraph for summarization.', 'Another paragraph for summarization.']
								SCREAMING_SNAKE_CASE													=					[0, 250, 251, 17_818, 13, 39_186, 1_938, 4, 2]

								for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
														SCREAMING_SNAKE_CASE													=					tokenizer(lowerCAmelCase__						,       max_length=len(lowerCAmelCase__				)						,       padding=lowerCAmelCase__						,       return_tensors='pt'				)
														self.assertIsInstance(lowerCAmelCase__						,       lowerCAmelCase__				)

														self.assertEqual((2, 9)						,       batch.input_ids.shape				)
														self.assertEqual((2, 9)						,       batch.attention_mask.shape				)
														SCREAMING_SNAKE_CASE													=					batch.input_ids.tolist()[0]
														self.assertListEqual(lowerCAmelCase__						,       lowerCAmelCase__				)







		@require_torch
		def 							__A						(	self				)     ->							Tuple:
								SCREAMING_SNAKE_CASE													=					['A long paragraph for summarization.', 'Another paragraph for summarization.']
								for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
														SCREAMING_SNAKE_CASE													=					tokenizer(lowerCAmelCase__						,       padding=lowerCAmelCase__						,       return_tensors='pt'				)
														self.assertIn('input_ids'						,       lowerCAmelCase__				)
														self.assertIn('attention_mask'						,       lowerCAmelCase__				)
														self.assertNotIn('labels'						,       lowerCAmelCase__				)
														self.assertNotIn('decoder_attention_mask'						,       lowerCAmelCase__				)







		@require_torch
		def 							__A						(	self				)     ->							List[str]:
								SCREAMING_SNAKE_CASE													=					[
								    'Summary of the text.',
								    'Another summary.',
								]
								for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
														SCREAMING_SNAKE_CASE													=					tokenizer(text_target=lowerCAmelCase__						,       max_length=32						,       padding='max_length'						,       return_tensors='pt'				)
														self.assertEqual(32						,       targets['input_ids'].shape[1]				)







		@require_torch
		def 							__A						(	self				)     ->							str:
								for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
														SCREAMING_SNAKE_CASE													=					tokenizer(
														    ['I am a small frog' * 1_024, 'I am a small frog']						,       padding=lowerCAmelCase__						,       truncation=lowerCAmelCase__						,       return_tensors='pt'				)
														self.assertIsInstance(lowerCAmelCase__						,       lowerCAmelCase__				)
														self.assertEqual(batch.input_ids.shape						,       (2, 5_122)				)







		@require_torch
		def 							__A						(	self				)     ->							Optional[int]:
								SCREAMING_SNAKE_CASE													=					['A long paragraph for summarization.']
								SCREAMING_SNAKE_CASE													=					[
								    'Summary of the text.',
								]
								for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
														SCREAMING_SNAKE_CASE													=					tokenizer(lowerCAmelCase__						,       return_tensors='pt'				)
														SCREAMING_SNAKE_CASE													=					tokenizer(text_target=lowerCAmelCase__						,       return_tensors='pt'				)
														SCREAMING_SNAKE_CASE													=					inputs['input_ids']
														SCREAMING_SNAKE_CASE													=					targets['input_ids']
														self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()				)
														self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()				)
														self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()				)
														self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()				)







		@require_torch
		def 							__A						(	self				)     ->							List[Any]:
								for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
														SCREAMING_SNAKE_CASE													=					['Summary of the text.', 'Another summary.']
														SCREAMING_SNAKE_CASE													=					[[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

														SCREAMING_SNAKE_CASE													=					tokenizer(lowerCAmelCase__						,       padding=lowerCAmelCase__				)
														SCREAMING_SNAKE_CASE													=					[[0] * len(lowerCAmelCase__				) for x in encoded_output['input_ids']]
														SCREAMING_SNAKE_CASE													=					tokenizer.pad(lowerCAmelCase__				)
														self.assertSequenceEqual(outputs['global_attention_mask']						,       lowerCAmelCase__				)







		def 							__A						(	self				)     ->							List[str]:
								pass







		def 							__A						(	self				)     ->							str:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'				):
																				SCREAMING_SNAKE_CASE													=					self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__						,       **lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE													=					self.tokenizer_class.from_pretrained(lowerCAmelCase__						,       **lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE													=					'A, <mask> AllenNLP sentence.'
																				SCREAMING_SNAKE_CASE													=					tokenizer_r.encode_plus(lowerCAmelCase__						,       add_special_tokens=lowerCAmelCase__						,       return_token_type_ids=lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE													=					tokenizer_p.encode_plus(lowerCAmelCase__						,       add_special_tokens=lowerCAmelCase__						,       return_token_type_ids=lowerCAmelCase__				)
																				self.assertEqual(sum(tokens_r['token_type_ids']				)						,       sum(tokens_p['token_type_ids']				)				)
																				self.assertEqual(
																				    sum(tokens_r['attention_mask']				) / len(tokens_r['attention_mask']				)						,       sum(tokens_p['attention_mask']				) / len(tokens_p['attention_mask']				)						,       )

																				SCREAMING_SNAKE_CASE													=					tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']				)
																				SCREAMING_SNAKE_CASE													=					tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']				)
																				self.assertSequenceEqual(tokens_p['input_ids']						,       [0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]				)
																				self.assertSequenceEqual(tokens_r['input_ids']						,       [0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]				)

																				self.assertSequenceEqual(
																				    lowerCAmelCase__						,       ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']				)
																				self.assertSequenceEqual(
																				    lowerCAmelCase__						,       ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']				)






"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__UpperCamelCase						=      logging.get_logger(__name__)

__UpperCamelCase						=      {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__UpperCamelCase						=      [
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

__UpperCamelCase						=      {
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
__UpperCamelCase						=      {f'''funnel-transformer/{name}''': 512 for name in _model_names}
__UpperCamelCase						=      {f'''funnel-transformer/{name}''': {'''do_lower_case''': True} for name in _model_names}
class 					lowerCAmelCase	(					lowerCamelCase_    ):




		'''simple docstring'''







		SCREAMING_SNAKE_CASE_						:     int								=     VOCAB_FILES_NAMES
		SCREAMING_SNAKE_CASE_						:     List[str]								=     PRETRAINED_VOCAB_FILES_MAP
		SCREAMING_SNAKE_CASE_						:     List[Any]								=     PRETRAINED_INIT_CONFIGURATION
		SCREAMING_SNAKE_CASE_						:     str								=     FunnelTokenizer
		SCREAMING_SNAKE_CASE_						:     Dict								=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		SCREAMING_SNAKE_CASE_						:     int								=     2







		def __init__(	self						,       lowerCAmelCase__=None						,       lowerCAmelCase__=None						,       lowerCAmelCase__=True						,       lowerCAmelCase__="<unk>"						,       lowerCAmelCase__="<sep>"						,       lowerCAmelCase__="<pad>"						,       lowerCAmelCase__="<cls>"						,       lowerCAmelCase__="<mask>"						,       lowerCAmelCase__="<s>"						,       lowerCAmelCase__="</s>"						,       lowerCAmelCase__=True						,       lowerCAmelCase__=True						,       lowerCAmelCase__=None						,       lowerCAmelCase__="##"						,       **lowerCAmelCase__						,       )     ->							Tuple:
								super().__init__(
								    lowerCAmelCase__						,       tokenizer_file=lowerCAmelCase__						,       do_lower_case=lowerCAmelCase__						,       unk_token=lowerCAmelCase__						,       sep_token=lowerCAmelCase__						,       pad_token=lowerCAmelCase__						,       cls_token=lowerCAmelCase__						,       mask_token=lowerCAmelCase__						,       bos_token=lowerCAmelCase__						,       eos_token=lowerCAmelCase__						,       clean_text=lowerCAmelCase__						,       tokenize_chinese_chars=lowerCAmelCase__						,       strip_accents=lowerCAmelCase__						,       wordpieces_prefix=lowerCAmelCase__						,       **lowerCAmelCase__						,       )

								SCREAMING_SNAKE_CASE													=					json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
								if (
								    normalizer_state.get('lowercase'						,       lowerCAmelCase__				) != do_lower_case
								    or normalizer_state.get('strip_accents'						,       lowerCAmelCase__				) != strip_accents
								    or normalizer_state.get('handle_chinese_chars'						,       lowerCAmelCase__				) != tokenize_chinese_chars
								):
														SCREAMING_SNAKE_CASE													=					getattr(lowerCAmelCase__						,       normalizer_state.pop('type'				)				)
														SCREAMING_SNAKE_CASE													=					do_lower_case
														SCREAMING_SNAKE_CASE													=					strip_accents
														SCREAMING_SNAKE_CASE													=					tokenize_chinese_chars
														SCREAMING_SNAKE_CASE													=					normalizer_class(**lowerCAmelCase__				)

								SCREAMING_SNAKE_CASE													=					do_lower_case







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__=None				)     ->							Optional[Any]:
								SCREAMING_SNAKE_CASE													=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
														output += token_ids_a + [self.sep_token_id]

								return output







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__ = None				)     ->							List[int]:
								SCREAMING_SNAKE_CASE													=					[self.sep_token_id]
								SCREAMING_SNAKE_CASE													=					[self.cls_token_id]
								if token_ids_a is None:
														return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0]
								return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__ = None				)     ->							Tuple[str]:
								SCREAMING_SNAKE_CASE													=					self._tokenizer.model.save(lowerCAmelCase__						,       name=lowerCAmelCase__				)
								return tuple(lowerCAmelCase__				)


