
import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XGLMTokenizer, XGLMTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowercase__		:Union[str, Any]   =    get_tests_dir("fixtures/test_sentencepiece.model")







@require_sentencepiece
@require_tokenizers
class 				lowercase		(					SCREAMING_SNAKE_CASE__       ,  unittest.TestCase		):
							lowercase_       :       Union[str, Any]       					=XGLMTokenizer
							lowercase_       :       Tuple       					=XGLMTokenizerFast
							lowercase_       :       Optional[Any]       					=True
							lowercase_       :       Dict       					=True




							def  A__      (				self):
												super().setUp()

												# We have a SentencePiece fixture for testing
												lowercase							=						XGLMTokenizer(A__   ,keep_accents=A__)
												tokenizer.save_pretrained(self.tmpdirname)




							def  A__      (				self):
												lowercase							=						'''<pad>'''
												lowercase							=						1

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(A__)   ,A__)
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(A__)   ,A__)




							def  A__      (				self):
												lowercase							=						list(self.get_tokenizer().get_vocab().keys())

												self.assertEqual(vocab_keys[0]   ,'''<s>''')
												self.assertEqual(vocab_keys[1]   ,'''<pad>''')
												self.assertEqual(len(A__)   ,1_0_0_8)




							def  A__      (				self):
												self.assertEqual(self.get_tokenizer().vocab_size   ,1_0_0_8)




							def  A__      (				self):
												lowercase							=						XGLMTokenizer(A__   ,keep_accents=A__)

												lowercase							=						tokenizer.tokenize('''This is a test''')
												self.assertListEqual(A__   ,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

												self.assertListEqual(
												    tokenizer.convert_tokens_to_ids(A__)   ,[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]   ,)

												lowercase							=						tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
												self.assertListEqual(
												    A__   ,[
												        SPIECE_UNDERLINE + '''I''',
												        SPIECE_UNDERLINE + '''was''',
												        SPIECE_UNDERLINE + '''b''',
												        '''or''',
												        '''n''',
												        SPIECE_UNDERLINE + '''in''',
												        SPIECE_UNDERLINE + '''''',
												        '''9''',
												        '''2''',
												        '''0''',
												        '''0''',
												        '''0''',
												        ''',''',
												        SPIECE_UNDERLINE + '''and''',
												        SPIECE_UNDERLINE + '''this''',
												        SPIECE_UNDERLINE + '''is''',
												        SPIECE_UNDERLINE + '''f''',
												        '''al''',
												        '''s''',
												        '''é''',
												        '''.''',
												    ]   ,)
												lowercase							=						tokenizer.convert_tokens_to_ids(A__)
												self.assertListEqual(
												    A__   ,[
												        value + tokenizer.fairseq_offset
												        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
												    ]   ,)

												lowercase							=						tokenizer.convert_ids_to_tokens(A__)
												self.assertListEqual(
												    A__   ,[
												        SPIECE_UNDERLINE + '''I''',
												        SPIECE_UNDERLINE + '''was''',
												        SPIECE_UNDERLINE + '''b''',
												        '''or''',
												        '''n''',
												        SPIECE_UNDERLINE + '''in''',
												        SPIECE_UNDERLINE + '''''',
												        '''<unk>''',
												        '''2''',
												        '''0''',
												        '''0''',
												        '''0''',
												        ''',''',
												        SPIECE_UNDERLINE + '''and''',
												        SPIECE_UNDERLINE + '''this''',
												        SPIECE_UNDERLINE + '''is''',
												        SPIECE_UNDERLINE + '''f''',
												        '''al''',
												        '''s''',
												        '''<unk>''',
												        '''.''',
												    ]   ,)




							@cached_property
							def  A__      (				self):
												return XGLMTokenizer.from_pretrained('''facebook/xglm-564M''')




							def  A__      (				self):
												with tempfile.NamedTemporaryFile() as f:
																	shutil.copyfile(A__   ,f.name)
																	lowercase							=						XGLMTokenizer(f.name   ,keep_accents=A__)
																	lowercase							=						pickle.dumps(A__)
												pickle.loads(A__)




							def  A__      (				self):
												if not self.test_rust_tokenizer:
																	return

												lowercase							=						self.get_tokenizer()
												lowercase							=						self.get_rust_tokenizer()

												lowercase							=						'''I was born in 92000, and this is falsé.'''

												lowercase							=						tokenizer.tokenize(A__)
												lowercase							=						rust_tokenizer.tokenize(A__)
												self.assertListEqual(A__   ,A__)

												lowercase							=						tokenizer.encode(A__   ,add_special_tokens=A__)
												lowercase							=						rust_tokenizer.encode(A__   ,add_special_tokens=A__)
												self.assertListEqual(A__   ,A__)

												lowercase							=						self.get_rust_tokenizer()
												lowercase							=						tokenizer.encode(A__)
												lowercase							=						rust_tokenizer.encode(A__)
												self.assertListEqual(A__   ,A__)




							@slow
							def  A__      (				self):
												lowercase							=						'''Hello World!'''
												lowercase							=						[2, 3_1_2_2_7, 4_4_4_7, 3_5]

												self.assertListEqual(A__   ,self.big_tokenizer.encode(A__))




							@slow
							def  A__      (				self):
												lowercase							=						(
												    '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
												    ''' add words that should not exsist and be tokenized to unk, such as saoneuhaoesuth'''
												)
												# fmt: off
												lowercase							=						[2, 1_0_1_8, 6_7, 1_1, 1_9_8_8, 2_6_1_7, 5_6_3_1, 2_7_8, 1_1, 3_4_0_7, 4_8, 7_1_6_3_0, 2_8_0_8_5, 4, 3_2_3_4, 1_5_7, 1_3, 6, 5, 6, 4, 3_5_2_6, 7_6_8, 1_5, 6_5_9, 5_7, 2_9_8, 3_9_8_3, 8_6_4, 1_2_9, 2_1, 6, 5, 1_3_6_7_5, 3_7_7, 6_5_2, 7_5_8_0, 1_0_3_4_1, 1_5_5, 2_8_1_7, 4_2_2, 1_6_6_6, 7, 1_6_7_4, 5_3, 1_1_3, 2_0_2_2_7_7, 1_7_8_9_2, 3_3, 6_0, 8_7, 4, 3_2_3_4, 1_5_7, 6_1, 2_6_6_7, 5_2_3_7_6, 1_9, 8_8, 2_3, 7_3_5]
												# fmt: on

												self.assertListEqual(A__   ,self.big_tokenizer.encode(A__))





							@slow
							def  A__      (				self):
												# fmt: off
												lowercase							=						{
												    '''input_ids''': [[2, 1_0_8_8_2_5, 1_1_6_3, 1_5, 8_8_0_1_0, 4_7_3, 1_5_8_9_8, 1_5_7, 1_3_6_7_2, 1_8_5_7, 3_1_2, 8, 2_3_8_0_2_1, 1_1_6_3, 5_3, 1_3_6_7_2, 1_8_5_7, 3_1_2, 8, 5_3_2_8_3, 1_8_2_3_9_6, 8, 1_8_5_6_6, 1_6, 3_6_7_3_3, 4_1_0_1, 8, 2_3_0, 2_4_4_0_1_7, 1_2_2_5_5_3, 7, 1_5, 1_3_2_5_9_7, 4, 2_9_3, 1_2_5_1_1, 7_6_1_0, 4, 3_4_1_4, 1_3_2_5_9_7, 9, 4, 3_2_3_6_1, 3_6_2, 4, 7_3_4, 2_8_5_1_2, 3_2_5_6_9, 1_8, 4, 3_2_3_6_1, 2_6_0_9_6, 1_4_9_8_2, 7_3, 1_8_7_1_5, 2_1_4_3_3, 2_3_5_2_6_1, 1_5, 4_9_2, 1_2_4_2_7, 1_6, 5_3, 1_8_7_1_5, 2_1_4_3_3, 6_5_4_5_4, 1_5, 2_3_6_5_9, 5_6_3, 1_6, 2_7_8, 5_9_7, 2_8_4_3, 5_9_5, 7_9_3_1, 1_8_2_3_9_6, 6_4_1_8_6, 2_2, 8_8_6, 5_9_5, 1_3_2_9_8_1, 5_3, 2_5_5_4_0, 3_4_4_9, 4_3_9_8_2, 3_9_9_0_1, 5_9_5_1, 8_7_8, 3_3_0, 4, 2_7_6_9_4, 8_0_2_6_9, 3_1_2, 5_3, 6_5_1_7, 1_1_7_8_0, 6_1_1, 2_0_4_0_8, 5], [2, 6, 1_3_2_5_9_7, 6_7, 4_2_8_9_7, 3_3, 5_9_2, 8, 1_6_3_7_2_9, 2_5_5_4_0, 3_6_1, 1_3_6_9_9_7, 1_0_9_5_1_4, 1_7_3_2_3_0, 7, 5_0_1, 6_0, 1_0_2_9_1_3, 1_9_6, 5_6_3_1, 2_3_5, 6_3_2_4_3, 4_7_3, 6, 2_3_1_7_5_7, 7_4, 5_2_7_7, 7_9_0_5, 5_3, 3_0_9_5, 3_7_3_1_7, 2_2, 4_5_4, 1_8_3_8_7_4, 5], [2, 2_6_8, 3_1_2_9_8, 4_6_5_3_0, 6, 1_3_2_9_3_5, 4_3_8_3_1, 7, 5_9_7, 3_2, 2_4, 3_6_8_8, 9_8_6_5, 5]],
												    '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
												}  # noqa: E501
												# fmt: on

												self.tokenizer_integration_test_util(
												    expected_encoding=A__   ,model_name='''facebook/xglm-564M'''   ,padding=A__   ,)






import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__		:Dict   =    logging.get_logger(__name__)

lowercase__		:Optional[int]   =    {
    "BAAI/AltCLIP": "https://huggingface.co/BAAI/AltCLIP/resolve/main/config.json",
    # See all AltCLIP models at https://huggingface.co/models?filter=altclip
}







class 				lowercase		(					SCREAMING_SNAKE_CASE__		):
							lowercase_       :       Any       					='''altclip_text_model'''




							def __init__(				self   ,A__=2_5_0_0_0_2   ,A__=1_0_2_4   ,A__=2_4   ,A__=1_6   ,A__=4_0_9_6   ,A__="gelu"   ,A__=0.1   ,A__=0.1   ,A__=5_1_4   ,A__=1   ,A__=0.02   ,A__=0.02   ,A__=1E-05   ,A__=1   ,A__=0   ,A__=2   ,A__="absolute"   ,A__=True   ,A__=7_6_8   ,**A__   ,):
												super().__init__(pad_token_id=A__   ,bos_token_id=A__   ,eos_token_id=A__   ,**A__)

												lowercase							=						vocab_size
												lowercase							=						hidden_size
												lowercase							=						num_hidden_layers
												lowercase							=						num_attention_heads
												lowercase							=						hidden_act
												lowercase							=						intermediate_size
												lowercase							=						hidden_dropout_prob
												lowercase							=						attention_probs_dropout_prob
												lowercase							=						max_position_embeddings
												lowercase							=						type_vocab_size
												lowercase							=						initializer_range
												lowercase							=						initializer_factor
												lowercase							=						layer_norm_eps
												lowercase							=						position_embedding_type
												lowercase							=						use_cache
												lowercase							=						project_dim







class 				lowercase		(					SCREAMING_SNAKE_CASE__		):
							lowercase_       :       Optional[int]       					='''altclip_vision_model'''




							def __init__(				self   ,A__=7_6_8   ,A__=3_0_7_2   ,A__=5_1_2   ,A__=1_2   ,A__=1_2   ,A__=3   ,A__=2_2_4   ,A__=3_2   ,A__="quick_gelu"   ,A__=1E-5   ,A__=0.0   ,A__=0.02   ,A__=1.0   ,**A__   ,):
												super().__init__(**A__)

												lowercase							=						hidden_size
												lowercase							=						intermediate_size
												lowercase							=						projection_dim
												lowercase							=						num_hidden_layers
												lowercase							=						num_attention_heads
												lowercase							=						num_channels
												lowercase							=						patch_size
												lowercase							=						image_size
												lowercase							=						initializer_range
												lowercase							=						initializer_factor
												lowercase							=						attention_dropout
												lowercase							=						layer_norm_eps
												lowercase							=						hidden_act





							@classmethod
							def  A__      (				cls   ,A__   ,**A__):
												cls._set_token_in_kwargs(A__)

												lowercase	,						lowercase							=						cls.get_config_dict(A__   ,**A__)

												# get the vision config dict if we are loading from AltCLIPConfig
												if config_dict.get('''model_type''') == "altclip":
																	lowercase							=						config_dict['''vision_config''']

												if "model_type" in config_dict and hasattr(cls   ,'''model_type''') and config_dict["model_type"] != cls.model_type:
																	logger.warning(
																	    f'You are using a model of type {config_dict["model_type"]} to instantiate a model of type '
																	    f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.')

												return cls.from_dict(A__   ,**A__)







class 				lowercase		(					SCREAMING_SNAKE_CASE__		):
							lowercase_       :       Any       					='''altclip'''
							lowercase_       :       List[Any]       					=True




							def __init__(				self   ,A__=None   ,A__=None   ,A__=7_6_8   ,A__=2.6592   ,**A__):
												# If `_config_dict` exist, we use them for the backward compatibility.
												# We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot
												# of confusion!).
												lowercase							=						kwargs.pop('''text_config_dict'''   ,A__)
												lowercase							=						kwargs.pop('''vision_config_dict'''   ,A__)

												super().__init__(**A__)

												# Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in
												# `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most
												# cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`.
												if text_config_dict is not None:
																	if text_config is None:
																						lowercase							=						{}

																	# This is the complete result when using `text_config_dict`.
																	lowercase							=						AltCLIPTextConfig(**A__).to_dict()

																	# Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different.
																	for key, value in _text_config_dict.items():
																						if key in text_config and value != text_config[key] and key not in ["transformers_version"]:
																											# If specified in `text_config_dict`
																											if key in text_config_dict:
																																lowercase							=						(
																																    f'`{key}` is found in both `text_config_dict` and `text_config` but with different values. '
																																    f'The value `text_config_dict["{key}"]` will be used instead.'
																																)
																											# If inferred from default argument values (just to be super careful)
																											else:
																																lowercase							=						(
																																    f'`text_config_dict` is provided which will be used to initialize `AltCLIPTextConfig`. The '
																																    f'value `text_config["{key}"]` will be overriden.'
																																)
																											logger.warning(A__)

            # Update all values in `text_config` with the ones in `_text_config_dict`.
																	text_config.update(_text_config_dict)

												if vision_config_dict is not None:
																	if vision_config is None:
																						lowercase							=						{}

																	# This is the complete result when using `vision_config_dict`.
																	lowercase							=						AltCLIPVisionConfig(**A__).to_dict()
																	# convert keys to string instead of integer
																	if "id2label" in _vision_config_dict:
																						lowercase							=						{
																						    str(A__): value for key, value in _vision_config_dict['''id2label'''].items()
																						}

																	# Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different.
																	for key, value in _vision_config_dict.items():
																						if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]:
																											# If specified in `vision_config_dict`
																											if key in vision_config_dict:
																																lowercase							=						(
																																    f'`{key}` is found in both `vision_config_dict` and `vision_config` but with different '
																																    f'values. The value `vision_config_dict["{key}"]` will be used instead.'
																																)
																											# If inferred from default argument values (just to be super careful)
																											else:
																																lowercase							=						(
																																    f'`vision_config_dict` is provided which will be used to initialize `AltCLIPVisionConfig`. '
																																    f'The value `vision_config["{key}"]` will be overriden.'
																																)
																											logger.warning(A__)

            # Update all values in `vision_config` with the ones in `_vision_config_dict`.
																	vision_config.update(_vision_config_dict)

												if text_config is None:
																	lowercase							=						{}
																	logger.info('''`text_config` is `None`. Initializing the `AltCLIPTextConfig` with default values.''')

												if vision_config is None:
																	lowercase							=						{}
																	logger.info('''`vision_config` is `None`. initializing the `AltCLIPVisionConfig` with default values.''')

												lowercase							=						AltCLIPTextConfig(**A__)
												lowercase							=						AltCLIPVisionConfig(**A__)

												lowercase							=						projection_dim
												lowercase							=						logit_scale_init_value
												lowercase							=						1.0




							@classmethod
							def  A__      (				cls   ,A__   ,A__   ,**A__):
												return cls(text_config=text_config.to_dict()   ,vision_config=vision_config.to_dict()   ,**A__)





							def  A__      (				self):
												lowercase							=						copy.deepcopy(self.__dict__)
												lowercase							=						self.text_config.to_dict()
												lowercase							=						self.vision_config.to_dict()
												lowercase							=						self.__class__.model_type
												return output




