



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


__A							       =		{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__A							       =		{
    "vocab_file": {
        "unc-nlp/lxmert-base-uncased": "https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "unc-nlp/lxmert-base-uncased": (
            "https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json"
        ),
    },
}

__A							       =		{
    "unc-nlp/lxmert-base-uncased": 512,
}

__A							       =		{
    "unc-nlp/lxmert-base-uncased": {"do_lower_case": True},
}






class 						lowercase		(					__UpperCamelCase):






	"""simple docstring"""


	a__ :   List[str]						  =				VOCAB_FILES_NAMES
	a__ :   List[str]						  =				PRETRAINED_VOCAB_FILES_MAP
	a__ :   str						  =				PRETRAINED_INIT_CONFIGURATION
	a__ :   Dict						  =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a__ :   int						  =				LxmertTokenizer




	def __init__(      self      :		Optional[int]		,    __UpperCAmelCase      :		Tuple=None		,    __UpperCAmelCase      :		List[Any]=None		,    __UpperCAmelCase      :		Optional[int]=True		,    __UpperCAmelCase      :		List[str]="[UNK]"		,    __UpperCAmelCase      :		Optional[int]="[SEP]"		,    __UpperCAmelCase      :		Any="[PAD]"		,    __UpperCAmelCase      :		List[Any]="[CLS]"		,    __UpperCAmelCase      :		Any="[MASK]"		,    __UpperCAmelCase      :		Dict=True		,    __UpperCAmelCase      :		Dict=None		,    **__UpperCAmelCase      :		str		,    )     ->					Dict:
							super().__init__(
							    snake_case__		,    tokenizer_file=snake_case__		,    do_lower_case=snake_case__		,    unk_token=snake_case__		,    sep_token=snake_case__		,    pad_token=snake_case__		,    cls_token=snake_case__		,    mask_token=snake_case__		,    tokenize_chinese_chars=snake_case__		,    strip_accents=snake_case__		,    **snake_case__		,    )

							UpperCAmelCase_=  json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
							if (
							    normalizer_state.get("""lowercase"""		,    snake_case__	) != do_lower_case
							    or normalizer_state.get("""strip_accents"""		,    snake_case__	) != strip_accents
							    or normalizer_state.get("""handle_chinese_chars"""		,    snake_case__	) != tokenize_chinese_chars
							):
													UpperCAmelCase_=  getattr(snake_case__		,    normalizer_state.pop("""type"""	)	)
													UpperCAmelCase_=  do_lower_case
													UpperCAmelCase_=  strip_accents
													UpperCAmelCase_=  tokenize_chinese_chars
													UpperCAmelCase_=  normalizer_class(**snake_case__	)

							UpperCAmelCase_=  do_lower_case




	def     _SCREAMING_SNAKE_CASE						(      self      :		Union[str, Any]		,    __UpperCAmelCase      :		List[str]		,    __UpperCAmelCase      :		int=None	)     ->					int:
							UpperCAmelCase_=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

							return output




	def     _SCREAMING_SNAKE_CASE						(      self      :		List[Any]		,    __UpperCAmelCase      :		List[int]		,    __UpperCAmelCase      :		Optional[List[int]] = None	)     ->					Optional[int]:
							UpperCAmelCase_=  [self.sep_token_id]
							UpperCAmelCase_=  [self.cls_token_id]
							if token_ids_a is None:
													return len(cls + token_ids_a + sep	) * [0]
							return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]




	def     _SCREAMING_SNAKE_CASE						(      self      :		str		,    __UpperCAmelCase      :		str		,    __UpperCAmelCase      :		Optional[str] = None	)     ->					str:
							UpperCAmelCase_=  self._tokenizer.model.save(snake_case__		,    name=snake_case__	)
							return tuple(snake_case__	)





"""simple docstring"""




import time
from contextlib import contextmanager
from pathlib import Path

import pytest
import requests
from huggingface_hub.hf_api import HfApi, HfFolder


__lowerCAmelCase     :				str    				=    "__DUMMY_TRANSFORMERS_USER__"
__lowerCAmelCase     :				Dict    				=    "Dummy User"
__lowerCAmelCase     :				Optional[Any]    				=    "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"

__lowerCAmelCase     :				str    				=    "https://hub-ci.huggingface.co"
__lowerCAmelCase     :				int    				=    CI_HUB_ENDPOINT + "/datasets/{repo_id}/resolve/{revision}/{path}"
__lowerCAmelCase     :				Dict    				=    CI_HUB_ENDPOINT + "/{repo_id}/resolve/{revision}/{filename}"
__lowerCAmelCase     :				Optional[Any]    				=    Path("~/.huggingface/hub_ci_token").expanduser()







@pytest.fixture
def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		monkeypatch.setattr(
		    """huggingface_hub.file_download.HUGGINGFACE_CO_URL_TEMPLATE"""      ,	lowerCamelCase__  )







@pytest.fixture
def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		monkeypatch.setattr("""datasets.config.HF_ENDPOINT"""      ,	lowerCamelCase__  )
		monkeypatch.setattr("""datasets.config.HUB_DATASETS_URL"""      ,	lowerCamelCase__  )







@pytest.fixture
def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		monkeypatch.setattr("""huggingface_hub.hf_api.HfFolder.path_token"""      ,	lowerCamelCase__  )







@pytest.fixture
def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		HfFolder.save_token(lowerCamelCase__  )
		yield
		HfFolder.delete_token()







@pytest.fixture(scope="""session"""  )
def 				_UpperCAmelCase		(							):





		"""simple docstring"""




		return HfApi(endpoint=lowerCamelCase__  )







@pytest.fixture(scope="""session"""  )
def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		lowerCAmelCase__   = HfFolder.get_token()
		HfFolder.save_token(lowerCamelCase__  )
		yield CI_HUB_USER_TOKEN
		if previous_token is not None:
				HfFolder.save_token(lowerCamelCase__  )







@pytest.fixture
def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		def _cleanup_repo(lowerCamelCase__  ):
				hf_api.delete_repo(lowerCamelCase__      ,	token=lowerCamelCase__      ,	repo_type="""dataset"""  )

		return _cleanup_repo







@pytest.fixture
def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		@contextmanager
		def _temporary_repo(lowerCamelCase__  ):
				try:
						yield repo_id
				finally:
						cleanup_repo(lowerCamelCase__  )

		return _temporary_repo







@pytest.fixture(scope="""session"""  )
def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		lowerCAmelCase__   = f"""repo_txt_data-{int(time.time() * 10e3  )}"""
		lowerCAmelCase__   = f"""{CI_HUB_USER}/{repo_name}"""
		hf_api.create_repo(lowerCamelCase__      ,	token=lowerCamelCase__      ,	repo_type="""dataset"""      ,	private=lowerCamelCase__  )
		hf_api.upload_file(
		    token=lowerCamelCase__      ,	path_or_fileobj=str(lowerCamelCase__  )      ,	path_in_repo="""data/text_data.txt"""      ,	repo_id=lowerCamelCase__      ,	repo_type="""dataset"""      ,	)
		yield repo_id
		try:
				hf_api.delete_repo(lowerCamelCase__      ,	token=lowerCamelCase__      ,	repo_type="""dataset"""  )
		except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
				pass







@pytest.fixture()
def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		return hf_private_dataset_repo_txt_data_







@pytest.fixture(scope="""session"""  )
def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		lowerCAmelCase__   = f"""repo_zipped_txt_data-{int(time.time() * 10e3  )}"""
		lowerCAmelCase__   = f"""{CI_HUB_USER}/{repo_name}"""
		hf_api.create_repo(lowerCamelCase__      ,	token=lowerCamelCase__      ,	repo_type="""dataset"""      ,	private=lowerCamelCase__  )
		hf_api.upload_file(
		    token=lowerCamelCase__      ,	path_or_fileobj=str(lowerCamelCase__  )      ,	path_in_repo="""data.zip"""      ,	repo_id=lowerCamelCase__      ,	repo_type="""dataset"""      ,	)
		yield repo_id
		try:
				hf_api.delete_repo(lowerCamelCase__      ,	token=lowerCamelCase__      ,	repo_type="""dataset"""  )
		except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
				pass







@pytest.fixture()
def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		return hf_private_dataset_repo_zipped_txt_data_







@pytest.fixture(scope="""session"""  )
def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		lowerCAmelCase__   = f"""repo_zipped_img_data-{int(time.time() * 10e3  )}"""
		lowerCAmelCase__   = f"""{CI_HUB_USER}/{repo_name}"""
		hf_api.create_repo(lowerCamelCase__      ,	token=lowerCamelCase__      ,	repo_type="""dataset"""      ,	private=lowerCamelCase__  )
		hf_api.upload_file(
		    token=lowerCamelCase__      ,	path_or_fileobj=str(lowerCamelCase__  )      ,	path_in_repo="""data.zip"""      ,	repo_id=lowerCamelCase__      ,	repo_type="""dataset"""      ,	)
		yield repo_id
		try:
				hf_api.delete_repo(lowerCamelCase__      ,	token=lowerCamelCase__      ,	repo_type="""dataset"""  )
		except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
				pass







@pytest.fixture()
def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		return hf_private_dataset_repo_zipped_img_data_




