





"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


a		:					Any	      =     {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

a		:					Union[str, Any]	      =     {
    '''vocab_file''': {
        '''unc-nlp/lxmert-base-uncased''': '''https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''unc-nlp/lxmert-base-uncased''': (
            '''https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

a		:					List[str]	      =     {
    '''unc-nlp/lxmert-base-uncased''': 512,
}

a		:					Optional[Any]	      =     {
    '''unc-nlp/lxmert-base-uncased''': {'''do_lower_case''': True},
}


class 			__UpperCamelCase							(		a__		):
							lowerCamelCase  :   List[str]    =VOCAB_FILES_NAMES
							lowerCamelCase  :   List[Any]    =PRETRAINED_VOCAB_FILES_MAP
							lowerCamelCase  :   List[str]    =PRETRAINED_INIT_CONFIGURATION
							lowerCamelCase  :   Tuple    =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCamelCase  :   Union[str, Any]    =LxmertTokenizer






							def __init__(  self ,   lowerCAmelCase__=None ,   lowerCAmelCase__=None ,   lowerCAmelCase__=True ,   lowerCAmelCase__="[UNK]" ,   lowerCAmelCase__="[SEP]" ,   lowerCAmelCase__="[PAD]" ,   lowerCAmelCase__="[CLS]" ,   lowerCAmelCase__="[MASK]" ,   lowerCAmelCase__=True ,   lowerCAmelCase__=None ,   **lowerCAmelCase__ ,   )      ->					Optional[int]:
								super().__init__(
								    lowerCAmelCase__ ,   tokenizer_file=lowerCAmelCase__ ,   do_lower_case=lowerCAmelCase__ ,   unk_token=lowerCAmelCase__ ,   sep_token=lowerCAmelCase__ ,   pad_token=lowerCAmelCase__ ,   cls_token=lowerCAmelCase__ ,   mask_token=lowerCAmelCase__ ,   tokenize_chinese_chars=lowerCAmelCase__ ,   strip_accents=lowerCAmelCase__ ,   **lowerCAmelCase__ ,   )

								a :						Union[str, Any]      =							json.loads(self.backend_tokenizer.normalizer.__getstate__() )
								if (
								    normalizer_state.get("lowercase" ,   lowerCAmelCase__ ) != do_lower_case
								    or normalizer_state.get("strip_accents" ,   lowerCAmelCase__ ) != strip_accents
								    or normalizer_state.get("handle_chinese_chars" ,   lowerCAmelCase__ ) != tokenize_chinese_chars
								):
									a :						Dict      =							getattr(lowerCAmelCase__ ,   normalizer_state.pop("type" ) )
									a :						int      =							do_lower_case
									a :						List[Any]      =							strip_accents
									a :						str      =							tokenize_chinese_chars
									a :						Optional[int]      =							normalizer_class(**lowerCAmelCase__ )

								a :						Any      =							do_lower_case






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__=None )      ->					List[str]:
								a :						Optional[int]      =							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

								return output






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ = None )      ->					List[int]:
								a :						int      =							[self.sep_token_id]
								a :						int      =							[self.cls_token_id]
								if token_ids_a is None:
									return len(cls + token_ids_a + sep ) * [0]
								return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ = None )      ->					Tuple[str]:
								a :						Union[str, Any]      =							self._tokenizer.model.save(lowerCAmelCase__ ,   name=lowerCAmelCase__ )
								return tuple(lowerCAmelCase__ )









"""simple docstring"""




import gc
import threading
import time

import psutil
import torch


class 			__UpperCamelCase							:






							def __init__(  self )      ->					Any:
								a :						str      =							psutil.Process()
								a :						str      =							False






							def        __a  (  self )      ->					str:
								a :						List[Any]      =							-1

								while True:
									a :						str      =							max(self.process.memory_info().rss ,   self.cpu_memory_peak )

									# can't sleep or will not catch the peak right (this comment is here on purpose)
									if not self.peak_monitoring:
										break






							def        __a  (  self )      ->					List[str]:
								a :						Dict      =							True
								a :						Optional[Any]      =							threading.Thread(target=self.peak_monitor )
								a :						List[Any]      =							True
								self.thread.start()






							def        __a  (  self )      ->					List[str]:
								a :						Optional[int]      =							False
								self.thread.join()
								return self.cpu_memory_peak
a		:					Tuple	      =     PeakCPUMemory()






def  _SCREAMING_SNAKE_CASE						(      )					->List[Any]:






	'''simple docstring'''






	a :						List[str]      =							{"time": time.time()}

	gc.collect()
	torch.cuda.empty_cache()

	# CPU mem
	a :						Tuple      =							psutil.Process().memory_info().rss
	cpu_peak_tracker.start()

	# GPU mem
	for i in range(torch.cuda.device_count()   ):
		a :						Any      =							torch.cuda.memory_allocated(_lowercase   )
	torch.cuda.reset_peak_memory_stats()

	return measures






def  _SCREAMING_SNAKE_CASE						(      _lowercase		:       List[str]   )					->Union[str, Any]:






	'''simple docstring'''






	a :						int      =							{"time": time.time() - start_measures["time"]}

	gc.collect()
	torch.cuda.empty_cache()

	# CPU mem
	a :						List[Any]      =							(psutil.Process().memory_info().rss - start_measures["cpu"]) / 2**20
	a :						Union[str, Any]      =							(cpu_peak_tracker.stop() - start_measures["cpu"]) / 2**20

	# GPU mem
	for i in range(torch.cuda.device_count()   ):
		a :						Optional[Any]      =							(torch.cuda.memory_allocated(_lowercase   ) - start_measures[str(_lowercase   )]) / 2**20
		a :						Optional[Any]      =							(torch.cuda.max_memory_allocated(_lowercase   ) - start_measures[str(_lowercase   )]) / 2**20

	return measures







def  _SCREAMING_SNAKE_CASE						(      _lowercase		:       str       ,			_lowercase		:       Union[str, Any]   )					->Optional[int]:






	'''simple docstring'''






	print(F"""{description}:"""   )
	print(F"""- Time: {measures['time']:.2f}s"""   )
	for i in range(torch.cuda.device_count()   ):
		print(F"""- GPU {i} allocated: {measures[str(_lowercase   )]:.2f}MiB"""   )
		a :						str      =							measures[F"""{i}-peak"""]
		print(F"""- GPU {i} peak: {peak:.2f}MiB"""   )
	print(F"""- CPU RAM allocated: {measures['cpu']:.2f}MiB"""   )
	print(F"""- CPU RAM peak: {measures['cpu-peak']:.2f}MiB"""   )


