


import logging
import os
import threading
import time


try:
  import warnings
except ImportError:
  UpperCAmelCase__   	=							None

try:
  import msvcrt
except ImportError:
  UpperCAmelCase__   	=							None

try:
  import fcntl
except ImportError:
  UpperCAmelCase__   	=							None


# Backward compatibility
# ------------------------------------------------
try:
  TimeoutError
except NameError:
  UpperCAmelCase__   	=							OSError


# Data
# ------------------------------------------------
UpperCAmelCase__   	=							[
    '''Timeout''',
    '''BaseFileLock''',
    '''WindowsFileLock''',
    '''UnixFileLock''',
    '''SoftFileLock''',
    '''FileLock''',
]

UpperCAmelCase__   	=							'''3.0.12'''


UpperCAmelCase__   	=							None
def 				UpperCAmelCase_					(							)     ->			Optional[int]:







 """simple docstring"""


 global _logger
 _lowercase							      =_logger or logging.getLogger(__name__						)
 return _logger







class       lowerCamelCase__   (       lowerCAmelCase):





     def __init__(self		,     UpperCAmelCase       )     ->   Optional[Any]:
      _lowercase							      =lock_file
      return None





     def __str__(self       )     ->   str:
      _lowercase							      =f"The file lock '{self.lock_file}' could not be acquired."
      return temp







class       lowerCamelCase__   :





     def __init__(self		,     UpperCAmelCase       )     ->   Dict:
      _lowercase							      =lock
      return None





     def __enter__(self       )     ->   Optional[Any]:
      return self.lock





     def __exit__(self		,     UpperCAmelCase		,     UpperCAmelCase		,     UpperCAmelCase       )     ->   List[Any]:
      self.lock.release()
      return None







class       lowerCamelCase__   :





     def __init__(self		,     UpperCAmelCase		,     UpperCAmelCase=-1		,     UpperCAmelCase=None       )     ->   Dict:
      _lowercase							      =max_filename_length if max_filename_length is not None else 2_5_5
      # Hash the filename if it's too long
      _lowercase							      =self.hash_filename_if_too_long(UpperCAmelCase		,     UpperCAmelCase       )
      # The path to the lock file.
      _lowercase							      =lock_file

      # The file descriptor for the *_lock_file* as it is returned by the
      # os.open() function.
      # This file lock is only NOT None, if the object currently holds the
      # lock.
      _lowercase							      =None

      # The default timeout value.
      _lowercase							      =timeout

      # We use this lock primarily for the lock counter.
      _lowercase							      =threading.Lock()

      # The lock counter is used for implementing the nested locking
      # mechanism. Whenever the lock is acquired, the counter is increased and
      # the lock is only released, when this value is 0 again.
      _lowercase							      =0
      return None





     @property
     def      __A					(self       )     ->   int:
      return self._lock_file





     @property
     def      __A					(self       )     ->   Optional[Any]:
      return self._timeout





     @timeout.setter
     def      __A					(self		,     UpperCAmelCase       )     ->   Union[str, Any]:
      _lowercase							      =float(UpperCAmelCase       )
      return None





     def      __A					(self       )     ->   Any:
      raise NotImplementedError()





     def      __A					(self       )     ->   List[Any]:
      raise NotImplementedError()





     @property
     def      __A					(self       )     ->   Tuple:
      return self._lock_file_fd is not None





     def      __A					(self		,     UpperCAmelCase=None		,     UpperCAmelCase=0.05       )     ->   Union[str, Any]:
      # Use the default timeout, if no timeout is provided.
      if timeout is None:
       _lowercase							      =self.timeout

      # Increment the number right at the beginning.
      # We can still undo it, if something fails.
      with self._thread_lock:
       self._lock_counter += 1

      _lowercase							      =id(self       )
      _lowercase							      =self._lock_file
      _lowercase							      =time.time()
      try:
       while True:
        with self._thread_lock:
         if not self.is_locked:
          logger().debug(f"Attempting to acquire lock {lock_id} on {lock_filename}"       )
          self._acquire()

        if self.is_locked:
         logger().debug(f"Lock {lock_id} acquired on {lock_filename}"       )
         break
        elif timeout >= 0 and time.time() - start_time > timeout:
         logger().debug(f"Timeout on acquiring lock {lock_id} on {lock_filename}"       )
         raise Timeout(self._lock_file       )
        else:
         logger().debug(
             f"Lock {lock_id} not acquired on {lock_filename}, waiting {poll_intervall} seconds ..."       )
         time.sleep(UpperCAmelCase       )
      except:  # noqa
       # Something did go wrong, so decrement the counter.
       with self._thread_lock:
        _lowercase							      =max(0		,     self._lock_counter - 1       )

       raise
      return _Acquire_ReturnProxy(lock=self       )





     def      __A					(self		,     UpperCAmelCase=False       )     ->   Union[str, Any]:
      with self._thread_lock:
       if self.is_locked:
        self._lock_counter -= 1

        if self._lock_counter == 0 or force:
         _lowercase							      =id(self       )
         _lowercase							      =self._lock_file

         logger().debug(f"Attempting to release lock {lock_id} on {lock_filename}"       )
         self._release()
         _lowercase							      =0
         logger().debug(f"Lock {lock_id} released on {lock_filename}"       )

      return None





     def __enter__(self       )     ->   List[Any]:
      self.acquire()
      return self





     def __exit__(self		,     UpperCAmelCase		,     UpperCAmelCase		,     UpperCAmelCase       )     ->   Dict:
      self.release()
      return None





     def __del__(self       )     ->   Optional[Any]:
      self.release(force=UpperCAmelCase       )
      return None





     def      __A					(self		,     UpperCAmelCase		,     UpperCAmelCase       )     ->   str:
      _lowercase							      =os.path.basename(UpperCAmelCase       )
      if len(UpperCAmelCase       ) > max_length and max_length > 0:
       _lowercase							      =os.path.dirname(UpperCAmelCase       )
       _lowercase							      =str(hash(UpperCAmelCase       )       )
       _lowercase							      =filename[: max_length - len(UpperCAmelCase       ) - 8] + '''...''' + hashed_filename + '''.lock'''
       return os.path.join(UpperCAmelCase		,     UpperCAmelCase       )
      else:
       return path







class       lowerCamelCase__   (       lowerCAmelCase):





     def __init__(self		,     UpperCAmelCase		,     UpperCAmelCase=-1		,     UpperCAmelCase=None       )     ->   Tuple:
      from .file_utils import relative_to_absolute_path

      super().__init__(UpperCAmelCase		,     timeout=UpperCAmelCase		,     max_filename_length=UpperCAmelCase       )
      _lowercase							      ='''\\\\?\\''' + relative_to_absolute_path(self.lock_file       )





     def      __A					(self       )     ->   Any:
      _lowercase							      =os.O_RDWR | os.O_CREAT | os.O_TRUNC

      try:
       _lowercase							      =os.open(self._lock_file		,     UpperCAmelCase       )
      except OSError:
       pass
      else:
       try:
        msvcrt.locking(UpperCAmelCase		,     msvcrt.LK_NBLCK		,     1       )
       except OSError:
        os.close(UpperCAmelCase       )
       else:
        _lowercase							      =fd
      return None





     def      __A					(self       )     ->   Optional[int]:
      _lowercase							      =self._lock_file_fd
      _lowercase							      =None
      msvcrt.locking(UpperCAmelCase		,     msvcrt.LK_UNLCK		,     1       )
      os.close(UpperCAmelCase       )

      try:
       os.remove(self._lock_file       )
      # Probably another instance of the application
      # that acquired the file lock.
      except OSError:
       pass
      return None







class       lowerCamelCase__   (       lowerCAmelCase):





     def __init__(self		,     UpperCAmelCase		,     UpperCAmelCase=-1		,     UpperCAmelCase=None       )     ->   Tuple:
      _lowercase							      =os.statvfs(os.path.dirname(UpperCAmelCase       )       ).f_namemax
      super().__init__(UpperCAmelCase		,     timeout=UpperCAmelCase		,     max_filename_length=UpperCAmelCase       )





     def      __A					(self       )     ->   int:
      _lowercase							      =os.O_RDWR | os.O_CREAT | os.O_TRUNC
      _lowercase							      =os.open(self._lock_file		,     UpperCAmelCase       )

      try:
       fcntl.flock(UpperCAmelCase		,     fcntl.LOCK_EX | fcntl.LOCK_NB       )
      except OSError:
       os.close(UpperCAmelCase       )
      else:
       _lowercase							      =fd
      return None





     def      __A					(self       )     ->   Any:
      # Do not remove the lockfile:
      #
      #   https://github.com/benediktschmitt/py-filelock/issues/31
      #   https://stackoverflow.com/questions/17708885/flock-removing-locked-file-without-race-condition
      _lowercase							      =self._lock_file_fd
      _lowercase							      =None
      fcntl.flock(UpperCAmelCase		,     fcntl.LOCK_UN       )
      os.close(UpperCAmelCase       )
      return None







class       lowerCamelCase__   (       lowerCAmelCase):





     def      __A					(self       )     ->   Dict:
      _lowercase							      =os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
      try:
       _lowercase							      =os.open(self._lock_file		,     UpperCAmelCase       )
      except OSError:
       pass
      else:
       _lowercase							      =fd
      return None





     def      __A					(self       )     ->   Any:
      os.close(self._lock_file_fd       )
      _lowercase							      =None

      try:
       os.remove(self._lock_file       )
      # The file is already deleted and that's what we want.
      except OSError:
       pass
      return None



UpperCAmelCase__   	=							None

if msvcrt:
  UpperCAmelCase__   	=							WindowsFileLock
elif fcntl:
  UpperCAmelCase__   	=							UnixFileLock
else:
  UpperCAmelCase__   	=							SoftFileLock

  if warnings is not None:
    warnings.warn('''only soft file lock is available''')





"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__UpperCamelCase						=      logging.get_logger(__name__)

__UpperCamelCase						=      {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__UpperCamelCase						=      [
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

__UpperCamelCase						=      {
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
__UpperCamelCase						=      {f'''funnel-transformer/{name}''': 512 for name in _model_names}
__UpperCamelCase						=      {f'''funnel-transformer/{name}''': {'''do_lower_case''': True} for name in _model_names}
class 					lowerCAmelCase	(					lowerCamelCase_    ):




		'''simple docstring'''







		SCREAMING_SNAKE_CASE_						:     int								=     VOCAB_FILES_NAMES
		SCREAMING_SNAKE_CASE_						:     List[str]								=     PRETRAINED_VOCAB_FILES_MAP
		SCREAMING_SNAKE_CASE_						:     List[Any]								=     PRETRAINED_INIT_CONFIGURATION
		SCREAMING_SNAKE_CASE_						:     str								=     FunnelTokenizer
		SCREAMING_SNAKE_CASE_						:     Dict								=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		SCREAMING_SNAKE_CASE_						:     int								=     2







		def __init__(	self						,       lowerCAmelCase__=None						,       lowerCAmelCase__=None						,       lowerCAmelCase__=True						,       lowerCAmelCase__="<unk>"						,       lowerCAmelCase__="<sep>"						,       lowerCAmelCase__="<pad>"						,       lowerCAmelCase__="<cls>"						,       lowerCAmelCase__="<mask>"						,       lowerCAmelCase__="<s>"						,       lowerCAmelCase__="</s>"						,       lowerCAmelCase__=True						,       lowerCAmelCase__=True						,       lowerCAmelCase__=None						,       lowerCAmelCase__="##"						,       **lowerCAmelCase__						,       )     ->							Tuple:
								super().__init__(
								    lowerCAmelCase__						,       tokenizer_file=lowerCAmelCase__						,       do_lower_case=lowerCAmelCase__						,       unk_token=lowerCAmelCase__						,       sep_token=lowerCAmelCase__						,       pad_token=lowerCAmelCase__						,       cls_token=lowerCAmelCase__						,       mask_token=lowerCAmelCase__						,       bos_token=lowerCAmelCase__						,       eos_token=lowerCAmelCase__						,       clean_text=lowerCAmelCase__						,       tokenize_chinese_chars=lowerCAmelCase__						,       strip_accents=lowerCAmelCase__						,       wordpieces_prefix=lowerCAmelCase__						,       **lowerCAmelCase__						,       )

								SCREAMING_SNAKE_CASE													=					json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
								if (
								    normalizer_state.get('lowercase'						,       lowerCAmelCase__				) != do_lower_case
								    or normalizer_state.get('strip_accents'						,       lowerCAmelCase__				) != strip_accents
								    or normalizer_state.get('handle_chinese_chars'						,       lowerCAmelCase__				) != tokenize_chinese_chars
								):
														SCREAMING_SNAKE_CASE													=					getattr(lowerCAmelCase__						,       normalizer_state.pop('type'				)				)
														SCREAMING_SNAKE_CASE													=					do_lower_case
														SCREAMING_SNAKE_CASE													=					strip_accents
														SCREAMING_SNAKE_CASE													=					tokenize_chinese_chars
														SCREAMING_SNAKE_CASE													=					normalizer_class(**lowerCAmelCase__				)

								SCREAMING_SNAKE_CASE													=					do_lower_case







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__=None				)     ->							Optional[Any]:
								SCREAMING_SNAKE_CASE													=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
														output += token_ids_a + [self.sep_token_id]

								return output







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__ = None				)     ->							List[int]:
								SCREAMING_SNAKE_CASE													=					[self.sep_token_id]
								SCREAMING_SNAKE_CASE													=					[self.cls_token_id]
								if token_ids_a is None:
														return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0]
								return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__ = None				)     ->							Tuple[str]:
								SCREAMING_SNAKE_CASE													=					self._tokenizer.model.save(lowerCAmelCase__						,       name=lowerCAmelCase__				)
								return tuple(lowerCAmelCase__				)


