





import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


UpperCamelCase_												=	logging.get_logger(__name__)


UpperCamelCase_												=	{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all LED models at https://huggingface.co/models?filter=LED
UpperCamelCase_												=	{
    'vocab_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json',
    },
    'merges_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json',
    },
}

UpperCamelCase_												=	{
    'allenai/led-base-16384': 1_63_84,
}






@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def 				_UpperCAmelCase			(    ):




   '''simple docstring'''

   UpperCAmelCase__						=(
       list(range(ord("!"     )				,							ord("~"     ) + 1     )     ) + list(range(ord("¡"     )				,							ord("¬"     ) + 1     )     ) + list(range(ord("®"     )				,							ord("ÿ"     ) + 1     )     )
   )
   UpperCAmelCase__						=bs[:]
   UpperCAmelCase__						=0
   for b in range(2**8     ):
      if b not in bs:
         bs.append(A     )
         cs.append(2**8 + n     )
         n += 1
   UpperCAmelCase__						=[chr(A     ) for n in cs]
   return dict(zip(A				,							A     )     )






def 				_UpperCAmelCase			(    A     ):




   '''simple docstring'''

   UpperCAmelCase__						=set()
   UpperCAmelCase__						=word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)     )
      UpperCAmelCase__						=char
   return pairs




class     snake_case_		(   a							):






 '''simple docstring'''





 __UpperCamelCase       =      VOCAB_FILES_NAMES
 __UpperCamelCase       =      PRETRAINED_VOCAB_FILES_MAP
 __UpperCamelCase       =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __UpperCamelCase       =      ['input_ids', 'attention_mask']

 def __init__(  self,		A_,		A_,		A_="replace",		A_="<s>",		A_="</s>",		A_="</s>",		A_="<s>",		A_="<unk>",		A_="<pad>",		A_="<mask>",		A_=False,		**A_,		)	->       List[str]:
    UpperCAmelCase__						=AddedToken(A_,		lstrip=A_,		rstrip=A_		) if isinstance(A_,		A_		) else bos_token
    UpperCAmelCase__						=AddedToken(A_,		lstrip=A_,		rstrip=A_		) if isinstance(A_,		A_		) else eos_token
    UpperCAmelCase__						=AddedToken(A_,		lstrip=A_,		rstrip=A_		) if isinstance(A_,		A_		) else sep_token
    UpperCAmelCase__						=AddedToken(A_,		lstrip=A_,		rstrip=A_		) if isinstance(A_,		A_		) else cls_token
    UpperCAmelCase__						=AddedToken(A_,		lstrip=A_,		rstrip=A_		) if isinstance(A_,		A_		) else unk_token
    UpperCAmelCase__						=AddedToken(A_,		lstrip=A_,		rstrip=A_		) if isinstance(A_,		A_		) else pad_token

    # Mask token behave like a normal word, i.e. include the space before it
    UpperCAmelCase__						=AddedToken(A_,		lstrip=A_,		rstrip=A_		) if isinstance(A_,		A_		) else mask_token

    super().__init__(
        errors=A_,		bos_token=A_,		eos_token=A_,		unk_token=A_,		sep_token=A_,		cls_token=A_,		pad_token=A_,		mask_token=A_,		add_prefix_space=A_,		**A_,		)

    with open(A_,		encoding="utf-8"		) as vocab_handle:
       UpperCAmelCase__						=json.load(A_		)
    UpperCAmelCase__						={v: k for k, v in self.encoder.items()}
    UpperCAmelCase__						=errors  # how to handle errors in decoding
    UpperCAmelCase__						=bytes_to_unicode()
    UpperCAmelCase__						={v: k for k, v in self.byte_encoder.items()}
    with open(A_,		encoding="utf-8"		) as merges_handle:
       UpperCAmelCase__						=merges_handle.read().split("\n"		)[1:-1]
    UpperCAmelCase__						=[tuple(merge.split()		) for merge in bpe_merges]
    UpperCAmelCase__						=dict(zip(A_,		range(len(A_		)		)		)		)
    UpperCAmelCase__						={}
    UpperCAmelCase__						=add_prefix_space

    # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
    UpperCAmelCase__						=re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"		)

 @property
 # Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
 def  __UpperCAmelCase						(  self		)	->       Optional[int]:
    return len(self.encoder		)

 def  __UpperCAmelCase						(  self		)	->       Optional[Any]:
    return dict(self.encoder,		**self.added_tokens_encoder		)

 def  __UpperCAmelCase						(  self,		A_		)	->       Any:
    if token in self.cache:
       return self.cache[token]
    UpperCAmelCase__						=tuple(A_		)
    UpperCAmelCase__						=get_pairs(A_		)

    if not pairs:
       return token

    while True:
       UpperCAmelCase__						=min(A_,		key=lambda A_		: self.bpe_ranks.get(A_,		float("inf"		)		)		)
       if bigram not in self.bpe_ranks:
          break
       UpperCAmelCase__			,							UpperCAmelCase__						=bigram
       UpperCAmelCase__						=[]
       UpperCAmelCase__						=0
       while i < len(A_		):
          try:
             UpperCAmelCase__						=word.index(A_,		A_		)
          except ValueError:
             new_word.extend(word[i:]		)
             break
          else:
             new_word.extend(word[i:j]		)
             UpperCAmelCase__						=j

          if word[i] == first and i < len(A_		) - 1 and word[i + 1] == second:
             new_word.append(first + second		)
             i += 2
          else:
             new_word.append(word[i]		)
             i += 1
       UpperCAmelCase__						=tuple(A_		)
       UpperCAmelCase__						=new_word
       if len(A_		) == 1:
          break
       else:
          UpperCAmelCase__						=get_pairs(A_		)
    UpperCAmelCase__						=" ".join(A_		)
    UpperCAmelCase__						=word
    return word

 def  __UpperCAmelCase						(  self,		A_		)	->       str:
    UpperCAmelCase__						=[]
    for token in re.findall(self.pat,		A_		):
       UpperCAmelCase__						="".join(
           self.byte_encoder[b] for b in token.encode("utf-8"		)		)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
       bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A_		).split(" "		)		)
    return bpe_tokens

 def  __UpperCAmelCase						(  self,		A_		)	->       str:
    return self.encoder.get(A_,		self.encoder.get(self.unk_token		)		)

 def  __UpperCAmelCase						(  self,		A_		)	->       Dict:
    return self.decoder.get(A_		)

 def  __UpperCAmelCase						(  self,		A_		)	->       Dict:
    UpperCAmelCase__						="".join(A_		)
    UpperCAmelCase__						=bytearray([self.byte_decoder[c] for c in text]		).decode("utf-8",		errors=self.errors		)
    return text

 def  __UpperCAmelCase						(  self,		A_,		A_ = None		)	->       Tuple[str]:
    if not os.path.isdir(A_		):
       logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""		)
       return
    UpperCAmelCase__						=os.path.join(
        A_,		(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
    UpperCAmelCase__						=os.path.join(
        A_,		(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]		)

    with open(A_,		"w",		encoding="utf-8"		) as f:
       f.write(json.dumps(self.encoder,		indent=2,		sort_keys=A_,		ensure_ascii=A_		) + "\n"		)

    UpperCAmelCase__						=0
    with open(A_,		"w",		encoding="utf-8"		) as writer:
       writer.write("#version: 0.2\n"		)
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),		key=lambda A_		: kv[1]		):
          if index != token_index:
             logger.warning(
                 f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                 " Please check that the tokenizer is not corrupted!"		)
             UpperCAmelCase__						=token_index
          writer.write(" ".join(A_		) + "\n"		)
          index += 1

    return vocab_file, merge_file

 def  __UpperCAmelCase						(  self,		A_,		A_ = None		)	->       List[int]:
    if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
    UpperCAmelCase__						=[self.cls_token_id]
    UpperCAmelCase__						=[self.sep_token_id]
    return cls + token_ids_a + sep + sep + token_ids_a + sep

 def  __UpperCAmelCase						(  self,		A_,		A_ = None,		A_ = False		)	->       List[int]:
    if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=A_,		token_ids_a=A_,		already_has_special_tokens=A_		)

    if token_ids_a is None:
       return [1] + ([0] * len(A_		)) + [1]
    return [1] + ([0] * len(A_		)) + [1, 1] + ([0] * len(A_		)) + [1]

 def  __UpperCAmelCase						(  self,		A_,		A_ = None		)	->       List[int]:
    UpperCAmelCase__						=[self.sep_token_id]
    UpperCAmelCase__						=[self.cls_token_id]

    if token_ids_a is None:
       return len(cls + token_ids_a + sep		) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]

 def  __UpperCAmelCase						(  self,		A_,		A_=False,		**A_		)	->       Any:
    UpperCAmelCase__						=kwargs.pop("add_prefix_space",		self.add_prefix_space		)
    if (is_split_into_words or add_prefix_space) and (len(A_		) > 0 and not text[0].isspace()):
       UpperCAmelCase__						=" " + text
    return (text, kwargs)






 def  __UpperCAmelCase						(  self,		A_,		A_ = None,		A_ = PaddingStrategy.DO_NOT_PAD,		A_ = None,		A_ = None,		)	->       dict:
    UpperCAmelCase__						=super()._pad(
        encoded_inputs=A_,		max_length=A_,		padding_strategy=A_,		pad_to_multiple_of=A_,		return_attention_mask=A_,		)

    # Load from model defaults
    if return_attention_mask is None:
       UpperCAmelCase__						="attention_mask" in self.model_input_names

    if return_attention_mask and "global_attention_mask" in encoded_inputs:
       UpperCAmelCase__						=encoded_inputs[self.model_input_names[0]]
       # `global_attention_mask` need to have the same length as other (sequential) inputs.
       UpperCAmelCase__						=len(encoded_inputs["global_attention_mask"]		) != len(A_		)

       if needs_to_be_padded:
          UpperCAmelCase__						=len(A_		) - len(encoded_inputs["global_attention_mask"]		)

          if self.padding_side == "right":
             # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
             UpperCAmelCase__						=(
                 encoded_inputs["global_attention_mask"] + [-1] * difference
             )
          elif self.padding_side == "left":
             UpperCAmelCase__						=[-1] * difference + encoded_inputs[
                 "global_attention_mask"
             ]
          else:
             raise ValueError("Invalid padding strategy:" + str(self.padding_side		)		)

    return encoded_inputs










import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase_												=	logging.get_logger(__name__)

UpperCamelCase_												=	{
    'microsoft/wavlm-base': 'https://huggingface.co/microsoft/wavlm-base/resolve/main/config.json',
    # See all WavLM models at https://huggingface.co/models?filter=wavlm
}

class     snake_case_		(   a							):






 '''simple docstring'''





 __UpperCamelCase       =      'wavlm'

 def __init__(  self,		A_=32,		A_=768,		A_=12,		A_=12,		A_=3072,		A_="gelu",		A_=0.1,		A_=0.1,		A_=0.1,		A_=0.0,		A_=0.1,		A_=0.1,		A_=0.02,		A_=1E-5,		A_="group",		A_="gelu",		A_=(512, 512, 512, 512, 512, 512, 512),		A_=(5, 2, 2, 2, 2, 2, 2),		A_=(10, 3, 3, 3, 3, 2, 2),		A_=False,		A_=128,		A_=16,		A_=320,		A_=800,		A_=False,		A_=True,		A_=0.05,		A_=10,		A_=2,		A_=0.0,		A_=10,		A_=320,		A_=2,		A_=0.1,		A_=100,		A_=256,		A_=256,		A_=0.1,		A_="mean",		A_=False,		A_=False,		A_=256,		A_=(512, 512, 512, 512, 1500),		A_=(5, 3, 3, 1, 1),		A_=(1, 2, 3, 1, 1),		A_=512,		A_=80,		A_=0,		A_=1,		A_=2,		A_=False,		A_=3,		A_=2,		A_=3,		A_=None,		**A_,		)	->       Union[str, Any]:
    super().__init__(**A_,		pad_token_id=A_,		bos_token_id=A_,		eos_token_id=A_		)
    UpperCAmelCase__						=hidden_size
    UpperCAmelCase__						=feat_extract_norm
    UpperCAmelCase__						=feat_extract_activation
    UpperCAmelCase__						=list(A_		)
    UpperCAmelCase__						=list(A_		)
    UpperCAmelCase__						=list(A_		)
    UpperCAmelCase__						=conv_bias
    UpperCAmelCase__						=num_buckets
    UpperCAmelCase__						=max_bucket_distance
    UpperCAmelCase__						=num_conv_pos_embeddings
    UpperCAmelCase__						=num_conv_pos_embedding_groups
    UpperCAmelCase__						=len(self.conv_dim		)
    UpperCAmelCase__						=num_hidden_layers
    UpperCAmelCase__						=intermediate_size
    UpperCAmelCase__						=hidden_act
    UpperCAmelCase__						=num_attention_heads
    UpperCAmelCase__						=hidden_dropout
    UpperCAmelCase__						=attention_dropout
    UpperCAmelCase__						=activation_dropout
    UpperCAmelCase__						=feat_proj_dropout
    UpperCAmelCase__						=final_dropout
    UpperCAmelCase__						=layerdrop
    UpperCAmelCase__						=layer_norm_eps
    UpperCAmelCase__						=initializer_range
    UpperCAmelCase__						=num_ctc_classes
    UpperCAmelCase__						=vocab_size
    UpperCAmelCase__						=do_stable_layer_norm
    UpperCAmelCase__						=use_weighted_layer_sum
    UpperCAmelCase__						=classifier_proj_size

    if (
        (len(self.conv_stride		) != self.num_feat_extract_layers)
        or (len(self.conv_kernel		) != self.num_feat_extract_layers)
        or (len(self.conv_dim		) != self.num_feat_extract_layers)
    ):
       raise ValueError(
           "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
           " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
           f""" {len(self.conv_dim		)}`, `len(config.conv_stride) = {len(self.conv_stride		)}`,"""
           f""" `len(config.conv_kernel) = {len(self.conv_kernel		)}`."""		)

    # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
    UpperCAmelCase__						=apply_spec_augment
    UpperCAmelCase__						=mask_time_prob
    UpperCAmelCase__						=mask_time_length
    UpperCAmelCase__						=mask_time_min_masks
    UpperCAmelCase__						=mask_feature_prob
    UpperCAmelCase__						=mask_feature_length

    # parameters for pretraining with codevector quantized representations
    UpperCAmelCase__						=num_codevectors_per_group
    UpperCAmelCase__						=num_codevector_groups
    UpperCAmelCase__						=contrastive_logits_temperature
    UpperCAmelCase__						=num_negatives
    UpperCAmelCase__						=codevector_dim
    UpperCAmelCase__						=proj_codevector_dim
    UpperCAmelCase__						=diversity_loss_weight

    # ctc loss
    UpperCAmelCase__						=ctc_loss_reduction
    UpperCAmelCase__						=ctc_zero_infinity

    # adapter
    UpperCAmelCase__						=add_adapter
    UpperCAmelCase__						=adapter_kernel_size
    UpperCAmelCase__						=adapter_stride
    UpperCAmelCase__						=num_adapter_layers
    UpperCAmelCase__						=output_hidden_size or hidden_size

    # SequenceClassification-specific parameter. Feel free to ignore for other classes.
    UpperCAmelCase__						=classifier_proj_size

    # XVector-specific parameters. Feel free to ignore for other classes.
    UpperCAmelCase__						=list(A_		)
    UpperCAmelCase__						=list(A_		)
    UpperCAmelCase__						=list(A_		)
    UpperCAmelCase__						=xvector_output_dim






 @property
 def  __UpperCAmelCase						(  self		)	->       List[Any]:
    return functools.reduce(operator.mul,		self.conv_stride,		1		)



