






'''simple docstring'''
import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCamelCase__			=      logging.get_logger(__name__)

UpperCamelCase__			=      {'vocab_file': 'spiece.model'}

UpperCamelCase__			=      {
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

UpperCamelCase__			=      {
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
UpperCamelCase__			=      0
UpperCamelCase__			=      1
UpperCamelCase__			=      2
UpperCamelCase__			=      3
UpperCamelCase__			=      4


class 							_UpperCAmelCase    (		snake_case      ):
    __lowerCamelCase: Optional[Any]									=     VOCAB_FILES_NAMES
    __lowerCamelCase: List[str]									=     PRETRAINED_VOCAB_FILES_MAP
    __lowerCamelCase: Optional[int]									=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __lowerCamelCase: int									=     'left'




    def __init__(      self	:							Optional[Any]	,  a	:							int	,  a	:							Tuple=False	,  a	:							Dict=True	,  a	:							Tuple=False	,  a	:							List[Any]="<s>"	,  a	:							Tuple="</s>"	,  a	:							Any="<unk>"	,  a	:							Dict="<sep>"	,  a	:							Optional[int]="<pad>"	,  a	:							Dict="<cls>"	,  a	:							Union[str, Any]="<mask>"	,  a	:							Dict=["<eop>", "<eod>"]	,  a	:							Optional[Dict[str, Any]] = None	,  **a	:							Tuple	,  ):






       '''simple docstring'''






       lowercase_						:     Optional[Any]							     =  AddedToken(a	,  lstrip=a	,  rstrip=a      ) if isinstance(a	,  a      ) else mask_token

       lowercase_						:     int							     =  {} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           do_lower_case=a	,  remove_space=a	,  keep_accents=a	,  bos_token=a	,  eos_token=a	,  unk_token=a	,  sep_token=a	,  pad_token=a	,  cls_token=a	,  mask_token=a	,  additional_special_tokens=a	,  sp_model_kwargs=self.sp_model_kwargs	,  **a	,  )

       lowercase_						:     Tuple							     =  3

       lowercase_						:     Tuple							     =  do_lower_case
       lowercase_						:     Optional[Any]							     =  remove_space
       lowercase_						:     Any							     =  keep_accents
       lowercase_						:     Tuple							     =  vocab_file

       lowercase_						:     int							     =  spm.SentencePieceProcessor(**self.sp_model_kwargs      )
       self.sp_model.Load(a      )




    @property
    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       return len(self.sp_model      )




    def 					lowerCAmelCase__					(      self	:							Any      ):






       '''simple docstring'''






       lowercase_						:     str							     =  {self.convert_ids_to_tokens(a      ): i for i in range(self.vocab_size      )}
       vocab.update(self.added_tokens_encoder      )
       return vocab




    def __getstate__(      self	:							Union[str, Any]      ):






       '''simple docstring'''






       lowercase_						:     List[str]							     =  self.__dict__.copy()
       lowercase_						:     List[str]							     =  None
       return state




    def __setstate__(      self	:							Optional[int]	,  a	:							List[Any]      ):






       '''simple docstring'''






       lowercase_						:     str							     =  d

       # for backward compatibility
       if not hasattr(self	,  "sp_model_kwargs"      ):
          lowercase_						:     List[str]							     =  {}

       lowercase_						:     List[Any]							     =  spm.SentencePieceProcessor(**self.sp_model_kwargs      )
       self.sp_model.Load(self.vocab_file      )




    def 					lowerCAmelCase__					(      self	:							List[str]	,  a	:							List[Any]      ):






       '''simple docstring'''






       if self.remove_space:
          lowercase_						:     str							     =  " ".join(inputs.strip().split()      )
       else:
          lowercase_						:     Any							     =  inputs
       lowercase_						:     List[Any]							     =  outputs.replace("``"	,  "\""      ).replace("''"	,  "\""      )

       if not self.keep_accents:
          lowercase_						:     Optional[int]							     =  unicodedata.normalize("NFKD"	,  a      )
          lowercase_						:     str							     =  "".join([c for c in outputs if not unicodedata.combining(a      )]      )
       if self.do_lower_case:
          lowercase_						:     List[Any]							     =  outputs.lower()

       return outputs




    def 					lowerCAmelCase__					(      self	:							Dict	,  a	:							str      ):






       '''simple docstring'''






       lowercase_						:     Union[str, Any]							     =  self.preprocess_text(a      )
       lowercase_						:     Union[str, Any]							     =  self.sp_model.encode(a	,  out_type=a      )
       lowercase_						:     Optional[int]							     =  []
       for piece in pieces:
          if len(a      ) > 1 and piece[-1] == str(","      ) and piece[-2].isdigit():
             lowercase_						:     Optional[int]							     =  self.sp_model.EncodeAsPieces(piece[:-1].replace(a	,  ""      )      )
             if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                if len(cur_pieces[0]      ) == 1:
                   lowercase_						:     Union[str, Any]							     =  cur_pieces[1:]
                else:
                   lowercase_						:     Union[str, Any]							     =  cur_pieces[0][1:]
             cur_pieces.append(piece[-1]      )
             new_pieces.extend(a      )
          else:
             new_pieces.append(a      )

       return new_pieces




    def 					lowerCAmelCase__					(      self	:							Dict	,  a	:							List[str]      ):






       '''simple docstring'''






       return self.sp_model.PieceToId(a      )




    def 					lowerCAmelCase__					(      self	:							Optional[Any]	,  a	:							Union[str, Any]      ):






       '''simple docstring'''






       return self.sp_model.IdToPiece(a      )




    def 					lowerCAmelCase__					(      self	:							Optional[Any]	,  a	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     Tuple							     =  "".join(a      ).replace(a	,  " "      ).strip()
       return out_string




    def 					lowerCAmelCase__					(      self	:							Any	,  a	:							List[int]	,  a	:							bool = False	,  a	:							bool = None	,  a	:							bool = True	,  **a	:							Optional[Any]	,  ):






       '''simple docstring'''






       lowercase_						:     Optional[Any]							     =  kwargs.pop("use_source_tokenizer"	,  a      )

       lowercase_						:     Optional[int]							     =  self.convert_ids_to_tokens(a	,  skip_special_tokens=a      )

       # To avoid mixing byte-level and unicode for byte-level BPT
       # we need to build string separately for added tokens and byte-level tokens
       # cf. https://github.com/huggingface/transformers/issues/1133
       lowercase_						:     Optional[int]							     =  []
       lowercase_						:     Tuple							     =  []
       for token in filtered_tokens:
          if skip_special_tokens and token in self.all_special_ids:
             continue
          if token in self.added_tokens_encoder:
             if current_sub_text:
                sub_texts.append(self.convert_tokens_to_string(a      )      )
                lowercase_						:     Any							     =  []
             sub_texts.append(a      )
          else:
             current_sub_text.append(a      )
       if current_sub_text:
          sub_texts.append(self.convert_tokens_to_string(a      )      )

       # Mimic the behavior of the Rust tokenizer:
       # By default, there are no spaces between special tokens
       lowercase_						:     Optional[Any]							     =  "".join(a      )

       lowercase_						:     List[Any]							     =  (
           clean_up_tokenization_spaces
           if clean_up_tokenization_spaces is not None
           else self.clean_up_tokenization_spaces
       )
       if clean_up_tokenization_spaces:
          lowercase_						:     Any							     =  self.clean_up_tokenization(a      )
          return clean_text
       else:
          return text




    def 					lowerCAmelCase__					(      self	:							Any	,  a	:							List[int]	,  a	:							Optional[List[int]] = None      ):






       '''simple docstring'''






       lowercase_						:     str							     =  [self.sep_token_id]
       lowercase_						:     Any							     =  [self.cls_token_id]
       if token_ids_a is None:
          return token_ids_a + sep + cls
       return token_ids_a + sep + token_ids_a + sep + cls




    def 					lowerCAmelCase__					(      self	:							Union[str, Any]	,  a	:							List[int]	,  a	:							Optional[List[int]] = None	,  a	:							bool = False      ):






       '''simple docstring'''






       if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=a	,  token_ids_a=a	,  already_has_special_tokens=a      )

       if token_ids_a is not None:
          return ([0] * len(a      )) + [1] + ([0] * len(a      )) + [1, 1]
       return ([0] * len(a      )) + [1, 1]




    def 					lowerCAmelCase__					(      self	:							Dict	,  a	:							List[int]	,  a	:							Optional[List[int]] = None      ):






       '''simple docstring'''






       lowercase_						:     int							     =  [self.sep_token_id]
       lowercase_						:     str							     =  [2]

       if token_ids_a is None:
          return len(token_ids_a + sep      ) * [0] + cls_segment_id
       return len(token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1] + cls_segment_id





    def 					lowerCAmelCase__					(      self	:							Union[str, Any]	,  a	:							str	,  a	:							Optional[str] = None      ):






       '''simple docstring'''






       if not os.path.isdir(a      ):
          logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""      )
          return
       lowercase_						:     Union[str, Any]							     =  os.path.join(
           a	,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )

       if os.path.abspath(self.vocab_file      ) != os.path.abspath(a      ) and os.path.isfile(self.vocab_file      ):
          copyfile(self.vocab_file	,  a      )
       elif not os.path.isfile(self.vocab_file      ):
          with open(a	,  "wb"      ) as fi:
             lowercase_						:     List[Any]							     =  self.sp_model.serialized_model_proto()
             fi.write(a      )

       return (out_vocab_file,)












'''simple docstring'''
import flax.linen as nn
import jax
import jax.numpy as jnp


class 							_UpperCAmelCase    (		nn.Module      ):
    __lowerCamelCase: int
    __lowerCamelCase: jnp.dtype									=     jnp.floataa




    def 					lowerCAmelCase__					(      self	:							Optional[Any]      ):






       '''simple docstring'''






       lowercase_						:     List[Any]							     =  nn.Conv(
           self.out_channels	,  kernel_size=(3, 3)	,  strides=(1, 1)	,  padding=((1, 1), (1, 1))	,  dtype=self.dtype	,  )





    def __call__(      self	:							List[str]	,  a	:							Optional[int]      ):






       '''simple docstring'''






       lowercase_     ,		lowercase_     ,		lowercase_     ,		lowercase_						:     Optional[Any]							     =  hidden_states.shape
       lowercase_						:     Tuple							     =  jax.image.resize(
           a	,  shape=(batch, height * 2, width * 2, channels)	,  method="nearest"	,  )
       lowercase_						:     List[Any]							     =  self.conv(a      )
       return hidden_states


class 							_UpperCAmelCase    (		nn.Module      ):
    __lowerCamelCase: int
    __lowerCamelCase: jnp.dtype									=     jnp.floataa




    def 					lowerCAmelCase__					(      self	:							Optional[Any]      ):






       '''simple docstring'''






       lowercase_						:     Any							     =  nn.Conv(
           self.out_channels	,  kernel_size=(3, 3)	,  strides=(2, 2)	,  padding=((1, 1), (1, 1))	,  dtype=self.dtype	,  )





    def __call__(      self	:							Any	,  a	:							int      ):






       '''simple docstring'''






       lowercase_						:     Any							     =  self.conv(a      )
       return hidden_states


class 							_UpperCAmelCase    (		nn.Module      ):
    __lowerCamelCase: int
    __lowerCamelCase: int									=     None
    __lowerCamelCase: float									=     0.0
    __lowerCamelCase: bool									=     None
    __lowerCamelCase: jnp.dtype									=     jnp.floataa




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       lowercase_						:     Union[str, Any]							     =  self.in_channels if self.out_channels is None else self.out_channels

       lowercase_						:     Optional[int]							     =  nn.GroupNorm(num_groups=3_2	,  epsilon=1e-5      )
       lowercase_						:     Tuple							     =  nn.Conv(
           a	,  kernel_size=(3, 3)	,  strides=(1, 1)	,  padding=((1, 1), (1, 1))	,  dtype=self.dtype	,  )

       lowercase_						:     List[str]							     =  nn.Dense(a	,  dtype=self.dtype      )

       lowercase_						:     Optional[int]							     =  nn.GroupNorm(num_groups=3_2	,  epsilon=1e-5      )
       lowercase_						:     Any							     =  nn.Dropout(self.dropout_prob      )
       lowercase_						:     Dict							     =  nn.Conv(
           a	,  kernel_size=(3, 3)	,  strides=(1, 1)	,  padding=((1, 1), (1, 1))	,  dtype=self.dtype	,  )

       lowercase_						:     Tuple							     =  self.in_channels != out_channels if self.use_nin_shortcut is None else self.use_nin_shortcut

       lowercase_						:     Optional[Any]							     =  None
       if use_nin_shortcut:
          lowercase_						:     Union[str, Any]							     =  nn.Conv(
              a	,  kernel_size=(1, 1)	,  strides=(1, 1)	,  padding="VALID"	,  dtype=self.dtype	,  )





    def __call__(      self	:							List[str]	,  a	:							str	,  a	:							Dict	,  a	:							List[str]=True      ):






       '''simple docstring'''






       lowercase_						:     Dict							     =  hidden_states
       lowercase_						:     int							     =  self.norma(a      )
       lowercase_						:     List[Any]							     =  nn.swish(a      )
       lowercase_						:     Dict							     =  self.conva(a      )

       lowercase_						:     Optional[int]							     =  self.time_emb_proj(nn.swish(a      )      )
       lowercase_						:     Tuple							     =  jnp.expand_dims(jnp.expand_dims(a	,  1      )	,  1      )
       lowercase_						:     List[str]							     =  hidden_states + temb

       lowercase_						:     Optional[Any]							     =  self.norma(a      )
       lowercase_						:     Any							     =  nn.swish(a      )
       lowercase_						:     List[str]							     =  self.dropout(a	,  a      )
       lowercase_						:     int							     =  self.conva(a      )

       if self.conv_shortcut is not None:
          lowercase_						:     List[str]							     =  self.conv_shortcut(a      )

       return hidden_states + residual




