

import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
 import torch


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


_UpperCamelCase 			=		logging.get_logger(__name__)
_UpperCamelCase 			=		{"vocab_file": "spiece.model"}

_UpperCamelCase 			=		{
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

_UpperCamelCase 			=		{
    "AI-Sweden/gpt-sw3-126m": 2048,
    "AI-Sweden/gpt-sw3-350m": 2048,
    "AI-Sweden/gpt-sw3-1.6b": 2048,
    "AI-Sweden/gpt-sw3-6.7b": 2048,
    "AI-Sweden/gpt-sw3-20b": 2048,
}







class   __lowercase      (_UpperCAmelCase ):
 _UpperCamelCase			  =  VOCAB_FILES_NAMES
 _UpperCamelCase			  =  PRETRAINED_VOCAB_FILES_MAP
 _UpperCamelCase			  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _UpperCamelCase			  =  ["""input_ids""", """attention_mask"""]
 def __init__(							self      ,							A_      ,							A_=False      ,							A_=False      ,							A_=False      ,							A_=None      ,							A_=None      ,							A_=None      ,							A_=None      ,							A_ = None      ,							**A_      ,							)					->None:



       '''simple docstring'''





       __lowerCAmelCase					:						Optional[int]					=					{} if sp_model_kwargs is None else sp_model_kwargs

       __lowerCAmelCase					:						int					=					kwargs.get('''name_or_path'''					)
       if name_or_path is None:
             logger.warning(
                 '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
                 ''' you are testing the model, this can safely be ignored'''					)
             __lowerCAmelCase					:						Union[str, Any]					=					'''None'''

       # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
       __lowerCAmelCase					:						str					=					'''<|endoftext|>''' if eos_token is None else eos_token
       __lowerCAmelCase					:						Any					=					'''<unk>''' if unk_token is None else unk_token
       if "gpt-sw3-7b" in name_or_path:
             __lowerCAmelCase					:						Dict					=					unk_token if pad_token is None else pad_token
             __lowerCAmelCase					:						int					=					eos_token if bos_token is None else bos_token
       else:
             __lowerCAmelCase					:						Optional[int]					=					'''<pad>''' if pad_token is None else pad_token
             __lowerCAmelCase					:						List[str]					=					'''<s>''' if bos_token is None else bos_token

       super().__init__(
           do_lower_case=A_      ,							remove_space=A_      ,							keep_accents=A_      ,							bos_token=A_      ,							eos_token=A_      ,							unk_token=A_      ,							pad_token=A_      ,							sp_model_kwargs=self.sp_model_kwargs      ,							**A_      ,							)

       __lowerCAmelCase					:						Union[str, Any]					=					do_lower_case
       __lowerCAmelCase					:						Union[str, Any]					=					remove_space
       __lowerCAmelCase					:						int					=					keep_accents
       __lowerCAmelCase					:						Union[str, Any]					=					vocab_file

       __lowerCAmelCase					:						int					=					spm.SentencePieceProcessor(**self.sp_model_kwargs					)
       self.sp_model.Load(A_					)

       # Used for whitespace normalization in input texts
       # fmt : off
       __lowerCAmelCase					:						List[Any]					=					{''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
       # fmt : on

       # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
       __lowerCAmelCase					:						int					=					re.compile(
           f"""[{"".join(map(A_      ,							list(range(0      ,							9					)					) + list(range(11      ,							32					)					) + list(range(127      ,							160					)					) + [160, 173, 8203]					)					)}]"""					)
 def __getstate__(							self					)					->Dict:



       '''simple docstring'''





       __lowerCAmelCase					:						Union[str, Any]					=					self.__dict__.copy()
       __lowerCAmelCase					:						List[Any]					=					None
       return state
 def __setstate__(							self      ,							A_					)					->Tuple:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					d

       # for backward compatibility
       if not hasattr(self      ,							'''sp_model_kwargs'''					):
             __lowerCAmelCase					:						List[Any]					=					{}

       __lowerCAmelCase					:						List[str]					=					spm.SentencePieceProcessor(**self.sp_model_kwargs					)
       self.sp_model.Load(self.vocab_file					)
 @property
 # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
 def 	UpperCamelCase__  (							self					)					->int:



       '''simple docstring'''





       return len(self.sp_model					)
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					self.non_printing_characters_re.sub(''''''      ,							A_					)

       # Normalize whitespaces
       __lowerCAmelCase					:						List[str]					=					''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]					)

       # NFC Unicode normalization
       __lowerCAmelCase					:						Tuple					=					unicodedata.normalize('''NFC'''      ,							A_					)
       return text
 def 	UpperCamelCase__  (							self      ,							A_      ,							**A_					)					->List[str]:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					self.preprocess_text(A_					)
       return self.sp_model.encode(A_      ,							out_type=A_					)
 def 	UpperCamelCase__  (							self      ,							A_					)					->int:



       '''simple docstring'''





       return self.sp_model.PieceToId(A_					)
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       return self.sp_model.IdToPiece(A_					)
 @staticmethod
 def 	UpperCamelCase__  (							A_					)					->str:



       '''simple docstring'''





       return out_string
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       __lowerCAmelCase					:						str					=					[]
       __lowerCAmelCase					:						Tuple					=					''''''
       __lowerCAmelCase					:						int					=					False
       for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                   # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                   if not prev_is_special:
                         out_string += " "

                   out_string += self.sp_model.decode(A_					) + token
                   __lowerCAmelCase					:						Optional[Any]					=					True
                   __lowerCAmelCase					:						Optional[int]					=					[]
             else:
                   current_sub_tokens.append(A_					)
                   __lowerCAmelCase					:						str					=					False
       out_string += self.sp_model.decode(A_					)

       return out_string
 def 	UpperCamelCase__  (							self					)					->Dict[str, int]:



       '''simple docstring'''





       __lowerCAmelCase					:						str					=					{self.convert_ids_to_tokens(A_					): i for i in range(self.vocab_size					)}
       vocab.update(self.added_tokens_encoder					)
       return vocab
 def 	UpperCamelCase__  (							self      ,							A_      ,							A_ = None					)					->Tuple[str]:



       '''simple docstring'''





       if not os.path.isdir(A_					):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""					)
             return
       __lowerCAmelCase					:						Any					=					os.path.join(
           A_      ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

       if os.path.abspath(self.vocab_file					) != os.path.abspath(A_					) and os.path.isfile(self.vocab_file					):
             copyfile(self.vocab_file      ,							A_					)
       elif not os.path.isfile(self.vocab_file					):
             with open(A_      ,							'''wb'''					) as fi:
                   __lowerCAmelCase					:						Dict					=					self.sp_model.serialized_model_proto()
                   fi.write(A_					)

       return (out_vocab_file,)
 def 	UpperCamelCase__  (							self      ,							A_      ,							A_ = False					)					->Union[List[int], List[List[int]], "torch.Tensor"]:



       '''simple docstring'''





       if isinstance(A_      ,							A_					):
             __lowerCAmelCase					:						Optional[Any]					=					self.preprocess_text(A_					)
             __lowerCAmelCase					:						Dict					=					self.sp_model.encode(A_					)
       else:
             __lowerCAmelCase					:						Dict					=					[self.preprocess_text(A_					) for t in text]
             __lowerCAmelCase					:						Optional[int]					=					self.sp_model.encode(A_					)

       if return_tensors is True or return_tensors == "pt":
             __lowerCAmelCase					:						Tuple					=					torch.tensor(A_					)

       return token_ids
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       return self.sp_model.decode(A_					)





 def 	UpperCamelCase__  (							self      ,							A_					)					->List[int]:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					[f"""User: {text}""" if is_user else f"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
       __lowerCAmelCase					:						Any					=					(
           f"""{self.eos_token}{self.bos_token}""" + f"""{self.bos_token}""".join(A_					) + f"""{self.bos_token}Bot:"""
       )
       return self.encode(text=A_					)





import itertools
import os
import random
import tempfile
import unittest

import numpy as np
from datasets import load_dataset

from transformers import is_speech_available
from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_speech_available():
 from transformers import WhisperFeatureExtractor

if is_torch_available():
 import torch

_UpperCamelCase 			=		random.Random()







def 		_lowercase (     lowercase__   ,	lowercase__=1.0   ,	lowercase__=None   ,	lowercase__=None						):
      if rng is None:
            __lowerCAmelCase					:						Any					=					global_rng

      __lowerCAmelCase					:						str					=					[]
      for batch_idx in range(shape[0]						):
            values.append([]						)
            for _ in range(shape[1]						):
                  values[-1].append(rng.random() * scale						)

      return values







@require_torch
@require_torchaudio
class   __lowercase      (unittest.TestCase ):
 def __init__(							self      ,							A_      ,							A_=7      ,							A_=400      ,							A_=2000      ,							A_=10      ,							A_=160      ,							A_=8      ,							A_=0.0      ,							A_=4000      ,							A_=False      ,							A_=True      ,							)					->List[str]:



       '''simple docstring'''





       __lowerCAmelCase					:						Optional[int]					=					parent
       __lowerCAmelCase					:						Dict					=					batch_size
       __lowerCAmelCase					:						str					=					min_seq_length
       __lowerCAmelCase					:						int					=					max_seq_length
       __lowerCAmelCase					:						Any					=					(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
       __lowerCAmelCase					:						Any					=					padding_value
       __lowerCAmelCase					:						str					=					sampling_rate
       __lowerCAmelCase					:						Optional[Any]					=					return_attention_mask
       __lowerCAmelCase					:						Optional[Any]					=					do_normalize
       __lowerCAmelCase					:						Optional[Any]					=					feature_size
       __lowerCAmelCase					:						Optional[int]					=					chunk_length
       __lowerCAmelCase					:						Optional[Any]					=					hop_length
 def 	UpperCamelCase__  (							self					)					->Optional[Any]:



       '''simple docstring'''





       return {
           "feature_size": self.feature_size,
           "hop_length": self.hop_length,
           "chunk_length": self.chunk_length,
           "padding_value": self.padding_value,
           "sampling_rate": self.sampling_rate,
           "return_attention_mask": self.return_attention_mask,
           "do_normalize": self.do_normalize,
       }





 def 	UpperCamelCase__  (							self      ,							A_=False      ,							A_=False					)					->Optional[Any]:



       '''simple docstring'''





       def _flatten(A_					):
             return list(itertools.chain(*A_					)					)

       if equal_length:
             __lowerCAmelCase					:						str					=					[floats_list((self.max_seq_length, self.feature_size)					) for _ in range(self.batch_size					)]
       else:
             # make sure that inputs increase in size
             __lowerCAmelCase					:						Any					=					[
                 floats_list((x, self.feature_size)					)
                 for x in range(self.min_seq_length      ,							self.max_seq_length      ,							self.seq_length_diff					)
             ]
       if numpify:
             __lowerCAmelCase					:						Optional[Any]					=					[np.asarray(A_					) for x in speech_inputs]
       return speech_inputs







@require_torch
@require_torchaudio
class   __lowercase      (_UpperCAmelCase		,   unittest.TestCase ):
 _UpperCamelCase			  =  WhisperFeatureExtractor if is_speech_available() else None
 def 	UpperCamelCase__  (							self					)					->Optional[int]:



       '''simple docstring'''





       __lowerCAmelCase					:						Tuple					=					WhisperFeatureExtractionTester(self					)
 def 	UpperCamelCase__  (							self					)					->int:



       '''simple docstring'''





       __lowerCAmelCase					:						str					=					self.feature_extraction_class(**self.feat_extract_dict					)

       with tempfile.TemporaryDirectory() as tmpdirname:
             __lowerCAmelCase					:						List[str]					=					feat_extract_first.save_pretrained(A_					)[0]
             check_json_file_has_correct_format(A_					)
             __lowerCAmelCase					:						int					=					self.feature_extraction_class.from_pretrained(A_					)

       __lowerCAmelCase					:						Dict					=					feat_extract_first.to_dict()
       __lowerCAmelCase					:						Union[str, Any]					=					feat_extract_second.to_dict()
       __lowerCAmelCase					:						Union[str, Any]					=					feat_extract_first.mel_filters
       __lowerCAmelCase					:						Dict					=					feat_extract_second.mel_filters
       self.assertTrue(np.allclose(A_      ,							A_					)					)
       self.assertEqual(A_      ,							A_					)
 def 	UpperCamelCase__  (							self					)					->str:



       '''simple docstring'''





       __lowerCAmelCase					:						str					=					self.feature_extraction_class(**self.feat_extract_dict					)

       with tempfile.TemporaryDirectory() as tmpdirname:
             __lowerCAmelCase					:						Union[str, Any]					=					os.path.join(A_      ,							'''feat_extract.json'''					)
             feat_extract_first.to_json_file(A_					)
             __lowerCAmelCase					:						List[str]					=					self.feature_extraction_class.from_json_file(A_					)

       __lowerCAmelCase					:						List[str]					=					feat_extract_first.to_dict()
       __lowerCAmelCase					:						Tuple					=					feat_extract_second.to_dict()
       __lowerCAmelCase					:						Any					=					feat_extract_first.mel_filters
       __lowerCAmelCase					:						List[str]					=					feat_extract_second.mel_filters
       self.assertTrue(np.allclose(A_      ,							A_					)					)
       self.assertEqual(A_      ,							A_					)
 def 	UpperCamelCase__  (							self					)					->List[Any]:



       '''simple docstring'''





       __lowerCAmelCase					:						List[str]					=					self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
       # create three inputs of length 800, 1000, and 1200
       __lowerCAmelCase					:						Dict					=					[floats_list((1, x)					)[0] for x in range(800      ,							1400      ,							200					)]
       __lowerCAmelCase					:						Dict					=					[np.asarray(A_					) for speech_input in speech_inputs]

       # Test feature size
       __lowerCAmelCase					:						Tuple					=					feature_extractor(A_      ,							padding='''max_length'''      ,							return_tensors='''np'''					).input_features
       self.assertTrue(input_features.ndim == 3					)
       self.assertTrue(input_features.shape[-1] == feature_extractor.nb_max_frames					)
       self.assertTrue(input_features.shape[-2] == feature_extractor.feature_size					)

       # Test not batched input
       __lowerCAmelCase					:						Dict					=					feature_extractor(speech_inputs[0]      ,							return_tensors='''np'''					).input_features
       __lowerCAmelCase					:						List[str]					=					feature_extractor(np_speech_inputs[0]      ,							return_tensors='''np'''					).input_features
       self.assertTrue(np.allclose(A_      ,							A_      ,							atol=1e-3					)					)

       # Test batched
       __lowerCAmelCase					:						Union[str, Any]					=					feature_extractor(A_      ,							return_tensors='''np'''					).input_features
       __lowerCAmelCase					:						List[Any]					=					feature_extractor(A_      ,							return_tensors='''np'''					).input_features
       for enc_seq_a, enc_seq_a in zip(A_      ,							A_					):
             self.assertTrue(np.allclose(A_      ,							A_      ,							atol=1e-3					)					)

       # Test 2-D numpy arrays are batched.
       __lowerCAmelCase					:						int					=					[floats_list((1, x)					)[0] for x in (800, 800, 800)]
       __lowerCAmelCase					:						Optional[int]					=					np.asarray(A_					)
       __lowerCAmelCase					:						Dict					=					feature_extractor(A_      ,							return_tensors='''np'''					).input_features
       __lowerCAmelCase					:						Any					=					feature_extractor(A_      ,							return_tensors='''np'''					).input_features
       for enc_seq_a, enc_seq_a in zip(A_      ,							A_					):
             self.assertTrue(np.allclose(A_      ,							A_      ,							atol=1e-3					)					)

       # Test truncation required
       __lowerCAmelCase					:						Optional[int]					=					[floats_list((1, x)					)[0] for x in range(200      ,							(feature_extractor.n_samples + 500)      ,							200					)]
       __lowerCAmelCase					:						Dict					=					[np.asarray(A_					) for speech_input in speech_inputs]

       __lowerCAmelCase					:						Tuple					=					[x[: feature_extractor.n_samples] for x in speech_inputs]
       __lowerCAmelCase					:						Optional[int]					=					[np.asarray(A_					) for speech_input in speech_inputs_truncated]

       __lowerCAmelCase					:						Any					=					feature_extractor(A_      ,							return_tensors='''np'''					).input_features
       __lowerCAmelCase					:						List[str]					=					feature_extractor(A_      ,							return_tensors='''np'''					).input_features
       for enc_seq_a, enc_seq_a in zip(A_      ,							A_					):
             self.assertTrue(np.allclose(A_      ,							A_      ,							atol=1e-3					)					)
 def 	UpperCamelCase__  (							self					)					->Dict:



       '''simple docstring'''





       import torch

       __lowerCAmelCase					:						Tuple					=					self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
       __lowerCAmelCase					:						List[Any]					=					np.random.rand(100      ,							32					).astype(np.floataa					)
       __lowerCAmelCase					:						Any					=					np_speech_inputs.tolist()

       for inputs in [py_speech_inputs, np_speech_inputs]:
             __lowerCAmelCase					:						Tuple					=					feature_extractor.pad([{'''input_features''': inputs}]      ,							return_tensors='''np'''					)
             self.assertTrue(np_processed.input_features.dtype == np.floataa					)
             __lowerCAmelCase					:						int					=					feature_extractor.pad([{'''input_features''': inputs}]      ,							return_tensors='''pt'''					)
             self.assertTrue(pt_processed.input_features.dtype == torch.floataa					)
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       __lowerCAmelCase					:						Any					=					load_dataset('''hf-internal-testing/librispeech_asr_dummy'''      ,							'''clean'''      ,							split='''validation'''					)
       # automatic decoding with librispeech
       __lowerCAmelCase					:						Union[str, Any]					=					ds.sort('''id'''					).select(range(A_					)					)[:num_samples]['''audio''']

       return [x["array"] for x in speech_samples]
 def 	UpperCamelCase__  (							self					)					->int:



       '''simple docstring'''





       __lowerCAmelCase					:						Optional[int]					=					torch.tensor(
           [
               0.1_193, -0.0_946, -0.1_098, -0.0_196, 0.0_225, -0.0_690, -0.1_736, 0.0_951,
               0.0_971, -0.0_817, -0.0_702, 0.0_162, 0.0_260, 0.0_017, -0.0_192, -0.1_678,
               0.0_709, -0.1_867, -0.0_655, -0.0_274, -0.0_234, -0.1_884, -0.0_516, -0.0_554,
               -0.0_274, -0.1_425, -0.1_423, 0.0_837, 0.0_377, -0.0_854
           ]					)
       # fmt: on

       __lowerCAmelCase					:						int					=					self._load_datasamples(1					)
       __lowerCAmelCase					:						Any					=					WhisperFeatureExtractor()
       __lowerCAmelCase					:						Optional[Any]					=					feature_extractor(A_      ,							return_tensors='''pt'''					).input_features
       self.assertEqual(input_features.shape      ,							(1, 80, 3000)					)
       self.assertTrue(torch.allclose(input_features[0, 0, :30]      ,							A_      ,							atol=1e-4					)					)





 def 	UpperCamelCase__  (							self					)					->Any:



       '''simple docstring'''





       __lowerCAmelCase					:						Tuple					=					self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
       __lowerCAmelCase					:						str					=					self._load_datasamples(1					)[0]
       __lowerCAmelCase					:						Optional[Any]					=					((audio - audio.min()) / (audio.max() - audio.min())) * 6_5535  # Rescale to [0, 65535] to show issue
       __lowerCAmelCase					:						Union[str, Any]					=					feat_extract.zero_mean_unit_var_norm([audio]      ,							attention_mask=A_					)[0]

       self.assertTrue(np.all(np.mean(A_					) < 1e-3					)					)
       self.assertTrue(np.all(np.abs(np.var(A_					) - 1					) < 1e-3					)					)


