





# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import importlib.metadata
import json
import os
from dataclasses import dataclass
from typing import Any, Dict, Union

from packaging import version

from ..utils import is_torch_available, logging


if is_torch_available():
	import torch


lowerCAmelCase_    =				logging.get_logger(__name__)



@dataclass
class 					__lowerCAmelCase       :






			def __init__(self					,							__magic_name__=False					,							__magic_name__=False					,							__magic_name__=6.0					,							__magic_name__=None					,							__magic_name__=False					,							__magic_name__=False					,							__magic_name__=None					,							__magic_name__="fp4"					,							__magic_name__=False					,							**__magic_name__					,							)     ->   Optional[int]:


							'''simple docstring'''



							snake_case_						:    List[Any]				=  load_in_abit
							snake_case_						:    Optional[int]				=  load_in_abit
							snake_case_						:    str				=  llm_inta_threshold
							snake_case_						:    List[str]				=  llm_inta_skip_modules
							snake_case_						:    Optional[int]				=  llm_inta_enable_fpaa_cpu_offload
							snake_case_						:    List[Any]				=  llm_inta_has_fpaa_weight
							snake_case_						:    int				=  bnb_abit_quant_type
							snake_case_						:    List[Any]				=  bnb_abit_use_double_quant

							if bnb_abit_compute_dtype is None:
											snake_case_						:    Tuple				=  torch.floataa
							elif isinstance(__magic_name__					,							__magic_name__    ):
											snake_case_						:    List[str]				=  getattr(__magic_name__					,							__magic_name__    )
							elif isinstance(__magic_name__					,							torch.dtype    ):
											snake_case_						:    Any				=  bnb_abit_compute_dtype
							else:
											raise ValueError('''bnb_4bit_compute_dtype must be a string or a torch.dtype'''    )

							self.post_init()






			def 			lowerCamelCase  (self    )     ->   str:


							'''simple docstring'''



							if not isinstance(self.llm_inta_threshold					,							__magic_name__    ):
											raise ValueError('''llm_int8_threshold must be a float'''    )

							if self.llm_inta_skip_modules is not None and not isinstance(self.llm_inta_skip_modules					,							__magic_name__    ):
											raise ValueError('''llm_int8_skip_modules must be a list of strings'''    )
							if not isinstance(self.llm_inta_enable_fpaa_cpu_offload					,							__magic_name__    ):
											raise ValueError('''llm_int8_enable_fp32_cpu_offload must be a boolean'''    )

							if not isinstance(self.llm_inta_has_fpaa_weight					,							__magic_name__    ):
											raise ValueError('''llm_int8_has_fp16_weight must be a boolean'''    )

							if self.bnb_abit_compute_dtype is not None and not isinstance(self.bnb_abit_compute_dtype					,							torch.dtype    ):
											raise ValueError('''bnb_4bit_compute_dtype must be torch.dtype'''    )

							if not isinstance(self.bnb_abit_quant_type					,							__magic_name__    ):
											raise ValueError('''bnb_4bit_quant_type must be a string'''    )

							if not isinstance(self.bnb_abit_use_double_quant					,							__magic_name__    ):
											raise ValueError('''bnb_4bit_use_double_quant must be a boolean'''    )

							if self.load_in_abit and not version.parse(importlib.metadata.version('''bitsandbytes'''    )    ) >= version.parse(
							    '''0.39.0'''    ):
											raise ValueError(
											    '''4 bit quantization requires bitsandbytes>=0.39.0 - please upgrade your bitsandbytes version'''    )






			def 			lowerCamelCase  (self    )     ->   Dict:


							'''simple docstring'''



							return self.load_in_abit or self.load_in_abit






			def 			lowerCamelCase  (self    )     ->   Optional[int]:


							'''simple docstring'''



							if self.load_in_abit:
											return "llm_int8"
							elif self.load_in_abit and self.bnb_abit_quant_type == "fp4":
											return "fp4"
							elif self.load_in_abit and self.bnb_abit_quant_type == "nf4":
											return "nf4"
							else:
											return None






			@classmethod
			def 			lowerCamelCase  (cls					,							__magic_name__					,							__magic_name__					,							**__magic_name__    )     ->   int:


							'''simple docstring'''



							snake_case_						:    Tuple				=  cls(**__magic_name__    )

							snake_case_						:    Dict				=  []
							for key, value in kwargs.items():
											if hasattr(__magic_name__					,							__magic_name__    ):
															setattr(__magic_name__					,							__magic_name__					,							__magic_name__    )
															to_remove.append(__magic_name__    )
							for key in to_remove:
											kwargs.pop(__magic_name__					,							__magic_name__    )

							if return_unused_kwargs:
											return config, kwargs
							else:
											return config






			def 			lowerCamelCase  (self					,							__magic_name__    )     ->   Optional[int]:


							'''simple docstring'''



							with open(__magic_name__					,							'''w'''					,							encoding='''utf-8'''    ) as writer:
											snake_case_						:    Tuple				=  self.to_dict()
											snake_case_						:    List[Any]				=  json.dumps(__magic_name__					,							indent=2					,							sort_keys=__magic_name__    ) + '''\n'''

											writer.write(__magic_name__    )






			def 			lowerCamelCase  (self    )     ->   Dict[str, Any]:


							'''simple docstring'''



							snake_case_						:    Tuple				=  copy.deepcopy(self.__dict__    )
							snake_case_						:    Any				=  str(output['''bnb_4bit_compute_dtype''']    ).split('''.'''    )[1]

							return output






			def __repr__(self    )     ->   Optional[Any]:


							'''simple docstring'''



							return F'''{self.__class__.__name__} {self.to_json_string()}'''






			def 			lowerCamelCase  (self					,							__magic_name__ = True    )     ->   str:


							'''simple docstring'''



							if use_diff is True:
											snake_case_						:    Tuple				=  self.to_diff_dict()
							else:
											snake_case_						:    List[Any]				=  self.to_dict()
							return json.dumps(__magic_name__					,							indent=2					,							sort_keys=__magic_name__    ) + "\n"






			def 			lowerCamelCase  (self    )     ->   Dict[str, Any]:


							'''simple docstring'''



							snake_case_						:    Optional[Any]				=  self.to_dict()

							# get the default config dict
							snake_case_						:    Dict				=  BitsAndBytesConfig().to_dict()

							snake_case_						:    str				=  {}

							# only serialize values that differ from the default config
							for key, value in config_dict.items():
											if value != default_config_dict[key]:
															snake_case_						:    Optional[int]				=  value

							return serializable_config_dict







import json
import os
import unittest

from transformers.models.biogpt.tokenization_biogpt import VOCAB_FILES_NAMES, BioGptTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin



class 					__lowerCAmelCase       (			_a,     unittest.TestCase     ):
			lowerCamelCase_     :     Any       			=      BioGptTokenizer
			lowerCamelCase_     :     Optional[Any]       			=      False






			def 			lowerCamelCase  (self    )     ->   List[Any]:


							'''simple docstring'''



							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							snake_case_						:    Optional[Any]				=  [
							    '''l''',
							    '''o''',
							    '''w''',
							    '''e''',
							    '''r''',
							    '''s''',
							    '''t''',
							    '''i''',
							    '''d''',
							    '''n''',
							    '''w</w>''',
							    '''r</w>''',
							    '''t</w>''',
							    '''lo''',
							    '''low''',
							    '''er</w>''',
							    '''low</w>''',
							    '''lowest</w>''',
							    '''newer</w>''',
							    '''wider</w>''',
							    '''<unk>''',
							]
							snake_case_						:    Union[str, Any]				=  dict(zip(__magic_name__					,							range(len(__magic_name__    )    )    )    )
							snake_case_						:    Union[str, Any]				=  ['''l o 123''', '''lo w 1456''', '''e r</w> 1789''', '''''']

							snake_case_						:    Any				=  os.path.join(self.tmpdirname					,							VOCAB_FILES_NAMES['''vocab_file''']    )
							snake_case_						:    Dict				=  os.path.join(self.tmpdirname					,							VOCAB_FILES_NAMES['''merges_file''']    )
							with open(self.vocab_file					,							'''w'''    ) as fp:
											fp.write(json.dumps(__magic_name__    )    )
							with open(self.merges_file					,							'''w'''    ) as fp:
											fp.write('''\n'''.join(__magic_name__    )    )






			def 			lowerCamelCase  (self					,							__magic_name__    )     ->   int:


							'''simple docstring'''



							snake_case_						:    str				=  '''lower newer'''
							snake_case_						:    Dict				=  '''lower newer'''
							return input_text, output_text






			def 			lowerCamelCase  (self    )     ->   Union[str, Any]:


							'''simple docstring'''



							snake_case_						:    Optional[int]				=  BioGptTokenizer(self.vocab_file					,							self.merges_file    )

							snake_case_						:    Union[str, Any]				=  '''lower'''
							snake_case_						:    Optional[int]				=  ['''low''', '''er</w>''']
							snake_case_						:    Any				=  tokenizer.tokenize(__magic_name__    )
							self.assertListEqual(__magic_name__					,							__magic_name__    )

							snake_case_						:    Optional[int]				=  tokens + ['''<unk>''']
							snake_case_						:    List[str]				=  [14, 15, 20]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__    )					,							__magic_name__    )






			@slow
			def 			lowerCamelCase  (self    )     ->   Optional[Any]:


							'''simple docstring'''



							snake_case_						:    Optional[int]				=  BioGptTokenizer.from_pretrained('''microsoft/biogpt'''    )

							snake_case_						:    List[Any]				=  tokenizer.encode('''sequence builders'''					,							add_special_tokens=__magic_name__    )
							snake_case_						:    str				=  tokenizer.encode('''multi-sequence build'''					,							add_special_tokens=__magic_name__    )

							snake_case_						:    str				=  tokenizer.build_inputs_with_special_tokens(__magic_name__    )
							snake_case_						:    List[str]				=  tokenizer.build_inputs_with_special_tokens(__magic_name__					,							__magic_name__    )

							self.assertTrue(encoded_sentence == [2] + text    )
							self.assertTrue(encoded_pair == [2] + text + [2] + text_a    )
