
"""simple docstring"""
# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import importlib.metadata
import json
import os
from dataclasses import dataclass
from typing import Any, Dict, Union

from packaging import version

from ..utils import is_torch_available, logging


if is_torch_available():
      import torch


lowercase_							:    List[Any]         =						logging.get_logger(__name__)
@dataclass
class       UpperCamelCase   :
     def __init__(  self	,  snake_case__=False	,  snake_case__=False	,  snake_case__=6.0	,  snake_case__=None	,  snake_case__=False	,  snake_case__=False	,  snake_case__=None	,  snake_case__="fp4"	,  snake_case__=False	,  **snake_case__	,  ):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE :					str				     =			load_in_abit
       _SCREAMING_SNAKE_CASE :					Union[str, Any]				     =			load_in_abit
       _SCREAMING_SNAKE_CASE :					int				     =			llm_inta_threshold
       _SCREAMING_SNAKE_CASE :					Optional[int]				     =			llm_inta_skip_modules
       _SCREAMING_SNAKE_CASE :					int				     =			llm_inta_enable_fpaa_cpu_offload
       _SCREAMING_SNAKE_CASE :					Dict				     =			llm_inta_has_fpaa_weight
       _SCREAMING_SNAKE_CASE :					Optional[int]				     =			bnb_abit_quant_type
       _SCREAMING_SNAKE_CASE :					List[str]				     =			bnb_abit_use_double_quant

       if bnb_abit_compute_dtype is None:
         _SCREAMING_SNAKE_CASE :					Union[str, Any]				     =			torch.floataa
       elif isinstance(__lowerCAmelCase	,  __lowerCAmelCase					):
         _SCREAMING_SNAKE_CASE :					Optional[Any]				     =			getattr(__lowerCAmelCase	,  __lowerCAmelCase					)
       elif isinstance(__lowerCAmelCase	,  torch.dtype					):
         _SCREAMING_SNAKE_CASE :					Any				     =			bnb_abit_compute_dtype
       else:
         raise ValueError("bnb_4bit_compute_dtype must be a string or a torch.dtype"					)

       self.post_init()
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       if not isinstance(self.llm_inta_threshold	,  __lowerCAmelCase					):
         raise ValueError("llm_int8_threshold must be a float"					)

       if self.llm_inta_skip_modules is not None and not isinstance(self.llm_inta_skip_modules	,  __lowerCAmelCase					):
         raise ValueError("llm_int8_skip_modules must be a list of strings"					)
       if not isinstance(self.llm_inta_enable_fpaa_cpu_offload	,  __lowerCAmelCase					):
         raise ValueError("llm_int8_enable_fp32_cpu_offload must be a boolean"					)

       if not isinstance(self.llm_inta_has_fpaa_weight	,  __lowerCAmelCase					):
         raise ValueError("llm_int8_has_fp16_weight must be a boolean"					)

       if self.bnb_abit_compute_dtype is not None and not isinstance(self.bnb_abit_compute_dtype	,  torch.dtype					):
         raise ValueError("bnb_4bit_compute_dtype must be torch.dtype"					)

       if not isinstance(self.bnb_abit_quant_type	,  __lowerCAmelCase					):
         raise ValueError("bnb_4bit_quant_type must be a string"					)

       if not isinstance(self.bnb_abit_use_double_quant	,  __lowerCAmelCase					):
         raise ValueError("bnb_4bit_use_double_quant must be a boolean"					)

       if self.load_in_abit and not version.parse(importlib.metadata.version("bitsandbytes"					)					) >= version.parse(
           "0.39.0"					):
         raise ValueError(
             "4 bit quantization requires bitsandbytes>=0.39.0 - please upgrade your bitsandbytes version"					)
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       return self.load_in_abit or self.load_in_abit
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       if self.load_in_abit:
         return "llm_int8"
       elif self.load_in_abit and self.bnb_abit_quant_type == "fp4":
         return "fp4"
       elif self.load_in_abit and self.bnb_abit_quant_type == "nf4":
         return "nf4"
       else:
         return None
     @classmethod
     def 					__SCREAMING_SNAKE_CASE				(  cls	,  snake_case__	,  snake_case__	,  **snake_case__					):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE :					Tuple				     =			cls(**__lowerCAmelCase					)

       _SCREAMING_SNAKE_CASE :					List[Any]				     =			[]
       for key, value in kwargs.items():
         if hasattr(__lowerCAmelCase	,  __lowerCAmelCase					):
           setattr(__lowerCAmelCase	,  __lowerCAmelCase	,  __lowerCAmelCase					)
           to_remove.append(__lowerCAmelCase					)
       for key in to_remove:
         kwargs.pop(__lowerCAmelCase	,  __lowerCAmelCase					)

       if return_unused_kwargs:
         return config, kwargs
       else:
         return config
     def 					__SCREAMING_SNAKE_CASE				(  self	,  snake_case__					):




       """simple docstring"""


       with open(__lowerCAmelCase	,  "w"	,  encoding="utf-8"					) as writer:
         _SCREAMING_SNAKE_CASE :					List[str]				     =			self.to_dict()
         _SCREAMING_SNAKE_CASE :					Any				     =			json.dumps(__lowerCAmelCase	,  indent=2	,  sort_keys=__lowerCAmelCase					) + "\n"

         writer.write(__lowerCAmelCase					)
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE :					int				     =			copy.deepcopy(self.__dict__					)
       _SCREAMING_SNAKE_CASE :					Any				     =			str(output["bnb_4bit_compute_dtype"]					).split("."					)[1]

       return output
     def __repr__(  self					):




       """simple docstring"""


       return F'''{self.__class__.__name__} {self.to_json_string()}'''
     def 					__SCREAMING_SNAKE_CASE				(  self	,  snake_case__ = True					):




       """simple docstring"""


       if use_diff is True:
         _SCREAMING_SNAKE_CASE :					int				     =			self.to_diff_dict()
       else:
         _SCREAMING_SNAKE_CASE :					Tuple				     =			self.to_dict()
       return json.dumps(__lowerCAmelCase	,  indent=2	,  sort_keys=__lowerCAmelCase					) + "\n"




     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE :					int				     =			self.to_dict()

       # get the default config dict
       _SCREAMING_SNAKE_CASE :					List[str]				     =			BitsAndBytesConfig().to_dict()

       _SCREAMING_SNAKE_CASE :					Any				     =			{}

       # only serialize values that differ from the default config
       for key, value in config_dict.items():
         if value != default_config_dict[key]:
           _SCREAMING_SNAKE_CASE :					Tuple				     =			value

       return serializable_config_dict










import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 							__SCREAMING_SNAKE_CASE       (  _a ,		unittest.TestCase							):
							snake_case  :   Dict      =							CodeGenTokenizer
							snake_case  :   Dict      =							CodeGenTokenizerFast
							snake_case  :   Tuple      =							True
							snake_case  :   Optional[int]      =							{"""add_prefix_space""": True}
							snake_case  :   int      =							False






							def 					_lowerCamelCase		(    self	):
														super().setUp()

														# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
														UpperCamelCase__					  =      [
														    """l""",
														    """o""",
														    """w""",
														    """e""",
														    """r""",
														    """s""",
														    """t""",
														    """i""",
														    """d""",
														    """n""",
														    """\u0120""",
														    """\u0120l""",
														    """\u0120n""",
														    """\u0120lo""",
														    """\u0120low""",
														    """er""",
														    """\u0120lowest""",
														    """\u0120newer""",
														    """\u0120wider""",
														    """<unk>""",
														    """<|endoftext|>""",
														]
														UpperCamelCase__					  =      dict(zip(__lowerCAmelCase	,			range(len(__lowerCAmelCase	)	)	)	)
														UpperCamelCase__					  =      ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
														UpperCamelCase__					  =      {"""unk_token""": """<unk>"""}

														UpperCamelCase__					  =      os.path.join(self.tmpdirname	,			VOCAB_FILES_NAMES["""vocab_file"""]	)
														UpperCamelCase__					  =      os.path.join(self.tmpdirname	,			VOCAB_FILES_NAMES["""merges_file"""]	)
														with open(self.vocab_file	,			"""w"""	,			encoding="""utf-8"""	) as fp:
																					fp.write(json.dumps(__lowerCAmelCase	) + """\n"""	)
														with open(self.merges_file	,			"""w"""	,			encoding="""utf-8"""	) as fp:
																					fp.write("""\n""".join(__lowerCAmelCase	)	)






							def 					_lowerCamelCase		(    self	,			**__lowerCAmelCase	):
														kwargs.update(self.special_tokens_map	)
														return CodeGenTokenizer.from_pretrained(self.tmpdirname	,			**__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			**__lowerCAmelCase	):
														kwargs.update(self.special_tokens_map	)
														return CodeGenTokenizerFast.from_pretrained(self.tmpdirname	,			**__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      """lower newer"""
														UpperCamelCase__					  =      """lower newer"""
														return input_text, output_text






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      CodeGenTokenizer(self.vocab_file	,			self.merges_file	,			**self.special_tokens_map	)
														UpperCamelCase__					  =      """lower newer"""
														UpperCamelCase__					  =      ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
														UpperCamelCase__					  =      tokenizer.tokenize(__lowerCAmelCase	,			add_prefix_space=__lowerCAmelCase	)
														self.assertListEqual(__lowerCAmelCase	,			__lowerCAmelCase	)

														UpperCamelCase__					  =      tokens + [tokenizer.unk_token]
														UpperCamelCase__					  =      [14, 15, 10, 9, 3, 2, 15, 19]
														self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCAmelCase	)	,			__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	):
														if not self.test_rust_tokenizer:
																					return

														UpperCamelCase__					  =      self.get_tokenizer()
														UpperCamelCase__					  =      self.get_rust_tokenizer(add_prefix_space=__lowerCAmelCase	)

														UpperCamelCase__					  =      """lower newer"""

														# Testing tokenization
														UpperCamelCase__					  =      tokenizer.tokenize(__lowerCAmelCase	,			add_prefix_space=__lowerCAmelCase	)
														UpperCamelCase__					  =      rust_tokenizer.tokenize(__lowerCAmelCase	)
														self.assertListEqual(__lowerCAmelCase	,			__lowerCAmelCase	)

														# Testing conversion to ids without special tokens
														UpperCamelCase__					  =      tokenizer.encode(__lowerCAmelCase	,			add_special_tokens=__lowerCAmelCase	,			add_prefix_space=__lowerCAmelCase	)
														UpperCamelCase__					  =      rust_tokenizer.encode(__lowerCAmelCase	,			add_special_tokens=__lowerCAmelCase	)
														self.assertListEqual(__lowerCAmelCase	,			__lowerCAmelCase	)

														# Testing conversion to ids with special tokens
														UpperCamelCase__					  =      self.get_rust_tokenizer(add_prefix_space=__lowerCAmelCase	)
														UpperCamelCase__					  =      tokenizer.encode(__lowerCAmelCase	,			add_prefix_space=__lowerCAmelCase	)
														UpperCamelCase__					  =      rust_tokenizer.encode(__lowerCAmelCase	)
														self.assertListEqual(__lowerCAmelCase	,			__lowerCAmelCase	)

														# Testing the unknown token
														UpperCamelCase__					  =      tokens + [rust_tokenizer.unk_token]
														UpperCamelCase__					  =      [14, 15, 10, 9, 3, 2, 15, 19]
														self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__lowerCAmelCase	)	,			__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			*__lowerCAmelCase	,			**__lowerCAmelCase	):
														# It's very difficult to mix/test pretokenization with byte-level
														# And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
														pass






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase=15	):
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
																												UpperCamelCase__					  =      self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase	,			**__lowerCAmelCase	)

																												# Simple input
																												UpperCamelCase__					  =      """This is a simple input"""
																												UpperCamelCase__					  =      ["""This is a simple input 1""", """This is a simple input 2"""]
																												UpperCamelCase__					  =      ("""This is a simple input""", """This is a pair""")
																												UpperCamelCase__					  =      [
																												    ("""This is a simple input 1""", """This is a simple input 2"""),
																												    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																												]

																												# Simple input tests
																												self.assertRaises(__lowerCAmelCase	,			tokenizer_r.encode	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	)

																												# Simple input
																												self.assertRaises(__lowerCAmelCase	,			tokenizer_r.encode_plus	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	)

																												# Simple input
																												self.assertRaises(
																												    __lowerCAmelCase	,			tokenizer_r.batch_encode_plus	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	,			)

																												# Pair input
																												self.assertRaises(__lowerCAmelCase	,			tokenizer_r.encode	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	)

																												# Pair input
																												self.assertRaises(__lowerCAmelCase	,			tokenizer_r.encode_plus	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	)

																												# Pair input
																												self.assertRaises(
																												    __lowerCAmelCase	,			tokenizer_r.batch_encode_plus	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	,			)






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      CodeGenTokenizer.from_pretrained(self.tmpdirname	,			pad_token="""<pad>"""	)

														# Simple input
														UpperCamelCase__					  =      """This is a simple input"""
														UpperCamelCase__					  =      ["""This is a simple input looooooooong""", """This is a simple input"""]
														UpperCamelCase__					  =      ("""This is a simple input""", """This is a pair""")
														UpperCamelCase__					  =      [
														    ("""This is a simple input loooooong""", """This is a simple input"""),
														    ("""This is a simple pair loooooong""", """This is a simple pair"""),
														]

														UpperCamelCase__					  =      tokenizer.pad_token_id

														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	,			padding="""max_length"""	,			max_length=30	,			return_tensors="""np"""	)
														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	,			padding=__lowerCAmelCase	,			truncate=__lowerCAmelCase	,			return_tensors="""np"""	)
														UpperCamelCase__					  =      tokenizer(*__lowerCAmelCase	,			padding="""max_length"""	,			max_length=60	,			return_tensors="""np"""	)
														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	,			padding=__lowerCAmelCase	,			truncate=__lowerCAmelCase	,			return_tensors="""np"""	)

														# s
														# test single string max_length padding
														self.assertEqual(out_s["""input_ids"""].shape[-1]	,			30	)
														self.assertTrue(pad_token_id in out_s["""input_ids"""]	)
														self.assertTrue(0 in out_s["""attention_mask"""]	)

														# s2
														# test automatic padding
														self.assertEqual(out_sa["""input_ids"""].shape[-1]	,			33	)
														# long slice doesn't have padding
														self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]	)
														self.assertFalse(0 in out_sa["""attention_mask"""][0]	)
														# short slice does have padding
														self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]	)
														self.assertTrue(0 in out_sa["""attention_mask"""][1]	)

														# p
														# test single pair max_length padding
														self.assertEqual(out_p["""input_ids"""].shape[-1]	,			60	)
														self.assertTrue(pad_token_id in out_p["""input_ids"""]	)
														self.assertTrue(0 in out_p["""attention_mask"""]	)

														# p2
														# test automatic padding pair
														self.assertEqual(out_pa["""input_ids"""].shape[-1]	,			52	)
														# long slice pair doesn't have padding
														self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]	)
														self.assertFalse(0 in out_pa["""attention_mask"""][0]	)
														# short slice pair does have padding
														self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]	)
														self.assertTrue(0 in out_pa["""attention_mask"""][1]	)






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      """$$$"""
														UpperCamelCase__					  =      CodeGenTokenizer.from_pretrained(self.tmpdirname	,			bos_token=__lowerCAmelCase	,			add_bos_token=__lowerCAmelCase	)

														UpperCamelCase__					  =      """This is a simple input"""
														UpperCamelCase__					  =      ["""This is a simple input 1""", """This is a simple input 2"""]

														UpperCamelCase__					  =      tokenizer.bos_token_id

														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	)
														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	)

														self.assertEqual(out_s.input_ids[0]	,			__lowerCAmelCase	)
														self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids	)	)

														UpperCamelCase__					  =      tokenizer.decode(out_s.input_ids	)
														UpperCamelCase__					  =      tokenizer.batch_decode(out_sa.input_ids	)

														self.assertEqual(decode_s.split()[0]	,			__lowerCAmelCase	)
														self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa	)	)






							@slow
							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""	)

														UpperCamelCase__					  =      """\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
														UpperCamelCase__					  =      """\nif len_a > len_b:      result = a\nelse:      result = b"""

														UpperCamelCase__					  =      tokenizer.encode(__lowerCAmelCase	)
														UpperCamelCase__					  =      ["""^#""", re.escape("""<|endoftext|>"""	), """^'''""", """^\"\"\"""", """\n\n\n"""]
														UpperCamelCase__					  =      tokenizer.decode(__lowerCAmelCase	,			truncate_before_pattern=__lowerCAmelCase	)
														self.assertEqual(__lowerCAmelCase	,			__lowerCAmelCase	)







							def 					_lowerCamelCase		(    self	):
														pass




