





import argparse
import os
import re

import packaging.version


UpperCamelCase__           =			"examples/"
UpperCamelCase__           =			{
    "examples": (re.compile(r"^check_min_version\(\"[^\"]+\"\)\s*$", re.MULTILINE), "check_min_version(\"VERSION\")\n"),
    "init": (re.compile(r"^__version__\s+=\s+\"([^\"]+)\"\s*$", re.MULTILINE), "__version__ = \"VERSION\"\n"),
    "setup": (re.compile(r"^(\s*)version\s*=\s*\"[^\"]+\",", re.MULTILINE), r"\1version=\"VERSION\","),
    "doc": (re.compile(r"^(\s*)release\s*=\s*\"[^\"]+\"$", re.MULTILINE), "release = \"VERSION\"\n"),
}
UpperCamelCase__           =			{
    "init": "src/transformers/__init__.py",
    "setup": "setup.py",
}
UpperCamelCase__           =			"README.md"





def      _UpperCamelCase  (a__ :Union[str, Any] ,      a__ :Dict ,      a__ :int							):






							"""simple docstring"""





							with open(a__ ,      """r""" ,      encoding="""utf-8""" ,      newline="""\n"""							) as f:
														UpperCamelCase__					  =      f.read()
							UpperCamelCase__  ,  UpperCamelCase__					  =      REPLACE_PATTERNS[pattern]
							UpperCamelCase__					  =      replace.replace("""VERSION""" ,      a__							)
							UpperCamelCase__					  =      re_pattern.sub(a__ ,      a__							)
							with open(a__ ,      """w""" ,      encoding="""utf-8""" ,      newline="""\n"""							) as f:
														f.write(a__							)





def      _UpperCamelCase  (a__ :List[str]							):






							"""simple docstring"""





							for folder, directories, fnames in os.walk(a__							):
														# Removing some of the folders with non-actively maintained examples from the walk
														if "research_projects" in directories:
																					directories.remove("""research_projects"""							)
														if "legacy" in directories:
																					directories.remove("""legacy"""							)
														for fname in fnames:
																					if fname.endswith(""".py"""							):
																												update_version_in_file(os.path.join(a__ ,      a__							) ,      a__ ,      pattern="""examples"""							)





def      _UpperCamelCase  (a__ :Optional[int] ,      a__ :List[Any]=False							):






							"""simple docstring"""





							for pattern, fname in REPLACE_FILES.items():
														update_version_in_file(a__ ,      a__ ,      a__							)
							if not patch:
														update_version_in_examples(a__							)





def      _UpperCamelCase  ():






							"""simple docstring"""





							UpperCamelCase__					  =      """🤗 Transformers currently provides the following architectures"""
							UpperCamelCase__					  =      """1. Want to contribute a new model?"""
							with open(a__ ,      """r""" ,      encoding="""utf-8""" ,      newline="""\n"""							) as f:
														UpperCamelCase__					  =      f.readlines()

							# Find the start of the list.
							UpperCamelCase__					  =      0
							while not lines[start_index].startswith(_start_prompt							):
														start_index += 1
							start_index += 1

							UpperCamelCase__					  =      start_index
							# Update the lines in the model list.
							while not lines[index].startswith(_end_prompt							):
														if lines[index].startswith("""1."""							):
																					UpperCamelCase__					  =      lines[index].replace(
																					    """https://huggingface.co/docs/transformers/main/model_doc""" ,      """https://huggingface.co/docs/transformers/model_doc""" ,      )
														index += 1

							with open(a__ ,      """w""" ,      encoding="""utf-8""" ,      newline="""\n"""							) as f:
														f.writelines(a__							)





def      _UpperCamelCase  ():






							"""simple docstring"""





							with open(REPLACE_FILES["""init"""] ,      """r"""							) as f:
														UpperCamelCase__					  =      f.read()
							UpperCamelCase__					  =      REPLACE_PATTERNS["""init"""][0].search(a__							).groups()[0]
							return packaging.version.parse(a__							)





def      _UpperCamelCase  (a__ :Tuple=False							):






							"""simple docstring"""





							UpperCamelCase__					  =      get_version()
							if patch and default_version.is_devrelease:
														raise ValueError("""Can't create a patch version from the dev branch, checkout a released version!"""							)
							if default_version.is_devrelease:
														UpperCamelCase__					  =      default_version.base_version
							elif patch:
														UpperCamelCase__					  =      f"""{default_version.major}.{default_version.minor}.{default_version.micro + 1}"""
							else:
														UpperCamelCase__					  =      f"""{default_version.major}.{default_version.minor + 1}.0"""

							# Now let's ask nicely if that's the right one.
							UpperCamelCase__					  =      input(f"""Which version are you releasing? [{default_version}]"""							)
							if len(a__							) == 0:
														UpperCamelCase__					  =      default_version

							print(f"""Updating version to {version}."""							)
							global_version_update(a__ ,      patch=a__							)
							if not patch:
														print("""Cleaning main README, don't forget to run `make fix-copies`."""							)
														clean_main_ref_in_model_list()





def      _UpperCamelCase  ():






							"""simple docstring"""





							UpperCamelCase__					  =      get_version()
							UpperCamelCase__					  =      f"""{current_version.major}.{current_version.minor + 1}.0.dev0"""
							UpperCamelCase__					  =      current_version.base_version

							# Check with the user we got that right.
							UpperCamelCase__					  =      input(f"""Which version are we developing now? [{dev_version}]"""							)
							if len(a__							) == 0:
														UpperCamelCase__					  =      dev_version

							print(f"""Updating version to {version}."""							)
							global_version_update(a__							)
							print("""Cleaning main README, don't forget to run `make fix-copies`."""							)
							clean_main_ref_in_model_list()


if __name__ == "__main__":
					UpperCamelCase__           =			argparse.ArgumentParser()
					parser.add_argument("--post_release", action="store_true", help="Whether this is pre or post release.")
					parser.add_argument("--patch", action="store_true", help="Whether or not this is a patch release.")
					UpperCamelCase__           =			parser.parse_args()
					if not args.post_release:
										pre_release_work(patch=args.patch)
					elif args.patch:
										print("Nothing to do after a patch :-)")
					else:
										post_release_work()











import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 							__SCREAMING_SNAKE_CASE       (  _a ,		unittest.TestCase							):
							snake_case  :   Dict      =							CodeGenTokenizer
							snake_case  :   Dict      =							CodeGenTokenizerFast
							snake_case  :   Tuple      =							True
							snake_case  :   Optional[int]      =							{"""add_prefix_space""": True}
							snake_case  :   int      =							False






							def 					_lowerCamelCase		(    self	):
														super().setUp()

														# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
														UpperCamelCase__					  =      [
														    """l""",
														    """o""",
														    """w""",
														    """e""",
														    """r""",
														    """s""",
														    """t""",
														    """i""",
														    """d""",
														    """n""",
														    """\u0120""",
														    """\u0120l""",
														    """\u0120n""",
														    """\u0120lo""",
														    """\u0120low""",
														    """er""",
														    """\u0120lowest""",
														    """\u0120newer""",
														    """\u0120wider""",
														    """<unk>""",
														    """<|endoftext|>""",
														]
														UpperCamelCase__					  =      dict(zip(__lowerCAmelCase	,			range(len(__lowerCAmelCase	)	)	)	)
														UpperCamelCase__					  =      ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
														UpperCamelCase__					  =      {"""unk_token""": """<unk>"""}

														UpperCamelCase__					  =      os.path.join(self.tmpdirname	,			VOCAB_FILES_NAMES["""vocab_file"""]	)
														UpperCamelCase__					  =      os.path.join(self.tmpdirname	,			VOCAB_FILES_NAMES["""merges_file"""]	)
														with open(self.vocab_file	,			"""w"""	,			encoding="""utf-8"""	) as fp:
																					fp.write(json.dumps(__lowerCAmelCase	) + """\n"""	)
														with open(self.merges_file	,			"""w"""	,			encoding="""utf-8"""	) as fp:
																					fp.write("""\n""".join(__lowerCAmelCase	)	)






							def 					_lowerCamelCase		(    self	,			**__lowerCAmelCase	):
														kwargs.update(self.special_tokens_map	)
														return CodeGenTokenizer.from_pretrained(self.tmpdirname	,			**__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			**__lowerCAmelCase	):
														kwargs.update(self.special_tokens_map	)
														return CodeGenTokenizerFast.from_pretrained(self.tmpdirname	,			**__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      """lower newer"""
														UpperCamelCase__					  =      """lower newer"""
														return input_text, output_text






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      CodeGenTokenizer(self.vocab_file	,			self.merges_file	,			**self.special_tokens_map	)
														UpperCamelCase__					  =      """lower newer"""
														UpperCamelCase__					  =      ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
														UpperCamelCase__					  =      tokenizer.tokenize(__lowerCAmelCase	,			add_prefix_space=__lowerCAmelCase	)
														self.assertListEqual(__lowerCAmelCase	,			__lowerCAmelCase	)

														UpperCamelCase__					  =      tokens + [tokenizer.unk_token]
														UpperCamelCase__					  =      [14, 15, 10, 9, 3, 2, 15, 19]
														self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCAmelCase	)	,			__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	):
														if not self.test_rust_tokenizer:
																					return

														UpperCamelCase__					  =      self.get_tokenizer()
														UpperCamelCase__					  =      self.get_rust_tokenizer(add_prefix_space=__lowerCAmelCase	)

														UpperCamelCase__					  =      """lower newer"""

														# Testing tokenization
														UpperCamelCase__					  =      tokenizer.tokenize(__lowerCAmelCase	,			add_prefix_space=__lowerCAmelCase	)
														UpperCamelCase__					  =      rust_tokenizer.tokenize(__lowerCAmelCase	)
														self.assertListEqual(__lowerCAmelCase	,			__lowerCAmelCase	)

														# Testing conversion to ids without special tokens
														UpperCamelCase__					  =      tokenizer.encode(__lowerCAmelCase	,			add_special_tokens=__lowerCAmelCase	,			add_prefix_space=__lowerCAmelCase	)
														UpperCamelCase__					  =      rust_tokenizer.encode(__lowerCAmelCase	,			add_special_tokens=__lowerCAmelCase	)
														self.assertListEqual(__lowerCAmelCase	,			__lowerCAmelCase	)

														# Testing conversion to ids with special tokens
														UpperCamelCase__					  =      self.get_rust_tokenizer(add_prefix_space=__lowerCAmelCase	)
														UpperCamelCase__					  =      tokenizer.encode(__lowerCAmelCase	,			add_prefix_space=__lowerCAmelCase	)
														UpperCamelCase__					  =      rust_tokenizer.encode(__lowerCAmelCase	)
														self.assertListEqual(__lowerCAmelCase	,			__lowerCAmelCase	)

														# Testing the unknown token
														UpperCamelCase__					  =      tokens + [rust_tokenizer.unk_token]
														UpperCamelCase__					  =      [14, 15, 10, 9, 3, 2, 15, 19]
														self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__lowerCAmelCase	)	,			__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			*__lowerCAmelCase	,			**__lowerCAmelCase	):
														# It's very difficult to mix/test pretokenization with byte-level
														# And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
														pass






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase=15	):
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
																												UpperCamelCase__					  =      self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase	,			**__lowerCAmelCase	)

																												# Simple input
																												UpperCamelCase__					  =      """This is a simple input"""
																												UpperCamelCase__					  =      ["""This is a simple input 1""", """This is a simple input 2"""]
																												UpperCamelCase__					  =      ("""This is a simple input""", """This is a pair""")
																												UpperCamelCase__					  =      [
																												    ("""This is a simple input 1""", """This is a simple input 2"""),
																												    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																												]

																												# Simple input tests
																												self.assertRaises(__lowerCAmelCase	,			tokenizer_r.encode	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	)

																												# Simple input
																												self.assertRaises(__lowerCAmelCase	,			tokenizer_r.encode_plus	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	)

																												# Simple input
																												self.assertRaises(
																												    __lowerCAmelCase	,			tokenizer_r.batch_encode_plus	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	,			)

																												# Pair input
																												self.assertRaises(__lowerCAmelCase	,			tokenizer_r.encode	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	)

																												# Pair input
																												self.assertRaises(__lowerCAmelCase	,			tokenizer_r.encode_plus	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	)

																												# Pair input
																												self.assertRaises(
																												    __lowerCAmelCase	,			tokenizer_r.batch_encode_plus	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	,			)






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      CodeGenTokenizer.from_pretrained(self.tmpdirname	,			pad_token="""<pad>"""	)

														# Simple input
														UpperCamelCase__					  =      """This is a simple input"""
														UpperCamelCase__					  =      ["""This is a simple input looooooooong""", """This is a simple input"""]
														UpperCamelCase__					  =      ("""This is a simple input""", """This is a pair""")
														UpperCamelCase__					  =      [
														    ("""This is a simple input loooooong""", """This is a simple input"""),
														    ("""This is a simple pair loooooong""", """This is a simple pair"""),
														]

														UpperCamelCase__					  =      tokenizer.pad_token_id

														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	,			padding="""max_length"""	,			max_length=30	,			return_tensors="""np"""	)
														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	,			padding=__lowerCAmelCase	,			truncate=__lowerCAmelCase	,			return_tensors="""np"""	)
														UpperCamelCase__					  =      tokenizer(*__lowerCAmelCase	,			padding="""max_length"""	,			max_length=60	,			return_tensors="""np"""	)
														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	,			padding=__lowerCAmelCase	,			truncate=__lowerCAmelCase	,			return_tensors="""np"""	)

														# s
														# test single string max_length padding
														self.assertEqual(out_s["""input_ids"""].shape[-1]	,			30	)
														self.assertTrue(pad_token_id in out_s["""input_ids"""]	)
														self.assertTrue(0 in out_s["""attention_mask"""]	)

														# s2
														# test automatic padding
														self.assertEqual(out_sa["""input_ids"""].shape[-1]	,			33	)
														# long slice doesn't have padding
														self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]	)
														self.assertFalse(0 in out_sa["""attention_mask"""][0]	)
														# short slice does have padding
														self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]	)
														self.assertTrue(0 in out_sa["""attention_mask"""][1]	)

														# p
														# test single pair max_length padding
														self.assertEqual(out_p["""input_ids"""].shape[-1]	,			60	)
														self.assertTrue(pad_token_id in out_p["""input_ids"""]	)
														self.assertTrue(0 in out_p["""attention_mask"""]	)

														# p2
														# test automatic padding pair
														self.assertEqual(out_pa["""input_ids"""].shape[-1]	,			52	)
														# long slice pair doesn't have padding
														self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]	)
														self.assertFalse(0 in out_pa["""attention_mask"""][0]	)
														# short slice pair does have padding
														self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]	)
														self.assertTrue(0 in out_pa["""attention_mask"""][1]	)






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      """$$$"""
														UpperCamelCase__					  =      CodeGenTokenizer.from_pretrained(self.tmpdirname	,			bos_token=__lowerCAmelCase	,			add_bos_token=__lowerCAmelCase	)

														UpperCamelCase__					  =      """This is a simple input"""
														UpperCamelCase__					  =      ["""This is a simple input 1""", """This is a simple input 2"""]

														UpperCamelCase__					  =      tokenizer.bos_token_id

														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	)
														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	)

														self.assertEqual(out_s.input_ids[0]	,			__lowerCAmelCase	)
														self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids	)	)

														UpperCamelCase__					  =      tokenizer.decode(out_s.input_ids	)
														UpperCamelCase__					  =      tokenizer.batch_decode(out_sa.input_ids	)

														self.assertEqual(decode_s.split()[0]	,			__lowerCAmelCase	)
														self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa	)	)






							@slow
							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""	)

														UpperCamelCase__					  =      """\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
														UpperCamelCase__					  =      """\nif len_a > len_b:      result = a\nelse:      result = b"""

														UpperCamelCase__					  =      tokenizer.encode(__lowerCAmelCase	)
														UpperCamelCase__					  =      ["""^#""", re.escape("""<|endoftext|>"""	), """^'''""", """^\"\"\"""", """\n\n\n"""]
														UpperCamelCase__					  =      tokenizer.decode(__lowerCAmelCase	,			truncate_before_pattern=__lowerCAmelCase	)
														self.assertEqual(__lowerCAmelCase	,			__lowerCAmelCase	)







							def 					_lowerCamelCase		(    self	):
														pass




