






"""simple docstring"""



import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
						if is_torch_available():
												import torch
						if is_tf_available():
												import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


__A		:       Optional[Any]  				=		logging.get_logger(__name__)

__A		:       Tuple  				=		{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__A		:       List[str]  				=		{
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "Salesforce/codegen-350M-mono": (
            "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"
        ),
    },
}

__A		:       str  				=		{
    "Salesforce/codegen-350M-mono": 2_048,
}





class 					_UpperCAmelCase       ( lowercase__							):
					SCREAMING_SNAKE_CASE_		:				List[str]							    =   VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE_		:				Any							    =   PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE_		:				Tuple							    =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE_		:				List[str]							    =   ['input_ids', 'attention_mask']
					SCREAMING_SNAKE_CASE_		:				List[str]							    =   CodeGenTokenizer




					def __init__(				self     :						Dict    ,						A     :						Union[str, Any]=None    ,						A     :						str=None    ,						A     :						Dict=None    ,						A     :						Tuple="<|endoftext|>"    ,						A     :						Dict="<|endoftext|>"    ,						A     :						List[str]="<|endoftext|>"    ,						A     :						Optional[int]=False    ,						**A     :						Any    ,						)  ->  Union[str, Any]:
							super().__init__(
							    __lowercase    ,						__lowercase    ,						tokenizer_file=__lowercase    ,						unk_token=__lowercase    ,						bos_token=__lowercase    ,						eos_token=__lowercase    ,						add_prefix_space=__lowercase    ,						**__lowercase    ,						)

							if kwargs.pop('''add_bos_token'''    ,						__lowercase   ):
									lowercase_ : List[Any]							       =   kwargs.pop('''name_or_path'''    ,						''''''   )
									raise ValueError(
									    '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
									    '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
									    F'''`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'''
									    F'''`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'''
									    '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
									    ''' so that the fast tokenizer works correctly.'''   )

							lowercase_ : List[Any]							       =   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
							if pre_tok_state.get('''add_prefix_space'''    ,						__lowercase   ) != add_prefix_space:
									lowercase_ : Any							       =   getattr(__lowercase    ,						pre_tok_state.pop('''type'''   )   )
									lowercase_ : Optional[int]							       =   add_prefix_space
									lowercase_ : Tuple							       =   pre_tok_class(**__lowercase   )

							lowercase_ : str							       =   add_prefix_space




					def   A  (				self     :						str    ,						*A     :						Optional[Any]    ,						**A     :						str   )  ->  BatchEncoding:
							lowercase_ : str							       =   kwargs.get('''is_split_into_words'''    ,						__lowercase   )
							assert self.add_prefix_space or not is_split_into_words, (
							    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
							    "to use it with pretokenized inputs."
							)

							return super()._batch_encode_plus(*__lowercase    ,						**__lowercase   )




					def   A  (				self     :						Dict    ,						*A     :						Any    ,						**A     :						Any   )  ->  BatchEncoding:
							lowercase_ : List[str]							       =   kwargs.get('''is_split_into_words'''    ,						__lowercase   )

							assert self.add_prefix_space or not is_split_into_words, (
							    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
							    "to use it with pretokenized inputs."
							)

							return super()._encode_plus(*__lowercase    ,						**__lowercase   )




					def   A  (				self     :						Tuple    ,						A     :						str    ,						A     :						Optional[str] = None   )  ->  Tuple[str]:
							lowercase_ : List[str]							       =   self._tokenizer.model.save(__lowercase    ,						name=__lowercase   )
							return tuple(__lowercase   )




					def   A  (				self     :						List[Any]    ,						A     :						Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]    ,						A     :						bool = False    ,						A     :						bool = None    ,						A     :						Optional[List[str]] = None    ,						**A     :						Union[str, Any]    ,						)  ->  str:
							lowercase_ : Any							       =   super().decode(
							    token_ids=__lowercase    ,						skip_special_tokens=__lowercase    ,						clean_up_tokenization_spaces=__lowercase    ,						**__lowercase    ,						)

							if truncate_before_pattern is not None and len(__lowercase   ) > 0:
									lowercase_ : List[str]							       =   self.truncate(__lowercase    ,						__lowercase   )

							return decoded_text






					def   A  (				self     :						int    ,						A     :						List[Any]    ,						A     :						Tuple   )  ->  List[Any]:
							def find_re(A     :						Optional[int]    ,						A     :						int    ,						A     :						Optional[Any]   ):
									lowercase_ : str							       =   pattern.search(__lowercase    ,						__lowercase   )
									return m.start() if m else -1

							lowercase_ : Dict							       =   [re.compile(__lowercase    ,						re.MULTILINE   ) for pattern in truncate_before_pattern]

							lowercase_ : List[Any]							       =   list(re.finditer('''^print'''    ,						__lowercase    ,						re.MULTILINE   )   )

							if len(__lowercase   ) > 1:
									lowercase_ : Tuple							       =   completion[: prints[1].start()]

							lowercase_ : Union[str, Any]							       =   list(re.finditer('''^def'''    ,						__lowercase    ,						re.MULTILINE   )   )

							if len(__lowercase   ) > 1:
									lowercase_ : Any							       =   completion[: defs[1].start()]

							lowercase_ : Optional[int]							       =   0

							lowercase_ : Any							       =   [
							    pos for pos in [find_re(__lowercase    ,						__lowercase    ,						__lowercase   ) for terminal in terminals] if pos != -1
							]

							if len(__lowercase   ) > 0:
									return completion[: min(__lowercase   )]
							else:
									return completion











from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
       import PIL


a		:       Union[str, Any]			     = logging.get_logger(__name__)
class     a    ( lowercase__   ):



      """simple docstring"""


      a  : Any							       =	['pixel_values']
      def __init__(      self					:			Optional[int]		,      __lowercase					:			bool = True		,      __lowercase					:			Dict[str, int] = None		,      __lowercase					:			int = 0.9		,      __lowercase					:			PILImageResampling = PILImageResampling.BICUBIC		,      __lowercase					:			bool = True		,      __lowercase					:			Dict[str, int] = None		,      __lowercase					:			Union[int, float] = 1 / 255		,      __lowercase					:			bool = True		,      __lowercase					:			bool = True		,      __lowercase					:			Optional[Union[float, List[float]]] = None		,      __lowercase					:			Optional[Union[float, List[float]]] = None		,      **__lowercase					:			Any		,      )							->      None:
        super().__init__(**__lowercase					)
        __UpperCAmelCase							:	Tuple           =  size if size is not None else {"""shortest_edge""": 224}
        __UpperCAmelCase							:	Union[str, Any]           =  get_size_dict(__lowercase		,      default_to_square=__lowercase					)
        __UpperCAmelCase							:	Union[str, Any]           =  crop_size if crop_size is not None else {"""height""": 224, """width""": 224}
        __UpperCAmelCase							:	Any           =  get_size_dict(__lowercase		,      param_name="""crop_size"""					)

        __UpperCAmelCase							:	Dict           =  do_resize
        __UpperCAmelCase							:	Dict           =  size
        __UpperCAmelCase							:	Tuple           =  crop_pct
        __UpperCAmelCase							:	List[Any]           =  resample
        __UpperCAmelCase							:	List[Any]           =  do_center_crop
        __UpperCAmelCase							:	List[Any]           =  crop_size
        __UpperCAmelCase							:	Any           =  do_rescale
        __UpperCAmelCase							:	Tuple           =  rescale_factor
        __UpperCAmelCase							:	int           =  do_normalize
        __UpperCAmelCase							:	List[Any]           =  image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
        __UpperCAmelCase							:	List[str]           =  image_std if image_std is not None else IMAGENET_DEFAULT_STD
      def 	UpperCAmelCase      (      self					:			Tuple		,      __lowercase					:			np.ndarray		,      __lowercase					:			Dict[str, int]		,      __lowercase					:			Optional[float] = None		,      __lowercase					:			PILImageResampling = PILImageResampling.BICUBIC		,      __lowercase					:			Optional[Union[str, ChannelDimension]] = None		,      **__lowercase					:			Optional[int]		,      )							->      np.ndarray:
        __UpperCAmelCase							:	Tuple           =  get_size_dict(__lowercase		,      default_to_square=__lowercase					)
        if "shortest_edge" not in size and ("height" not in size or "width" not in size):
          raise ValueError(f"""size must contain 'height' and 'width' or 'shortest_edge' as keys. Got {size.keys()}"""					)
        if crop_pct is not None:
          if "shortest_edge" in size:
            __UpperCAmelCase							:	Union[str, Any]           =  int(size["""shortest_edge"""] / crop_pct					)
          elif "height" in size and "width" in size:
            if size["height"] == size["width"]:
              __UpperCAmelCase							:	Tuple           =  int(size["""height"""] / crop_pct					)
            else:
              __UpperCAmelCase							:	str           =  (int(size["""height"""] / crop_pct					), int(size["""width"""] / crop_pct					))
          else:
            raise ValueError("""Invalid size for resize: {}""".format(__lowercase					)					)

          __UpperCAmelCase							:	str           =  get_resize_output_image_size(__lowercase		,      size=__lowercase		,      default_to_square=__lowercase					)
        else:
          if "shortest_edge" in size:
            __UpperCAmelCase							:	List[str]           =  get_resize_output_image_size(__lowercase		,      size=size["""shortest_edge"""]		,      default_to_square=__lowercase					)
          elif "height" in size and "width" in size:
            __UpperCAmelCase							:	int           =  (size["""height"""], size["""width"""])
          else:
            raise ValueError("""Invalid size for resize: {}""".format(__lowercase					)					)

        return resize(__lowercase		,      size=__lowercase		,      resample=__lowercase		,      data_format=__lowercase		,      **__lowercase					)
      def 	UpperCAmelCase      (      self					:			Dict		,      __lowercase					:			np.ndarray		,      __lowercase					:			Dict[str, int]		,      __lowercase					:			Optional[Union[str, ChannelDimension]] = None		,      **__lowercase					:			Union[str, Any]		,      )							->      np.ndarray:
        __UpperCAmelCase							:	Optional[Any]           =  get_size_dict(__lowercase					)
        if "height" not in size or "width" not in size:
          raise ValueError(f"""size must contain 'height' and 'width' as keys. Got {size.keys()}"""					)
        return center_crop(__lowercase		,      size=(size["""height"""], size["""width"""])		,      data_format=__lowercase		,      **__lowercase					)
      def 	UpperCAmelCase      (      self					:			List[str]		,      __lowercase					:			np.ndarray		,      __lowercase					:			Union[int, float]		,      __lowercase					:			Optional[Union[str, ChannelDimension]] = None		,      **__lowercase					:			int		,      )							->      int:
        return rescale(__lowercase		,      scale=__lowercase		,      data_format=__lowercase		,      **__lowercase					)
      def 	UpperCAmelCase      (      self					:			List[Any]		,      __lowercase					:			np.ndarray		,      __lowercase					:			Union[float, List[float]]		,      __lowercase					:			Union[float, List[float]]		,      __lowercase					:			Optional[Union[str, ChannelDimension]] = None		,      **__lowercase					:			List[Any]		,      )							->      np.ndarray:
        return normalize(__lowercase		,      mean=__lowercase		,      std=__lowercase		,      data_format=__lowercase		,      **__lowercase					)




      def 	UpperCAmelCase      (      self					:			Any		,      __lowercase					:			ImageInput		,      __lowercase					:			bool = None		,      __lowercase					:			Dict[str, int] = None		,      __lowercase					:			int = None		,      __lowercase					:			PILImageResampling = None		,      __lowercase					:			bool = None		,      __lowercase					:			Dict[str, int] = None		,      __lowercase					:			bool = None		,      __lowercase					:			float = None		,      __lowercase					:			bool = None		,      __lowercase					:			Optional[Union[float, List[float]]] = None		,      __lowercase					:			Optional[Union[float, List[float]]] = None		,      __lowercase					:			Optional[Union[str, TensorType]] = None		,      __lowercase					:			ChannelDimension = ChannelDimension.FIRST		,      **__lowercase					:			List[str]		,      )							->      PIL.Image.Image:
        __UpperCAmelCase							:	Any           =  do_resize if do_resize is not None else self.do_resize
        __UpperCAmelCase							:	Optional[int]           =  crop_pct if crop_pct is not None else self.crop_pct
        __UpperCAmelCase							:	Optional[Any]           =  resample if resample is not None else self.resample
        __UpperCAmelCase							:	List[Any]           =  do_center_crop if do_center_crop is not None else self.do_center_crop
        __UpperCAmelCase							:	Dict           =  do_rescale if do_rescale is not None else self.do_rescale
        __UpperCAmelCase							:	Tuple           =  rescale_factor if rescale_factor is not None else self.rescale_factor
        __UpperCAmelCase							:	Union[str, Any]           =  do_normalize if do_normalize is not None else self.do_normalize
        __UpperCAmelCase							:	Tuple           =  image_mean if image_mean is not None else self.image_mean
        __UpperCAmelCase							:	Any           =  image_std if image_std is not None else self.image_std

        __UpperCAmelCase							:	Optional[int]           =  size if size is not None else self.size
        __UpperCAmelCase							:	Dict           =  get_size_dict(__lowercase		,      default_to_square=__lowercase					)
        __UpperCAmelCase							:	Tuple           =  crop_size if crop_size is not None else self.crop_size
        __UpperCAmelCase							:	Tuple           =  get_size_dict(__lowercase		,      param_name="""crop_size"""					)

        __UpperCAmelCase							:	Dict           =  make_list_of_images(__lowercase					)

        if not valid_images(__lowercase					):
          raise ValueError(
              """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
              """torch.Tensor, tf.Tensor or jax.ndarray."""					)

        if do_resize and size is None or resample is None:
          raise ValueError("""Size and resample must be specified if do_resize is True."""					)

        if do_center_crop and crop_pct is None:
          raise ValueError("""Crop_pct must be specified if do_center_crop is True."""					)

        if do_rescale and rescale_factor is None:
          raise ValueError("""Rescale factor must be specified if do_rescale is True."""					)

        if do_normalize and (image_mean is None or image_std is None):
          raise ValueError("""Image mean and std must be specified if do_normalize is True."""					)

        # All transformations expect numpy arrays.
        __UpperCAmelCase							:	str           =  [to_numpy_array(__lowercase					) for image in images]

        if do_resize:
          __UpperCAmelCase							:	str           =  [self.resize(image=__lowercase		,      size=__lowercase		,      crop_pct=__lowercase		,      resample=__lowercase					) for image in images]

        if do_center_crop:
          __UpperCAmelCase							:	Any           =  [self.center_crop(image=__lowercase		,      size=__lowercase					) for image in images]

        if do_rescale:
          __UpperCAmelCase							:	List[str]           =  [self.rescale(image=__lowercase		,      scale=__lowercase					) for image in images]

        if do_normalize:
          __UpperCAmelCase							:	str           =  [self.normalize(image=__lowercase		,      mean=__lowercase		,      std=__lowercase					) for image in images]

        __UpperCAmelCase							:	List[str]           =  [to_channel_dimension_format(__lowercase		,      __lowercase					) for image in images]

        __UpperCAmelCase							:	Any           =  {"""pixel_values""": images}
        return BatchFeature(data=__lowercase		,      tensor_type=__lowercase					)


