




"""simple docstring"""




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase							:			Tuple     							=							logging.get_logger(__name__)

lowerCAmelCase							:			Optional[int]     							=							"""▁"""

lowerCAmelCase							:			int     							=							{"""vocab_file""": """sentencepiece.bpe.model"""}

lowerCAmelCase							:			int     							=							{
    """vocab_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model""",
    }
}

lowerCAmelCase							:			Tuple     							=							{
    """facebook/xglm-564M""": 2048,
}






class     __magic_name__							(       UpperCAmelCase__				):






				'''simple docstring'''
				__UpperCamelCase			  =						VOCAB_FILES_NAMES
				__UpperCamelCase			  =						PRETRAINED_VOCAB_FILES_MAP
				__UpperCamelCase			  =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__UpperCamelCase			  =						["input_ids", "attention_mask"]





				def __init__(       self		, _a		, _a="<s>"		, _a="</s>"		, _a="</s>"		, _a="<s>"		, _a="<unk>"		, _a="<pad>"		, _a = None		, **_a		, ):





							"""simple docstring"""
							lowerCamelCase			      =					{} if sp_model_kwargs is None else sp_model_kwargs

							# Compatibility with the original tokenizer
							lowerCamelCase			      =					7
							lowerCamelCase			      =					[f'<madeupword{i}>' for i in range(self.num_madeup_words )]

							lowerCamelCase			      =					kwargs.get("""additional_special_tokens"""		, [] )
							kwargs["additional_special_tokens"] += [
							    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    bos_token=_a		, eos_token=_a		, unk_token=_a		, sep_token=_a		, cls_token=_a		, pad_token=_a		, sp_model_kwargs=self.sp_model_kwargs		, **_a		, )

							lowerCamelCase			      =					spm.SentencePieceProcessor(**self.sp_model_kwargs )
							self.sp_model.Load(str(_a ) )
							lowerCamelCase			      =					vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							lowerCamelCase			      =					1

							# Mimic fairseq token-to-id alignment for the first 4 token
							lowerCamelCase			      =					{"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

							lowerCamelCase			      =					len(self.sp_model )
							lowerCamelCase			      =					{f'<madeupword{i}>': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words )}
							self.fairseq_tokens_to_ids.update(_a )

							lowerCamelCase			      =					{v: k for k, v in self.fairseq_tokens_to_ids.items()}





				def __getstate__(       self ):





							"""simple docstring"""
							lowerCamelCase			      =					self.__dict__.copy()
							lowerCamelCase			      =					None
							lowerCamelCase			      =					self.sp_model.serialized_model_proto()
							return state





				def __setstate__(       self		, _a ):





							"""simple docstring"""
							lowerCamelCase			      =					d

							# for backward compatibility
							if not hasattr(self		, """sp_model_kwargs""" ):
										lowerCamelCase			      =					{}

							lowerCamelCase			      =					spm.SentencePieceProcessor(**self.sp_model_kwargs )
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto )





				def    _lowerCAmelCase				(       self		, _a		, _a = None ):





							"""simple docstring"""

							if token_ids_a is None:
										return [self.sep_token_id] + token_ids_a
							lowerCamelCase			      =					[self.sep_token_id]
							return sep + token_ids_a + sep + sep + token_ids_a





				def    _lowerCAmelCase				(       self		, _a		, _a = None		, _a = False ):





							"""simple docstring"""

							if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=_a		, token_ids_a=_a		, already_has_special_tokens=_a )

							if token_ids_a is None:
										return [1] + ([0] * len(_a ))
							return [1] + ([0] * len(_a )) + [1, 1] + ([0] * len(_a ))





				def    _lowerCAmelCase				(       self		, _a		, _a = None ):





							"""simple docstring"""
							lowerCamelCase			      =					[self.sep_token_id]

							if token_ids_a is None:
										return len(sep + token_ids_a ) * [0]
							return len(sep + token_ids_a + sep + sep + token_ids_a ) * [0]





				@property
				def    _lowerCAmelCase				(       self ):





							"""simple docstring"""
							return len(self.sp_model ) + self.fairseq_offset + self.num_madeup_words





				def    _lowerCAmelCase				(       self ):





							"""simple docstring"""
							lowerCamelCase			      =					{self.convert_ids_to_tokens(_a ): i for i in range(self.vocab_size )}
							vocab.update(self.added_tokens_encoder )
							return vocab





				def    _lowerCAmelCase				(       self		, _a ):





							"""simple docstring"""
							return self.sp_model.encode(_a		, out_type=_a )





				def    _lowerCAmelCase				(       self		, _a ):





							"""simple docstring"""
							if token in self.fairseq_tokens_to_ids:
										return self.fairseq_tokens_to_ids[token]
							lowerCamelCase			      =					self.sp_model.PieceToId(_a )

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id





				def    _lowerCAmelCase				(       self		, _a ):





							"""simple docstring"""
							if index in self.fairseq_ids_to_tokens:
										return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset )





				def    _lowerCAmelCase				(       self		, _a ):





							"""simple docstring"""
							lowerCamelCase			      =					"""""".join(_a ).replace(_a		, """ """ ).strip()
							return out_string





				def    _lowerCAmelCase				(       self		, _a		, _a = None ):





							"""simple docstring"""
							if not os.path.isdir(_a ):
										logger.error(f'Vocabulary path ({save_directory}) should be a directory' )
										return
							lowerCamelCase			      =					os.path.join(
							    _a		, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

							if os.path.abspath(self.vocab_file ) != os.path.abspath(_a ) and os.path.isfile(self.vocab_file ):
										copyfile(self.vocab_file		, _a )
							elif not os.path.isfile(self.vocab_file ):
										with open(_a		, """wb""" ) as fi:
													lowerCamelCase			      =					self.sp_model.serialized_model_proto()
													fi.write(_a )

							return (out_vocab_file,)









"""simple docstring"""




from typing import Dict, List, Optional, Tuple, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_torch_available, is_torch_tensor, logging


if is_torch_available():
					import torch


lowerCAmelCase							:			int     							=							logging.get_logger(__name__)






class     __magic_name__							(       UpperCAmelCase__				):






				'''simple docstring'''
				__UpperCamelCase			  =						["pixel_values"]





				def __init__(       self		, _a = True		, _a = None		, _a = PILImageResampling.BILINEAR		, _a = True		, _a = None		, _a = True		, _a = 1 / 255		, _a = True		, _a = None		, _a = None		, **_a		, ):





							"""simple docstring"""
							super().__init__(**_a )
							lowerCamelCase			      =					size if size is not None else {"""shortest_edge""": 256}
							lowerCamelCase			      =					get_size_dict(_a		, default_to_square=_a )
							lowerCamelCase			      =					crop_size if crop_size is not None else {"""height""": 224, """width""": 224}
							lowerCamelCase			      =					get_size_dict(_a		, param_name="""crop_size""" )
							lowerCamelCase			      =					do_resize
							lowerCamelCase			      =					size
							lowerCamelCase			      =					resample
							lowerCamelCase			      =					do_center_crop
							lowerCamelCase			      =					crop_size
							lowerCamelCase			      =					do_rescale
							lowerCamelCase			      =					rescale_factor
							lowerCamelCase			      =					do_normalize
							lowerCamelCase			      =					image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
							lowerCamelCase			      =					image_std if image_std is not None else IMAGENET_STANDARD_STD





				def    _lowerCAmelCase				(       self		, _a		, _a		, _a = PILImageResampling.BICUBIC		, _a = None		, **_a		, ):





							"""simple docstring"""
							lowerCamelCase			      =					get_size_dict(_a		, default_to_square=_a )
							if "shortest_edge" not in size:
										raise ValueError(f'The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}' )
							lowerCamelCase			      =					get_resize_output_image_size(_a		, size=size["""shortest_edge"""]		, default_to_square=_a )
							return resize(_a		, size=_a		, resample=_a		, data_format=_a		, **_a )





				def    _lowerCAmelCase				(       self		, _a		, _a		, _a = None		, **_a		, ):





							"""simple docstring"""
							lowerCamelCase			      =					get_size_dict(_a )
							if "height" not in size or "width" not in size:
										raise ValueError(f'The `size` parameter must contain the keys `height` and `width`. Got {size.keys()}' )
							return center_crop(_a		, size=(size["""height"""], size["""width"""])		, data_format=_a		, **_a )





				def    _lowerCAmelCase				(       self		, _a		, _a		, _a = None		, **_a ):





							"""simple docstring"""
							return rescale(_a		, scale=_a		, data_format=_a		, **_a )





				def    _lowerCAmelCase				(       self		, _a		, _a		, _a		, _a = None		, **_a		, ):





							"""simple docstring"""
							return normalize(_a		, mean=_a		, std=_a		, data_format=_a		, **_a )





				def    _lowerCAmelCase				(       self		, _a		, _a = None		, _a = None		, _a = None		, _a = None		, _a = None		, _a = None		, _a = None		, _a = None		, _a = None		, _a = None		, _a = None		, _a = ChannelDimension.FIRST		, **_a		, ):





							"""simple docstring"""
							lowerCamelCase			      =					do_resize if do_resize is not None else self.do_resize
							lowerCamelCase			      =					size if size is not None else self.size
							lowerCamelCase			      =					get_size_dict(_a		, default_to_square=_a )
							lowerCamelCase			      =					resample if resample is not None else self.resample
							lowerCamelCase			      =					do_center_crop if do_center_crop is not None else self.do_center_crop
							lowerCamelCase			      =					crop_size if crop_size is not None else self.crop_size
							lowerCamelCase			      =					get_size_dict(_a		, param_name="""crop_size""" )
							lowerCamelCase			      =					do_rescale if do_rescale is not None else self.do_rescale
							lowerCamelCase			      =					rescale_factor if rescale_factor is not None else self.rescale_factor
							lowerCamelCase			      =					do_normalize if do_normalize is not None else self.do_normalize
							lowerCamelCase			      =					image_mean if image_mean is not None else self.image_mean
							lowerCamelCase			      =					image_std if image_std is not None else self.image_std

							lowerCamelCase			      =					make_list_of_images(_a )

							if not valid_images(_a ):
										raise ValueError(
										    """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
										    """torch.Tensor, tf.Tensor or jax.ndarray.""" )

							if do_resize and size is None:
										raise ValueError("""Size must be specified if do_resize is True.""" )

							if do_center_crop and crop_size is None:
										raise ValueError("""Crop size must be specified if do_center_crop is True.""" )

							if do_rescale and rescale_factor is None:
										raise ValueError("""Rescale factor must be specified if do_rescale is True.""" )

							if do_normalize and (image_mean is None or image_std is None):
										raise ValueError("""Image mean and std must be specified if do_normalize is True.""" )

							# All transformations expect numpy arrays.
							lowerCamelCase			      =					[to_numpy_array(_a ) for image in images]

							if do_resize:
										lowerCamelCase			      =					[self.resize(image=_a		, size=_a		, resample=_a ) for image in images]

							if do_center_crop:
										lowerCamelCase			      =					[self.center_crop(image=_a		, size=_a ) for image in images]

							if do_rescale:
										lowerCamelCase			      =					[self.rescale(image=_a		, scale=_a ) for image in images]

							if do_normalize:
										lowerCamelCase			      =					[self.normalize(image=_a		, mean=_a		, std=_a ) for image in images]

							lowerCamelCase			      =					[to_channel_dimension_format(_a		, _a ) for image in images]

							lowerCamelCase			      =					{"""pixel_values""": images}
							return BatchFeature(data=_a		, tensor_type=_a )





				def    _lowerCAmelCase				(       self		, _a		, _a = None ):





							"""simple docstring"""
							lowerCamelCase			      =					outputs.logits

							# Resize logits and compute semantic segmentation maps
							if target_sizes is not None:
										if len(_a ) != len(_a ):
													raise ValueError(
													    """Make sure that you pass in as many target sizes as the batch dimension of the logits""" )

										if is_torch_tensor(_a ):
													lowerCamelCase			      =					target_sizes.numpy()

										lowerCamelCase			      =					[]

										for idx in range(len(_a ) ):
													lowerCamelCase			      =					torch.nn.functional.interpolate(
													    logits[idx].unsqueeze(dim=0 )		, size=target_sizes[idx]		, mode="""bilinear"""		, align_corners=_a )
													lowerCamelCase			      =					resized_logits[0].argmax(dim=0 )
													semantic_segmentation.append(_a )
							else:
										lowerCamelCase			      =					logits.argmax(dim=1 )
										lowerCamelCase			      =					[semantic_segmentation[i] for i in range(semantic_segmentation.shape[0] )]

							return semantic_segmentation



