





import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
			from sagemaker import Session, TrainingJobAnalytics
			from sagemaker.huggingface import HuggingFace
@pytest.mark.skipif(
    literal_eval(os.getenv('TEST_SAGEMAKER'		,							'False'      )      ) is not True		,							reason='Skipping test because should only be run when releasing minor transformers version'		,							)
@pytest.mark.usefixtures('sm_env'      )
@parameterized_class(
    [
        {
            'framework': 'pytorch',
            'script': 'run_glue_model_parallelism.py',
            'model_name_or_path': 'roberta-large',
            'instance_type': 'ml.p3dn.24xlarge',
            'results': {'train_runtime': 1600, 'eval_accuracy': 0.3, 'eval_loss': 1.2},
        },
        {
            'framework': 'pytorch',
            'script': 'run_glue.py',
            'model_name_or_path': 'roberta-large',
            'instance_type': 'ml.p3dn.24xlarge',
            'results': {'train_runtime': 1600, 'eval_accuracy': 0.3, 'eval_loss': 1.2},
        },
    ]      )
class   a_ (			unittest.TestCase      ):

			def      lowerCAmelCase(   self   :  str						):






							"""simple docstring"""
							if self.framework == "pytorch":
											subprocess.run(
											    F"cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py".split()     , encoding='''utf-8'''     , check=__UpperCamelCase     , )
							assert hasattr(self     , '''env'''						)

			def      lowerCAmelCase(   self   :  Dict     , UpperCAmelCase__   :  Union[str, Any]						):






							"""simple docstring"""
							# configuration for running training on smdistributed Model Parallel
							snake_case   :					List[Any]    						=				{
							    """enabled""": True,
							    """processes_per_host""": 8,
							}
							snake_case   :					Optional[int]    						=				{
							    """enabled""": True,
							    """parameters""": {
							        """microbatches""": 4,
							        """placement_strategy""": """spread""",
							        """pipeline""": """interleaved""",
							        """optimize""": """speed""",
							        """partitions""": 4,
							        """ddp""": True,
							    },
							}

							snake_case   :					Tuple    						=				{"""smdistributed""": {"""modelparallel""": smp_options}, """mpi""": mpi_options}

							snake_case   :					Optional[Any]    						=				"""trainer""" if self.script == """run_glue.py""" else """smtrainer"""
							# creates estimator
							return HuggingFace(
							    entry_point=self.script     , source_dir=self.env.test_path     , role=self.env.role     , image_uri=self.env.image_uri     , base_job_name=F"{self.env.base_job_name}-{instance_count}-smp-{name_extension}"     , instance_count=__UpperCamelCase     , instance_type=self.instance_type     , debugger_hook_config=__UpperCamelCase     , hyperparameters={
							        **self.env.hyperparameters,
							        '''model_name_or_path''': self.model_name_or_path,
							        '''max_steps''': 500,
							    }     , metric_definitions=self.env.metric_definitions     , distribution=__UpperCamelCase     , py_version='''py36'''     , )

			def      lowerCAmelCase(   self   :  Optional[Any]     , UpperCAmelCase__   :  Union[str, Any]						):






							"""simple docstring"""
							TrainingJobAnalytics(__UpperCamelCase						).export_csv(F"{self.env.test_path}/{job_name}_metrics.csv"						)



			@parameterized.expand([(1,)]						)
			def      lowerCAmelCase(   self   :  Optional[int]     , UpperCAmelCase__   :  Any						):






							"""simple docstring"""
							# create estimator
							snake_case   :					Tuple    						=				self.create_estimator(__UpperCamelCase						)

							# run training
							estimator.fit()

							# result dataframe
							snake_case   :					Optional[Any]    						=				TrainingJobAnalytics(estimator.latest_training_job.name						).dataframe()

							# extract kpis
							snake_case   :					List[Any]    						=				list(result_metrics_df[result_metrics_df.metric_name == '''eval_accuracy''']['''value''']						)
							snake_case   :					Dict    						=				list(result_metrics_df[result_metrics_df.metric_name == '''eval_loss''']['''value''']						)
							# get train time from SageMaker job, this includes starting, preprocessing, stopping
							snake_case   :					List[str]    						=				(
							    Session().describe_training_job(estimator.latest_training_job.name						).get('''TrainingTimeInSeconds'''     , 999_999						)
							)

							# assert kpis
							assert train_runtime <= self.results["train_runtime"]
							assert all(t >= self.results['''eval_accuracy'''] for t in eval_accuracy						)
							assert all(t <= self.results['''eval_loss'''] for t in eval_loss						)

							# dump tests result into json file to share in PR
							with open(F"{estimator.latest_training_job.name}.json"     , '''w'''						) as outfile:
											json.dump({'''train_time''': train_runtime, '''eval_accuracy''': eval_accuracy, '''eval_loss''': eval_loss}     , __UpperCamelCase						)

from copy import deepcopy
from typing import Optional, Union

import numpy as np

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, is_tf_available, is_torch_available


if is_torch_available():
	import torch

if is_tf_available():
	import tensorflow as tf

class        UpperCamelCase						(				__a				):
						a__		:str      =			['''image_processor''']
						a__		:str      =			'''SamImageProcessor'''





						def __init__(self			,     __UpperCamelCase						)				->				int:
												super().__init__(__UpperCamelCase						)
												UpperCamelCase_		:							Any  =		self.image_processor
												UpperCamelCase_		:							int  =		-10
												UpperCamelCase_		:							Dict  =		self.image_processor.size["""longest_edge"""]





						def __call__(self			,     __UpperCamelCase=None			,     __UpperCamelCase=None			,     __UpperCamelCase=None			,     __UpperCamelCase=None			,     __UpperCamelCase = None			,     **__UpperCamelCase			,     )				->				BatchEncoding:
												UpperCamelCase_		:							int  =		self.image_processor(
												    __UpperCamelCase			,     return_tensors=__UpperCamelCase			,     **__UpperCamelCase			,     )

												# pop arguments that are not used in the foward but used nevertheless
												UpperCamelCase_		:							Optional[int]  =		encoding_image_processor["""original_sizes"""]

												if hasattr(__UpperCamelCase			,     """numpy"""						):  # Checks if Torch or TF tensor
																		UpperCamelCase_		:							Dict  =		original_sizes.numpy()

												UpperCamelCase_,UpperCamelCase_,UpperCamelCase_		:							str  =		self._check_and_preprocess_points(
												    input_points=__UpperCamelCase			,     input_labels=__UpperCamelCase			,     input_boxes=__UpperCamelCase			,     )

												UpperCamelCase_		:							List[str]  =		self._normalize_and_convert(
												    __UpperCamelCase			,     __UpperCamelCase			,     input_points=__UpperCamelCase			,     input_labels=__UpperCamelCase			,     input_boxes=__UpperCamelCase			,     return_tensors=__UpperCamelCase			,     )

												return encoding_image_processor





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase			,     __UpperCamelCase=None			,     __UpperCamelCase=None			,     __UpperCamelCase=None			,     __UpperCamelCase="pt"			,     )				->				Tuple:
												if input_points is not None:
																		if len(__UpperCamelCase						) != len(__UpperCamelCase						):
																								UpperCamelCase_		:							str  =		[
																								    self._normalize_coordinates(self.target_size			,     __UpperCamelCase			,     original_sizes[0]						) for point in input_points
																								]
																		else:
																								UpperCamelCase_		:							List[str]  =		[
																								    self._normalize_coordinates(self.target_size			,     __UpperCamelCase			,     __UpperCamelCase						)
																								    for point, original_size in zip(__UpperCamelCase			,     __UpperCamelCase						)
																								]
																		# check that all arrays have the same shape
																		if not all(point.shape == input_points[0].shape for point in input_points						):
																								if input_labels is not None:
																														UpperCamelCase_,UpperCamelCase_		:							Optional[Any]  =		self._pad_points_and_labels(__UpperCamelCase			,     __UpperCamelCase						)

																		UpperCamelCase_		:							Dict  =		np.array(__UpperCamelCase						)

												if input_labels is not None:
																		UpperCamelCase_		:							Optional[Any]  =		np.array(__UpperCamelCase						)

												if input_boxes is not None:
																		if len(__UpperCamelCase						) != len(__UpperCamelCase						):
																								UpperCamelCase_		:							Tuple  =		[
																								    self._normalize_coordinates(self.target_size			,     __UpperCamelCase			,     original_sizes[0]			,     is_bounding_box=__UpperCamelCase						)
																								    for box in input_boxes
																								]
																		else:
																								UpperCamelCase_		:							Any  =		[
																								    self._normalize_coordinates(self.target_size			,     __UpperCamelCase			,     __UpperCamelCase			,     is_bounding_box=__UpperCamelCase						)
																								    for box, original_size in zip(__UpperCamelCase			,     __UpperCamelCase						)
																								]
																		UpperCamelCase_		:							int  =		np.array(__UpperCamelCase						)

												if input_boxes is not None:
																		if return_tensors == "pt":
																								UpperCamelCase_		:							Union[str, Any]  =		torch.from_numpy(__UpperCamelCase						)
																								# boxes batch size of 1 by default
																								UpperCamelCase_		:							Any  =		input_boxes.unsqueeze(1						) if len(input_boxes.shape						) != 3 else input_boxes
																		elif return_tensors == "tf":
																								UpperCamelCase_		:							List[str]  =		tf.convert_to_tensor(__UpperCamelCase						)
																								# boxes batch size of 1 by default
																								UpperCamelCase_		:							Optional[int]  =		tf.expand_dims(__UpperCamelCase			,     1						) if len(input_boxes.shape						) != 3 else input_boxes
																		encoding_image_processor.update({"""input_boxes""": input_boxes}						)
												if input_points is not None:
																		if return_tensors == "pt":
																								UpperCamelCase_		:							Optional[Any]  =		torch.from_numpy(__UpperCamelCase						)
																								# point batch size of 1 by default
																								UpperCamelCase_		:							List[Any]  =		input_points.unsqueeze(1						) if len(input_points.shape						) != 4 else input_points
																		elif return_tensors == "tf":
																								UpperCamelCase_		:							Optional[Any]  =		tf.convert_to_tensor(__UpperCamelCase						)
																								# point batch size of 1 by default
																								UpperCamelCase_		:							Dict  =		tf.expand_dims(__UpperCamelCase			,     1						) if len(input_points.shape						) != 4 else input_points
																		encoding_image_processor.update({"""input_points""": input_points}						)
												if input_labels is not None:
																		if return_tensors == "pt":
																								UpperCamelCase_		:							Optional[int]  =		torch.from_numpy(__UpperCamelCase						)
																								# point batch size of 1 by default
																								UpperCamelCase_		:							Tuple  =		input_labels.unsqueeze(1						) if len(input_labels.shape						) != 3 else input_labels
																		elif return_tensors == "tf":
																								UpperCamelCase_		:							Optional[Any]  =		tf.convert_to_tensor(__UpperCamelCase						)
																								# point batch size of 1 by default
																								UpperCamelCase_		:							Dict  =		tf.expand_dims(__UpperCamelCase			,     1						) if len(input_labels.shape						) != 3 else input_labels
																		encoding_image_processor.update({"""input_labels""": input_labels}						)

												return encoding_image_processor





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase						)				->				Tuple:
												UpperCamelCase_		:							Union[str, Any]  =		max([point.shape[0] for point in input_points]						)
												UpperCamelCase_		:							Any  =		[]
												for i, point in enumerate(__UpperCamelCase						):
																		if point.shape[0] != expected_nb_points:
																								UpperCamelCase_		:							Optional[int]  =		np.concatenate(
																								    [point, np.zeros((expected_nb_points - point.shape[0], 2)						) + self.point_pad_value]			,     axis=0						)
																								UpperCamelCase_		:							Dict  =		np.append(input_labels[i]			,     [self.point_pad_value]						)
																		processed_input_points.append(__UpperCamelCase						)
												UpperCamelCase_		:							List[Any]  =		processed_input_points
												return input_points, input_labels





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase			,     __UpperCamelCase			,     __UpperCamelCase=False						)				->				np.ndarray:
												UpperCamelCase_,UpperCamelCase_		:							Optional[int]  =		original_size
												UpperCamelCase_,UpperCamelCase_		:							List[str]  =		self.image_processor._get_preprocess_shape(__UpperCamelCase			,     longest_edge=__UpperCamelCase						)
												UpperCamelCase_		:							str  =		deepcopy(__UpperCamelCase						).astype(__UpperCamelCase						)

												if is_bounding_box:
																		UpperCamelCase_		:							int  =		coords.reshape(-1			,     2			,     2						)

												UpperCamelCase_		:							Tuple  =		coords[..., 0] * (new_w / old_w)
												UpperCamelCase_		:							Optional[int]  =		coords[..., 1] * (new_h / old_h)

												if is_bounding_box:
																		UpperCamelCase_		:							Optional[Any]  =		coords.reshape(-1			,     4						)

												return coords





						def        A_							(self			,     __UpperCamelCase=None			,     __UpperCamelCase=None			,     __UpperCamelCase=None			,     )				->				List[str]:
												if input_points is not None:
																		if hasattr(__UpperCamelCase			,     """numpy"""						):  # Checks for TF or Torch tensor
																								UpperCamelCase_		:							int  =		input_points.numpy().tolist()

																		if not isinstance(__UpperCamelCase			,     __UpperCamelCase						) or not isinstance(input_points[0]			,     __UpperCamelCase						):
																								raise ValueError("""Input points must be a list of list of floating points."""						)
																		UpperCamelCase_		:							List[Any]  =		[np.array(__UpperCamelCase						) for input_point in input_points]
												else:
																		UpperCamelCase_		:							str  =		None

												if input_labels is not None:
																		if hasattr(__UpperCamelCase			,     """numpy"""						):
																								UpperCamelCase_		:							Any  =		input_labels.numpy().tolist()

																		if not isinstance(__UpperCamelCase			,     __UpperCamelCase						) or not isinstance(input_labels[0]			,     __UpperCamelCase						):
																								raise ValueError("""Input labels must be a list of list integers."""						)
																		UpperCamelCase_		:							List[str]  =		[np.array(__UpperCamelCase						) for label in input_labels]
												else:
																		UpperCamelCase_		:							List[str]  =		None

												if input_boxes is not None:
																		if hasattr(__UpperCamelCase			,     """numpy"""						):
																								UpperCamelCase_		:							Dict  =		input_boxes.numpy().tolist()

																		if (
																		    not isinstance(__UpperCamelCase			,     __UpperCamelCase						)
																		    or not isinstance(input_boxes[0]			,     __UpperCamelCase						)
																		    or not isinstance(input_boxes[0][0]			,     __UpperCamelCase						)
																		):
																								raise ValueError("""Input boxes must be a list of list of list of floating points."""						)
																		UpperCamelCase_		:							Tuple  =		[np.array(__UpperCamelCase						).astype(np.floataa						) for box in input_boxes]
												else:
																		UpperCamelCase_		:							List[str]  =		None

												return input_points, input_labels, input_boxes





						@property
						def        A_							(self						)				->				List[str]:
												UpperCamelCase_		:							str  =		self.image_processor.model_input_names
												return list(dict.fromkeys(__UpperCamelCase						)						)





						def        A_							(self			,     *__UpperCamelCase			,     **__UpperCamelCase						)				->				Optional[Any]:
												return self.image_processor.post_process_masks(*__UpperCamelCase			,     **__UpperCamelCase						)






