


from datasets.utils.patching import _PatchedModuleObj, patch_submodule

from . import _test_patching
def 				__lowerCamelCase						( ):




      """simple docstring"""





      import os as original_os
      from os import path as original_path
      from os import rename as original_rename
      from os.path import dirname as original_dirname
      from os.path import join as original_join

      assert _test_patching.os is original_os
      assert _test_patching.path is original_path
      assert _test_patching.join is original_join

      assert _test_patching.renamed_os is original_os
      assert _test_patching.renamed_path is original_path
      assert _test_patching.renamed_join is original_join

      a						:Optional[int] 							=       '''__test_patch_submodule_mock__'''
      with patch_submodule(_test_patching ,  '''os.path.join''' ,  UpperCAmelCase_						):
            # Every way to access os.path.join must be patched, and the rest must stay untouched

            # check os.path.join
            assert isinstance(_test_patching.os ,  _PatchedModuleObj						)
            assert isinstance(_test_patching.os.path ,  _PatchedModuleObj						)
            assert _test_patching.os.path.join is mock

            # check path.join
            assert isinstance(_test_patching.path ,  _PatchedModuleObj						)
            assert _test_patching.path.join is mock

            # check join
            assert _test_patching.join is mock

            # check that the other attributes are untouched
            assert _test_patching.os.rename is original_rename
            assert _test_patching.path.dirname is original_dirname
            assert _test_patching.os.path.dirname is original_dirname

            # Even renamed modules or objects must be patched

            # check renamed_os.path.join
            assert isinstance(_test_patching.renamed_os ,  _PatchedModuleObj						)
            assert isinstance(_test_patching.renamed_os.path ,  _PatchedModuleObj						)
            assert _test_patching.renamed_os.path.join is mock

            # check renamed_path.join
            assert isinstance(_test_patching.renamed_path ,  _PatchedModuleObj						)
            assert _test_patching.renamed_path.join is mock

            # check renamed_join
            assert _test_patching.renamed_join is mock

            # check that the other attributes are untouched
            assert _test_patching.renamed_os.rename is original_rename
            assert _test_patching.renamed_path.dirname is original_dirname
            assert _test_patching.renamed_os.path.dirname is original_dirname

      # check that everthing is back to normal when the patch is over

      assert _test_patching.os is original_os
      assert _test_patching.path is original_path
      assert _test_patching.join is original_join

      assert _test_patching.renamed_os is original_os
      assert _test_patching.renamed_path is original_path
      assert _test_patching.renamed_join is original_join
def 				__lowerCamelCase						( ):




      """simple docstring"""





      assert _test_patching.open is open

      a						:Dict 							=       '''__test_patch_submodule_builtin_mock__'''
      # _test_patching has "open" in its globals
      assert _test_patching.open is open
      with patch_submodule(_test_patching ,  '''open''' ,  UpperCAmelCase_						):
            assert _test_patching.open is mock

      # check that everthing is back to normal when the patch is over

      assert _test_patching.open is open
def 				__lowerCamelCase						( ):




      """simple docstring"""





      a						:Any 							=       '''__test_patch_submodule_missing_mock__'''
      with patch_submodule(_test_patching ,  '''pandas.read_csv''' ,  UpperCAmelCase_						):
            pass
def 				__lowerCamelCase						( ):




      """simple docstring"""





      a						:Any 							=       '''__test_patch_submodule_missing_builtin_mock__'''
      # _test_patching doesn't have "len" in its globals
      assert getattr(_test_patching ,  '''len''' ,  UpperCAmelCase_						) is None
      with patch_submodule(_test_patching ,  '''len''' ,  UpperCAmelCase_						):
            assert _test_patching.len is mock
      assert _test_patching.len is len
def 				__lowerCamelCase						( ):




      """simple docstring"""





      a						:Tuple 							=       '''__test_patch_submodule_start_and_stop_mock__'''
      a						:Any 							=       patch_submodule(_test_patching ,  '''open''' ,  UpperCAmelCase_						)
      assert _test_patching.open is open
      patch.start()
      assert _test_patching.open is mock
      patch.stop()
      assert _test_patching.open is open
def 				__lowerCamelCase						( ):




      """simple docstring"""





      from os import rename as original_rename
      from os.path import dirname as original_dirname
      from os.path import join as original_join

      a						:Dict 							=       '''__test_patch_submodule_successive_join__'''
      a						:List[str] 							=       '''__test_patch_submodule_successive_dirname__'''
      a						:Any 							=       '''__test_patch_submodule_successive_rename__'''
      assert _test_patching.os.path.join is original_join
      assert _test_patching.os.path.dirname is original_dirname
      assert _test_patching.os.rename is original_rename

      with patch_submodule(_test_patching ,  '''os.path.join''' ,  UpperCAmelCase_						):
            with patch_submodule(_test_patching ,  '''os.rename''' ,  UpperCAmelCase_						):
                  with patch_submodule(_test_patching ,  '''os.path.dirname''' ,  UpperCAmelCase_						):
                        assert _test_patching.os.path.join is mock_join
                        assert _test_patching.os.path.dirname is mock_dirname
                        assert _test_patching.os.rename is mock_rename

    # try another order
      with patch_submodule(_test_patching ,  '''os.rename''' ,  UpperCAmelCase_						):
            with patch_submodule(_test_patching ,  '''os.path.join''' ,  UpperCAmelCase_						):
                  with patch_submodule(_test_patching ,  '''os.path.dirname''' ,  UpperCAmelCase_						):
                        assert _test_patching.os.path.join is mock_join
                        assert _test_patching.os.path.dirname is mock_dirname
                        assert _test_patching.os.rename is mock_rename

      assert _test_patching.os.path.join is original_join
      assert _test_patching.os.path.dirname is original_dirname
      assert _test_patching.os.rename is original_rename






def 				__lowerCamelCase						( ):




      """simple docstring"""





      a						:Tuple 							=       '''__test_patch_submodule_doesnt_exist_mock__'''
      with patch_submodule(_test_patching ,  '''__module_that_doesn_exist__.__attribute_that_doesn_exist__''' ,  UpperCAmelCase_						):
            pass
      with patch_submodule(_test_patching ,  '''os.__attribute_that_doesn_exist__''' ,  UpperCAmelCase_						):
            pass










import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
						from sagemaker import Session, TrainingJobAnalytics
						from sagemaker.huggingface import HuggingFace



@pytest.mark.skipif(
    literal_eval(os.getenv('''TEST_SAGEMAKER'''						,'''False'''						)						) is not True						,reason='''Skipping test because should only be run when releasing minor transformers version'''						,)
@pytest.mark.usefixtures('''sm_env'''						)
@parameterized_class(
    [
        {
            '''framework''': '''pytorch''',
            '''script''': '''run_glue.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.g4dn.xlarge''',
            '''results''': {'''train_runtime''': 650, '''eval_accuracy''': 0.6, '''eval_loss''': 0.9},
        },
        {
            '''framework''': '''tensorflow''',
            '''script''': '''run_tf.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.g4dn.xlarge''',
            '''results''': {'''train_runtime''': 600, '''eval_accuracy''': 0.3, '''eval_loss''': 0.9},
        },
    ]						)
class 				__snake_case						(     unittest.TestCase						):




							def    UpperCAmelCase__			(       self     :       int):
									if self.framework == "pytorch":
											subprocess.run(
											    F"""cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py""".split()				,				encoding='''utf-8'''				,				check=A_				,				)
									assert hasattr(self				,				'''env''')




							def    UpperCAmelCase__			(       self     :       Union[str, Any]				,				A_     :       str=1):
									# creates estimator
									return HuggingFace(
									    entry_point=self.script				,				source_dir=self.env.test_path				,				role=self.env.role				,				image_uri=self.env.image_uri				,				base_job_name=F"""{self.env.base_job_name}-single"""				,				instance_count=A_				,				instance_type=self.instance_type				,				debugger_hook_config=A_				,				hyperparameters={**self.env.hyperparameters, '''model_name_or_path''': self.model_name_or_path}				,				metric_definitions=self.env.metric_definitions				,				py_version='''py36'''				,				)




							def    UpperCAmelCase__			(       self     :       List[str]				,				A_     :       Optional[Any]):
									TrainingJobAnalytics(A_).export_csv(F"""{self.env.test_path}/{job_name}_metrics.csv""")







							def    UpperCAmelCase__			(       self     :       int):
									# create estimator
									lowerCAmelCase_			:     List[Any]									= self.create_estimator()

									# run training
									estimator.fit()

									# result dataframe
									lowerCAmelCase_			:     Union[str, Any]									= TrainingJobAnalytics(estimator.latest_training_job.name).dataframe()

									# extract kpis
									lowerCAmelCase_			:     Optional[Any]									= list(result_metrics_df[result_metrics_df.metric_name == '''eval_accuracy''']['''value'''])
									lowerCAmelCase_			:     Optional[int]									= list(result_metrics_df[result_metrics_df.metric_name == '''eval_loss''']['''value'''])
									# get train time from SageMaker job, this includes starting, preprocessing, stopping
									lowerCAmelCase_			:     Dict									= (
									    Session().describe_training_job(estimator.latest_training_job.name).get('''TrainingTimeInSeconds'''				,				9_9_9_9_9_9)
									)

									# assert kpis
									assert train_runtime <= self.results["train_runtime"]
									assert all(t >= self.results['''eval_accuracy'''] for t in eval_accuracy)
									assert all(t <= self.results['''eval_loss'''] for t in eval_loss)

									# dump tests result into json file to share in PR
									with open(F"""{estimator.latest_training_job.name}.json"""				,				'''w''') as outfile:
											json.dump({'''train_time''': train_runtime, '''eval_accuracy''': eval_accuracy, '''eval_loss''': eval_loss}				,				A_)





