



import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
   from sagemaker import Session, TrainingJobAnalytics
   from sagemaker.huggingface import HuggingFace



@pytest.mark.skipif(
    literal_eval(os.getenv('''TEST_SAGEMAKER'''      ,   '''False'''  )  ) is not True      ,   reason='''Skipping test because should only be run when releasing minor transformers version'''      ,   )
@pytest.mark.usefixtures('''sm_env'''  )
@parameterized_class(
    [
        {
            '''framework''': '''pytorch''',
            '''script''': '''run_glue.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.p3.16xlarge''',
            '''results''': {'''train_runtime''': 650, '''eval_accuracy''': 0.7, '''eval_loss''': 0.6},
        },
        {
            '''framework''': '''pytorch''',
            '''script''': '''run_ddp.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.p3.16xlarge''',
            '''results''': {'''train_runtime''': 600, '''eval_accuracy''': 0.7, '''eval_loss''': 0.6},
        },
        {
            '''framework''': '''tensorflow''',
            '''script''': '''run_tf_dist.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.p3.16xlarge''',
            '''results''': {'''train_runtime''': 600, '''eval_accuracy''': 0.6, '''eval_loss''': 0.7},
        },
    ]  )
class      _UpperCAmelCase      (							unittest.TestCase  ):




     def 		snake_case_		(					self):
         if self.framework == "pytorch":
             subprocess.run(
                 F"cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py".split()			,    encoding='''utf-8'''			,    check=a__			,    )
         assert hasattr(self			,    '''env''')




     def 		snake_case_		(					self			,    a__):
         A__					=       F"{self.env.base_job_name}-{instance_count}-{'ddp' if 'ddp' in self.script else 'smd'}"
         # distributed data settings
         A__					=       {'''smdistributed''': {'''dataparallel''': {'''enabled''': True}}} if self.script != '''run_ddp.py''' else None

         # creates estimator
         return HuggingFace(
             entry_point=self.script			,    source_dir=self.env.test_path			,    role=self.env.role			,    image_uri=self.env.image_uri			,    base_job_name=a__			,    instance_count=a__			,    instance_type=self.instance_type			,    debugger_hook_config=a__			,    hyperparameters={**self.env.distributed_hyperparameters, '''model_name_or_path''': self.model_name_or_path}			,    metric_definitions=self.env.metric_definitions			,    distribution=a__			,    py_version='''py36'''			,    )




     def 		snake_case_		(					self			,    a__):
         TrainingJobAnalytics(a__).export_csv(F"{self.env.test_path}/{job_name}_metrics.csv")




     @parameterized.expand([(2,)])
     def 		snake_case_		(					self			,    a__):
         # create estimator
         A__					=       self.create_estimator(a__)

         # run training
         estimator.fit()

         # result dataframe
         A__					=       TrainingJobAnalytics(estimator.latest_training_job.name).dataframe()

         # extract kpis
         A__					=       list(result_metrics_df[result_metrics_df.metric_name == '''eval_accuracy''']['''value'''])
         A__					=       list(result_metrics_df[result_metrics_df.metric_name == '''eval_loss''']['''value'''])
         # get train time from SageMaker job, this includes starting, preprocessing, stopping
         A__					=       (
             Session().describe_training_job(estimator.latest_training_job.name).get('''TrainingTimeInSeconds'''			,    9_9_9_9_9_9)
         )

         # assert kpis
         assert train_runtime <= self.results["train_runtime"]
         assert all(t >= self.results['''eval_accuracy'''] for t in eval_accuracy)
         assert all(t <= self.results['''eval_loss'''] for t in eval_loss)

         # dump tests result into json file to share in PR
         with open(F"{estimator.latest_training_job.name}.json"			,    '''w''') as outfile:
             json.dump({'''train_time''': train_runtime, '''eval_accuracy''': eval_accuracy, '''eval_loss''': eval_loss}			,    a__)





import faiss  # noqa: F401 # Here to have a nice missing dependency error message early on
import numpy  # noqa: F401 # Here to have a nice missing dependency error message early on
import requests  # noqa: F401 # Here to have a nice missing dependency error message early on
import sklearn  # noqa: F401 # Here to have a nice missing dependency error message early on
import tqdm  # noqa: F401 # Here to have a nice missing dependency error message early on
from mauve import compute_mauve  # From: mauve-text

import datasets


_lowercase    	=		"\\n@inproceedings{pillutla-etal:mauve:neurips2021,\n  title={MAUVE: Measuring the Gap Between Neural Text and Human Text using Divergence Frontiers},\n  author={Pillutla, Krishna and Swayamdipta, Swabha and Zellers, Rowan and Thickstun, John and Welleck, Sean and Choi, Yejin and Harchaoui, Zaid},\n  booktitle = {NeurIPS},\n  year      = {2021}\n}\n\n"

_lowercase    	=		"\\nMAUVE is a library built on PyTorch and HuggingFace Transformers to measure the gap between neural text and human text with the eponymous MAUVE measure.\n\nMAUVE summarizes both Type I and Type II errors measured softly using Kullback–Leibler (KL) divergences.\n\nFor details, see the MAUVE paper: https://arxiv.org/abs/2102.01454 (Neurips, 2021).\n\nThis metrics is a wrapper around the official implementation of MAUVE:\nhttps://github.com/krishnap25/mauve\n"

_lowercase    	=		"\nCalculates MAUVE scores between two lists of generated text and reference text.\nArgs:\n    predictions: list of generated text to score. Each predictions\n        should be a string with tokens separated by spaces.\n    references: list of reference for each prediction. Each\n        reference should be a string with tokens separated by spaces.\nOptional Args:\n    num_buckets: the size of the histogram to quantize P and Q. Options: 'auto' (default) or an integer\n    pca_max_data: the number data points to use for PCA dimensionality reduction prior to clustering. If -1, use all the data. Default -1\n    kmeans_explained_var: amount of variance of the data to keep in dimensionality reduction by PCA. Default 0.9\n    kmeans_num_redo: number of times to redo k-means clustering (the best objective is kept). Default 5\n    kmeans_max_iter: maximum number of k-means iterations. Default 500\n    featurize_model_name: name of the model from which features are obtained. Default 'gpt2-large' Use one of ['gpt2', 'gpt2-medium', 'gpt2-large', 'gpt2-xl'].\n    device_id: Device for featurization. Supply a GPU id (e.g. 0 or 3) to use GPU. If no GPU with this id is found, use CPU\n    max_text_length: maximum number of tokens to consider. Default 1024\n    divergence_curve_discretization_size: Number of points to consider on the divergence curve. Default 25\n    mauve_scaling_factor: \"c\" from the paper. Default 5.\n    verbose: If True (default), print running time updates\n    seed: random seed to initialize k-means cluster assignments.\nReturns:\n    mauve: MAUVE score, a number between 0 and 1. Larger values indicate that P and Q are closer,\n    frontier_integral: Frontier Integral, a number between 0 and 1. Smaller values indicate that P and Q are closer,\n    divergence_curve: a numpy.ndarray of shape (m, 2); plot it with matplotlib to view the divergence curve,\n    p_hist: a discrete distribution, which is a quantized version of the text distribution p_text,\n    q_hist: same as above, but with q_text.\nExamples:\n\n    >>> # faiss segfaults in doctest for some reason, so the .compute call is not tested with doctest\n    >>> import datasets\n    >>> mauve = datasets.load_metric('mauve')\n    >>> predictions = [\"hello there\", \"general kenobi\"]\n    >>> references = [\"hello there\", \"general kenobi\"]\n    >>> out = mauve.compute(predictions=predictions, references=references) # doctest: +SKIP\n    >>> print(out.mauve) # doctest: +SKIP\n    1.0\n"



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION      ,   _KWARGS_DESCRIPTION  )
class      _UpperCAmelCase      (							datasets.Metric  ):




     def 		snake_case_		(					self):
         return datasets.MetricInfo(
             description=_DESCRIPTION			,    citation=_CITATION			,    homepage='''https://github.com/krishnap25/mauve'''			,    inputs_description=_KWARGS_DESCRIPTION			,    features=datasets.Features(
                 {
                     '''predictions''': datasets.Value('''string'''			,    id='''sequence'''),
                     '''references''': datasets.Value('''string'''			,    id='''sequence'''),
                 })			,    codebase_urls=['''https://github.com/krishnap25/mauve''']			,    reference_urls=[
                 '''https://arxiv.org/abs/2102.01454''',
                 '''https://github.com/krishnap25/mauve''',
             ]			,    )




     def 		snake_case_		(					self			,    a__			,    a__			,    a__=None			,    a__=None			,    a__=None			,    a__=None			,    a__="auto"			,    a__=-1			,    a__=0.9			,    a__=5			,    a__=5_0_0			,    a__="gpt2-large"			,    a__=-1			,    a__=1_0_2_4			,    a__=2_5			,    a__=5			,    a__=True			,    a__=2_5			,    ):
         A__					=       compute_mauve(
             p_text=a__			,    q_text=a__			,    p_features=a__			,    q_features=a__			,    p_tokens=a__			,    q_tokens=a__			,    num_buckets=a__			,    pca_max_data=a__			,    kmeans_explained_var=a__			,    kmeans_num_redo=a__			,    kmeans_max_iter=a__			,    featurize_model_name=a__			,    device_id=a__			,    max_text_length=a__			,    divergence_curve_discretization_size=a__			,    mauve_scaling_factor=a__			,    verbose=a__			,    seed=a__			,    )
         return out
