


import faiss  # noqa: F401 # Here to have a nice missing dependency error message early on
import numpy  # noqa: F401 # Here to have a nice missing dependency error message early on
import requests  # noqa: F401 # Here to have a nice missing dependency error message early on
import sklearn  # noqa: F401 # Here to have a nice missing dependency error message early on
import tqdm  # noqa: F401 # Here to have a nice missing dependency error message early on
from mauve import compute_mauve  # From: mauve-text

import datasets


lowerCamelCase__							:						Optional[Any]							     =   """\
@inproceedings{pillutla-etal:mauve:neurips2021,
  title={MAUVE: Measuring the Gap Between Neural Text and Human Text using Divergence Frontiers},
  author={Pillutla, Krishna and Swayamdipta, Swabha and Zellers, Rowan and Thickstun, John and Welleck, Sean and Choi, Yejin and Harchaoui, Zaid},
  booktitle = {NeurIPS},
  year      = {2021}
}

"""

lowerCamelCase__							:						int							     =   """\
MAUVE is a library built on PyTorch and HuggingFace Transformers to measure the gap between neural text and human text with the eponymous MAUVE measure.

MAUVE summarizes both Type I and Type II errors measured softly using Kullback–Leibler (KL) divergences.

For details, see the MAUVE paper: https://arxiv.org/abs/2102.01454 (Neurips, 2021).

This metrics is a wrapper around the official implementation of MAUVE:
https://github.com/krishnap25/mauve
"""

lowerCamelCase__							:						Any							     =   """
Calculates MAUVE scores between two lists of generated text and reference text.
Args:
    predictions: list of generated text to score. Each predictions
        should be a string with tokens separated by spaces.
    references: list of reference for each prediction. Each
        reference should be a string with tokens separated by spaces.
Optional Args:
    num_buckets: the size of the histogram to quantize P and Q. Options: 'auto' (default) or an integer
    pca_max_data: the number data points to use for PCA dimensionality reduction prior to clustering. If -1, use all the data. Default -1
    kmeans_explained_var: amount of variance of the data to keep in dimensionality reduction by PCA. Default 0.9
    kmeans_num_redo: number of times to redo k-means clustering (the best objective is kept). Default 5
    kmeans_max_iter: maximum number of k-means iterations. Default 500
    featurize_model_name: name of the model from which features are obtained. Default 'gpt2-large' Use one of ['gpt2', 'gpt2-medium', 'gpt2-large', 'gpt2-xl'].
    device_id: Device for featurization. Supply a GPU id (e.g. 0 or 3) to use GPU. If no GPU with this id is found, use CPU
    max_text_length: maximum number of tokens to consider. Default 1024
    divergence_curve_discretization_size: Number of points to consider on the divergence curve. Default 25
    mauve_scaling_factor: \"c\" from the paper. Default 5.
    verbose: If True (default), print running time updates
    seed: random seed to initialize k-means cluster assignments.
Returns:
    mauve: MAUVE score, a number between 0 and 1. Larger values indicate that P and Q are closer,
    frontier_integral: Frontier Integral, a number between 0 and 1. Smaller values indicate that P and Q are closer,
    divergence_curve: a numpy.ndarray of shape (m, 2); plot it with matplotlib to view the divergence curve,
    p_hist: a discrete distribution, which is a quantized version of the text distribution p_text,
    q_hist: same as above, but with q_text.
Examples:

    >>> # faiss segfaults in doctest for some reason, so the .compute call is not tested with doctest
    >>> import datasets
    >>> mauve = datasets.load_metric('mauve')
    >>> predictions = [\"hello there\", \"general kenobi\"]
    >>> references = [\"hello there\", \"general kenobi\"]
    >>> out = mauve.compute(predictions=predictions, references=references) # doctest: +SKIP
    >>> print(out.mauve) # doctest: +SKIP
    1.0
"""
@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION   ,_KWARGS_DESCRIPTION				)
class 		__magic_name__						(datasets.Metric				):





     '''simple docstring'''



     def 	SCREAMING_SNAKE_CASE__						(					self:int   ):
       return datasets.MetricInfo(
           description=_DESCRIPTION     ,		citation=_CITATION     ,		homepage='''https://github.com/krishnap25/mauve'''     ,		inputs_description=_KWARGS_DESCRIPTION     ,		features=datasets.Features(
               {
                   '''predictions''': datasets.Value('''string'''     ,		id='''sequence'''   ),
                   '''references''': datasets.Value('''string'''     ,		id='''sequence'''   ),
               }   )     ,		codebase_urls=['''https://github.com/krishnap25/mauve''']     ,		reference_urls=[
               '''https://arxiv.org/abs/2102.01454''',
               '''https://github.com/krishnap25/mauve''',
           ]     ,		)







     def 	SCREAMING_SNAKE_CASE__						(					self:List[str]     ,		_a:Any     ,		_a:Tuple     ,		_a:str=None     ,		_a:str=None     ,		_a:List[Any]=None     ,		_a:Dict=None     ,		_a:List[Any]="auto"     ,		_a:Optional[int]=-1     ,		_a:int=0.9     ,		_a:str=5     ,		_a:List[str]=5_00     ,		_a:Tuple="gpt2-large"     ,		_a:Union[str, Any]=-1     ,		_a:Optional[int]=10_24     ,		_a:Optional[Any]=25     ,		_a:Optional[Any]=5     ,		_a:Optional[Any]=True     ,		_a:List[str]=25     ,		):
       snake_case__						= compute_mauve(
           p_text=_a     ,		q_text=_a     ,		p_features=_a     ,		q_features=_a     ,		p_tokens=_a     ,		q_tokens=_a     ,		num_buckets=_a     ,		pca_max_data=_a     ,		kmeans_explained_var=_a     ,		kmeans_num_redo=_a     ,		kmeans_max_iter=_a     ,		featurize_model_name=_a     ,		device_id=_a     ,		max_text_length=_a     ,		divergence_curve_discretization_size=_a     ,		mauve_scaling_factor=_a     ,		verbose=_a     ,		seed=_a     ,		)
       return out


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


SCREAMING_SNAKE_CASE		:     Union[str, Any]         =	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE		:     Dict         =	{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

SCREAMING_SNAKE_CASE		:     Dict         =	[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

SCREAMING_SNAKE_CASE		:     int         =	{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
SCREAMING_SNAKE_CASE		:     Any         =	{F'''funnel-transformer/{name}''': 512 for name in _model_names}
SCREAMING_SNAKE_CASE		:     Tuple         =	{F'''funnel-transformer/{name}''': {"do_lower_case": True} for name in _model_names}

class        UpperCamelCase						(				__a				):
						a__		:Dict      =			VOCAB_FILES_NAMES
						a__		:Dict      =			PRETRAINED_VOCAB_FILES_MAP
						a__		:Dict      =			PRETRAINED_INIT_CONFIGURATION
						a__		:str      =			FunnelTokenizer
						a__		:List[str]      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						a__		:int      =			2





						def __init__(self			,     __UpperCamelCase=None			,     __UpperCamelCase=None			,     __UpperCamelCase=True			,     __UpperCamelCase="<unk>"			,     __UpperCamelCase="<sep>"			,     __UpperCamelCase="<pad>"			,     __UpperCamelCase="<cls>"			,     __UpperCamelCase="<mask>"			,     __UpperCamelCase="<s>"			,     __UpperCamelCase="</s>"			,     __UpperCamelCase=True			,     __UpperCamelCase=True			,     __UpperCamelCase=None			,     __UpperCamelCase="##"			,     **__UpperCamelCase			,     )				->				List[Any]:
												super().__init__(
												    __UpperCamelCase			,     tokenizer_file=__UpperCamelCase			,     do_lower_case=__UpperCamelCase			,     unk_token=__UpperCamelCase			,     sep_token=__UpperCamelCase			,     pad_token=__UpperCamelCase			,     cls_token=__UpperCamelCase			,     mask_token=__UpperCamelCase			,     bos_token=__UpperCamelCase			,     eos_token=__UpperCamelCase			,     clean_text=__UpperCamelCase			,     tokenize_chinese_chars=__UpperCamelCase			,     strip_accents=__UpperCamelCase			,     wordpieces_prefix=__UpperCamelCase			,     **__UpperCamelCase			,     )

												UpperCamelCase_		:							List[str]  =		json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
												if (
												    normalizer_state.get("""lowercase"""			,     __UpperCamelCase						) != do_lower_case
												    or normalizer_state.get("""strip_accents"""			,     __UpperCamelCase						) != strip_accents
												    or normalizer_state.get("""handle_chinese_chars"""			,     __UpperCamelCase						) != tokenize_chinese_chars
												):
																		UpperCamelCase_		:							List[Any]  =		getattr(__UpperCamelCase			,     normalizer_state.pop("""type"""						)						)
																		UpperCamelCase_		:							Union[str, Any]  =		do_lower_case
																		UpperCamelCase_		:							Tuple  =		strip_accents
																		UpperCamelCase_		:							Dict  =		tokenize_chinese_chars
																		UpperCamelCase_		:							Union[str, Any]  =		normalizer_class(**__UpperCamelCase						)

												UpperCamelCase_		:							Dict  =		do_lower_case





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase=None						)				->				Dict:
												UpperCamelCase_		:							Any  =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

												if token_ids_a:
																		output += token_ids_a + [self.sep_token_id]

												return output





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase = None						)				->				List[int]:
												UpperCamelCase_		:							Optional[Any]  =		[self.sep_token_id]
												UpperCamelCase_		:							Any  =		[self.cls_token_id]
												if token_ids_a is None:
																		return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0]
												return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase = None						)				->				Tuple[str]:
												UpperCamelCase_		:							int  =		self._tokenizer.model.save(__UpperCamelCase			,     name=__UpperCamelCase						)
												return tuple(__UpperCamelCase						)






