



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_lowercase    	=		logging.get_logger(__name__)

_lowercase    	=		"▁"

_lowercase    	=		{"vocab_file": "sentencepiece.bpe.model"}

_lowercase    	=		{
    "vocab_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/blob/main/sentencepiece.bpe.model"
        ),
    }
}

_lowercase    	=		{
    "facebook/nllb-200-distilled-600M": 1024,
}

# fmt: off
_lowercase    	=		["ace_Arab", "ace_Latn", "acm_Arab", "acq_Arab", "aeb_Arab", "afr_Latn", "ajp_Arab", "aka_Latn", "amh_Ethi", "apc_Arab", "arb_Arab", "ars_Arab", "ary_Arab", "arz_Arab", "asm_Beng", "ast_Latn", "awa_Deva", "ayr_Latn", "azb_Arab", "azj_Latn", "bak_Cyrl", "bam_Latn", "ban_Latn", "bel_Cyrl", "bem_Latn", "ben_Beng", "bho_Deva", "bjn_Arab", "bjn_Latn", "bod_Tibt", "bos_Latn", "bug_Latn", "bul_Cyrl", "cat_Latn", "ceb_Latn", "ces_Latn", "cjk_Latn", "ckb_Arab", "crh_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "dik_Latn", "dyu_Latn", "dzo_Tibt", "ell_Grek", "eng_Latn", "epo_Latn", "est_Latn", "eus_Latn", "ewe_Latn", "fao_Latn", "pes_Arab", "fij_Latn", "fin_Latn", "fon_Latn", "fra_Latn", "fur_Latn", "fuv_Latn", "gla_Latn", "gle_Latn", "glg_Latn", "grn_Latn", "guj_Gujr", "hat_Latn", "hau_Latn", "heb_Hebr", "hin_Deva", "hne_Deva", "hrv_Latn", "hun_Latn", "hye_Armn", "ibo_Latn", "ilo_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jav_Latn", "jpn_Jpan", "kab_Latn", "kac_Latn", "kam_Latn", "kan_Knda", "kas_Arab", "kas_Deva", "kat_Geor", "knc_Arab", "knc_Latn", "kaz_Cyrl", "kbp_Latn", "kea_Latn", "khm_Khmr", "kik_Latn", "kin_Latn", "kir_Cyrl", "kmb_Latn", "kon_Latn", "kor_Hang", "kmr_Latn", "lao_Laoo", "lvs_Latn", "lij_Latn", "lim_Latn", "lin_Latn", "lit_Latn", "lmo_Latn", "ltg_Latn", "ltz_Latn", "lua_Latn", "lug_Latn", "luo_Latn", "lus_Latn", "mag_Deva", "mai_Deva", "mal_Mlym", "mar_Deva", "min_Latn", "mkd_Cyrl", "plt_Latn", "mlt_Latn", "mni_Beng", "khk_Cyrl", "mos_Latn", "mri_Latn", "zsm_Latn", "mya_Mymr", "nld_Latn", "nno_Latn", "nob_Latn", "npi_Deva", "nso_Latn", "nus_Latn", "nya_Latn", "oci_Latn", "gaz_Latn", "ory_Orya", "pag_Latn", "pan_Guru", "pap_Latn", "pol_Latn", "por_Latn", "prs_Arab", "pbt_Arab", "quy_Latn", "ron_Latn", "run_Latn", "rus_Cyrl", "sag_Latn", "san_Deva", "sat_Beng", "scn_Latn", "shn_Mymr", "sin_Sinh", "slk_Latn", "slv_Latn", "smo_Latn", "sna_Latn", "snd_Arab", "som_Latn", "sot_Latn", "spa_Latn", "als_Latn", "srd_Latn", "srp_Cyrl", "ssw_Latn", "sun_Latn", "swe_Latn", "swh_Latn", "szl_Latn", "tam_Taml", "tat_Cyrl", "tel_Telu", "tgk_Cyrl", "tgl_Latn", "tha_Thai", "tir_Ethi", "taq_Latn", "taq_Tfng", "tpi_Latn", "tsn_Latn", "tso_Latn", "tuk_Latn", "tum_Latn", "tur_Latn", "twi_Latn", "tzm_Tfng", "uig_Arab", "ukr_Cyrl", "umb_Latn", "urd_Arab", "uzn_Latn", "vec_Latn", "vie_Latn", "war_Latn", "wol_Latn", "xho_Latn", "ydd_Hebr", "yor_Latn", "yue_Hant", "zho_Hans", "zho_Hant", "zul_Latn"]



class      _UpperCAmelCase      (							A__  ):
     UpperCamelCase__		     =					VOCAB_FILES_NAMES
     UpperCamelCase__		     =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCamelCase__		     =					PRETRAINED_VOCAB_FILES_MAP
     UpperCamelCase__		     =					['''input_ids''', '''attention_mask''']

     UpperCamelCase__		     =					[]
     UpperCamelCase__		     =					[]




     def __init__(					self			,    a__			,    a__="<s>"			,    a__="</s>"			,    a__="</s>"			,    a__="<s>"			,    a__="<unk>"			,    a__="<pad>"			,    a__="<mask>"			,    a__=None			,    a__=None			,    a__=None			,    a__ = None			,    a__=None			,    a__=False			,    **a__			,    ):
         # Mask token behave like a normal word, i.e. include the space before it
         A__					=       AddedToken(a__			,    lstrip=a__			,    rstrip=a__) if isinstance(a__			,    a__) else mask_token

         A__					=       {} if sp_model_kwargs is None else sp_model_kwargs
         A__					=       legacy_behaviour

         super().__init__(
             bos_token=a__			,    eos_token=a__			,    unk_token=a__			,    sep_token=a__			,    cls_token=a__			,    pad_token=a__			,    mask_token=a__			,    tokenizer_file=a__			,    src_lang=a__			,    tgt_lang=a__			,    additional_special_tokens=a__			,    sp_model_kwargs=self.sp_model_kwargs			,    legacy_behaviour=a__			,    **a__			,    )

         A__					=       spm.SentencePieceProcessor(**self.sp_model_kwargs)
         self.sp_model.Load(str(a__))
         A__					=       vocab_file
         # Original fairseq vocab and spm vocab must be "aligned":
         # Vocab    |    0    |    1    |   2    |    3    |  4   |  5   |  6   |   7  |   8  |  9
         # -------- | ------- | ------- | ------ | ------- | ---- | ---- | ---- | ---- | ---- | ----
         # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | 'an' | '▁n' | '▁m' | '▁t' | '▁k' | '▁a'
         # spm      | '<unk>' | '<s>'   | '</s>' | 'an'    | '▁n' | '▁m' | '▁t' | '▁k' | '▁a' | '▁s'

         # Mimic fairseq token-to-id alignment for the first 4 token
         A__					=       {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

         # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
         A__					=       1

         A__					=       len(self.sp_model)
         A__					=       {
             code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(a__)
         }
         A__					=       {v: k for k, v in self.lang_code_to_id.items()}
         A__					=       len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset

         self.fairseq_tokens_to_ids.update(self.lang_code_to_id)
         A__					=       {v: k for k, v in self.fairseq_tokens_to_ids.items()}
         A__					=       list(self.lang_code_to_id.keys())

         if additional_special_tokens is not None:
             # Only add those special tokens if they are not already there.
             self._additional_special_tokens.extend(
                 [t for t in additional_special_tokens if t not in self._additional_special_tokens])

         A__					=       src_lang if src_lang is not None else '''eng_Latn'''
         A__					=       self.lang_code_to_id[self._src_lang]
         A__					=       tgt_lang
         self.set_src_lang_special_tokens(self._src_lang)




     def __getstate__(					self):
         A__					=       self.__dict__.copy()
         A__					=       None
         A__					=       self.sp_model.serialized_model_proto()
         return state




     def __setstate__(					self			,    a__):
         A__					=       d

         # for backward compatibility
         if not hasattr(self			,    '''sp_model_kwargs'''):
             A__					=       {}

         A__					=       spm.SentencePieceProcessor(**self.sp_model_kwargs)
         self.sp_model.LoadFromSerializedProto(self.sp_model_proto)




     @property
     def 		snake_case_		(					self):
         return len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset + 1  # Plus 1 for the mask token




     @property
     def 		snake_case_		(					self):
         return self._src_lang




     @src_lang.setter
     def 		snake_case_		(					self			,    a__):
         A__					=       new_src_lang
         self.set_src_lang_special_tokens(self._src_lang)




     def 		snake_case_		(					self			,    a__			,    a__ = None			,    a__ = False):

         if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=a__			,    token_ids_a=a__			,    already_has_special_tokens=a__)

         A__					=       [1] * len(self.prefix_tokens)
         A__					=       [1] * len(self.suffix_tokens)
         if token_ids_a is None:
             return prefix_ones + ([0] * len(a__)) + suffix_ones
         return prefix_ones + ([0] * len(a__)) + ([0] * len(a__)) + suffix_ones




     def 		snake_case_		(					self			,    a__			,    a__ = None):
         if token_ids_a is None:
             return self.prefix_tokens + token_ids_a + self.suffix_tokens
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




     def 		snake_case_		(					self			,    a__			,    a__ = None):
         A__					=       [self.sep_token_id]
         A__					=       [self.cls_token_id]

         if token_ids_a is None:
             return len(cls + token_ids_a + sep) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]




     def 		snake_case_		(					self			,    a__			,    a__			,    a__			,    a__			,    **a__):
         if src_lang is None or tgt_lang is None:
             raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model''')
         A__					=       src_lang
         A__					=       self(a__			,    add_special_tokens=a__			,    return_tensors=a__			,    **a__)
         A__					=       self.convert_tokens_to_ids(a__)
         A__					=       tgt_lang_id
         return inputs




     def 		snake_case_		(					self):
         A__					=       {self.convert_ids_to_tokens(a__): i for i in range(self.vocab_size)}
         vocab.update(self.added_tokens_encoder)
         return vocab




     def 		snake_case_		(					self			,    a__):
         return self.sp_model.encode(a__			,    out_type=a__)




     def 		snake_case_		(					self			,    a__):
         if token in self.fairseq_tokens_to_ids:
             return self.fairseq_tokens_to_ids[token]
         A__					=       self.sp_model.PieceToId(a__)

         # Need to return unknown token if the SP model returned 0
         return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




     def 		snake_case_		(					self			,    a__):
         if index in self.fairseq_ids_to_tokens:
             return self.fairseq_ids_to_tokens[index]
         return self.sp_model.IdToPiece(index - self.fairseq_offset)




     def 		snake_case_		(					self			,    a__):
         A__					=       ''''''.join(a__).replace(a__			,    ''' ''').strip()
         return out_string




     def 		snake_case_		(					self			,    a__			,    a__ = None):
         if not os.path.isdir(a__):
             logger.error(F"Vocabulary path ({save_directory}) should be a directory")
             return
         A__					=       os.path.join(
             a__			,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

         if os.path.abspath(self.vocab_file) != os.path.abspath(a__) and os.path.isfile(self.vocab_file):
             copyfile(self.vocab_file			,    a__)
         elif not os.path.isfile(self.vocab_file):
             with open(a__			,    '''wb''') as fi:
                 A__					=       self.sp_model.serialized_model_proto()
                 fi.write(a__)

         return (out_vocab_file,)




     def 		snake_case_		(					self			,    a__			,    a__ = "eng_Latn"			,    a__ = None			,    a__ = "fra_Latn"			,    **a__			,    ):
         A__					=       src_lang
         A__					=       tgt_lang
         return super().prepare_seqaseq_batch(a__			,    a__			,    **a__)




     def 		snake_case_		(					self):
         return self.set_src_lang_special_tokens(self.src_lang)




     def 		snake_case_		(					self):
         return self.set_tgt_lang_special_tokens(self.tgt_lang)




     def 		snake_case_		(					self			,    a__):
         A__					=       self.lang_code_to_id[src_lang]
         if self.legacy_behaviour:
             A__					=       []
             A__					=       [self.eos_token_id, self.cur_lang_code]
         else:
             A__					=       [self.cur_lang_code]
             A__					=       [self.eos_token_id]




     def 		snake_case_		(					self			,    a__):
         A__					=       self.lang_code_to_id[lang]
         if self.legacy_behaviour:
             A__					=       []
             A__					=       [self.eos_token_id, self.cur_lang_code]
         else:
             A__					=       [self.cur_lang_code]
             A__					=       [self.eos_token_id]





import faiss  # noqa: F401 # Here to have a nice missing dependency error message early on
import numpy  # noqa: F401 # Here to have a nice missing dependency error message early on
import requests  # noqa: F401 # Here to have a nice missing dependency error message early on
import sklearn  # noqa: F401 # Here to have a nice missing dependency error message early on
import tqdm  # noqa: F401 # Here to have a nice missing dependency error message early on
from mauve import compute_mauve  # From: mauve-text

import datasets


_lowercase    	=		"\\n@inproceedings{pillutla-etal:mauve:neurips2021,\n  title={MAUVE: Measuring the Gap Between Neural Text and Human Text using Divergence Frontiers},\n  author={Pillutla, Krishna and Swayamdipta, Swabha and Zellers, Rowan and Thickstun, John and Welleck, Sean and Choi, Yejin and Harchaoui, Zaid},\n  booktitle = {NeurIPS},\n  year      = {2021}\n}\n\n"

_lowercase    	=		"\\nMAUVE is a library built on PyTorch and HuggingFace Transformers to measure the gap between neural text and human text with the eponymous MAUVE measure.\n\nMAUVE summarizes both Type I and Type II errors measured softly using Kullback–Leibler (KL) divergences.\n\nFor details, see the MAUVE paper: https://arxiv.org/abs/2102.01454 (Neurips, 2021).\n\nThis metrics is a wrapper around the official implementation of MAUVE:\nhttps://github.com/krishnap25/mauve\n"

_lowercase    	=		"\nCalculates MAUVE scores between two lists of generated text and reference text.\nArgs:\n    predictions: list of generated text to score. Each predictions\n        should be a string with tokens separated by spaces.\n    references: list of reference for each prediction. Each\n        reference should be a string with tokens separated by spaces.\nOptional Args:\n    num_buckets: the size of the histogram to quantize P and Q. Options: 'auto' (default) or an integer\n    pca_max_data: the number data points to use for PCA dimensionality reduction prior to clustering. If -1, use all the data. Default -1\n    kmeans_explained_var: amount of variance of the data to keep in dimensionality reduction by PCA. Default 0.9\n    kmeans_num_redo: number of times to redo k-means clustering (the best objective is kept). Default 5\n    kmeans_max_iter: maximum number of k-means iterations. Default 500\n    featurize_model_name: name of the model from which features are obtained. Default 'gpt2-large' Use one of ['gpt2', 'gpt2-medium', 'gpt2-large', 'gpt2-xl'].\n    device_id: Device for featurization. Supply a GPU id (e.g. 0 or 3) to use GPU. If no GPU with this id is found, use CPU\n    max_text_length: maximum number of tokens to consider. Default 1024\n    divergence_curve_discretization_size: Number of points to consider on the divergence curve. Default 25\n    mauve_scaling_factor: \"c\" from the paper. Default 5.\n    verbose: If True (default), print running time updates\n    seed: random seed to initialize k-means cluster assignments.\nReturns:\n    mauve: MAUVE score, a number between 0 and 1. Larger values indicate that P and Q are closer,\n    frontier_integral: Frontier Integral, a number between 0 and 1. Smaller values indicate that P and Q are closer,\n    divergence_curve: a numpy.ndarray of shape (m, 2); plot it with matplotlib to view the divergence curve,\n    p_hist: a discrete distribution, which is a quantized version of the text distribution p_text,\n    q_hist: same as above, but with q_text.\nExamples:\n\n    >>> # faiss segfaults in doctest for some reason, so the .compute call is not tested with doctest\n    >>> import datasets\n    >>> mauve = datasets.load_metric('mauve')\n    >>> predictions = [\"hello there\", \"general kenobi\"]\n    >>> references = [\"hello there\", \"general kenobi\"]\n    >>> out = mauve.compute(predictions=predictions, references=references) # doctest: +SKIP\n    >>> print(out.mauve) # doctest: +SKIP\n    1.0\n"



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION      ,   _KWARGS_DESCRIPTION  )
class      _UpperCAmelCase      (							datasets.Metric  ):




     def 		snake_case_		(					self):
         return datasets.MetricInfo(
             description=_DESCRIPTION			,    citation=_CITATION			,    homepage='''https://github.com/krishnap25/mauve'''			,    inputs_description=_KWARGS_DESCRIPTION			,    features=datasets.Features(
                 {
                     '''predictions''': datasets.Value('''string'''			,    id='''sequence'''),
                     '''references''': datasets.Value('''string'''			,    id='''sequence'''),
                 })			,    codebase_urls=['''https://github.com/krishnap25/mauve''']			,    reference_urls=[
                 '''https://arxiv.org/abs/2102.01454''',
                 '''https://github.com/krishnap25/mauve''',
             ]			,    )




     def 		snake_case_		(					self			,    a__			,    a__			,    a__=None			,    a__=None			,    a__=None			,    a__=None			,    a__="auto"			,    a__=-1			,    a__=0.9			,    a__=5			,    a__=5_0_0			,    a__="gpt2-large"			,    a__=-1			,    a__=1_0_2_4			,    a__=2_5			,    a__=5			,    a__=True			,    a__=2_5			,    ):
         A__					=       compute_mauve(
             p_text=a__			,    q_text=a__			,    p_features=a__			,    q_features=a__			,    p_tokens=a__			,    q_tokens=a__			,    num_buckets=a__			,    pca_max_data=a__			,    kmeans_explained_var=a__			,    kmeans_num_redo=a__			,    kmeans_max_iter=a__			,    featurize_model_name=a__			,    device_id=a__			,    max_text_length=a__			,    divergence_curve_discretization_size=a__			,    mauve_scaling_factor=a__			,    verbose=a__			,    seed=a__			,    )
         return out
