




from sklearn.metrics import matthews_corrcoef

import datasets


UpperCAmelCase_		:					Dict          =  "\nCompute the Matthews correlation coefficient (MCC)\n\nThe Matthews correlation coefficient is used in machine learning as a\nmeasure of the quality of binary and multiclass classifications. It takes\ninto account true and false positives and negatives and is generally\nregarded as a balanced measure which can be used even if the classes are of\nvery different sizes. The MCC is in essence a correlation coefficient value\nbetween -1 and +1. A coefficient of +1 represents a perfect prediction, 0\nan average random prediction and -1 an inverse prediction.  The statistic\nis also known as the phi coefficient. [source: Wikipedia]\n"

UpperCAmelCase_		:					Any          =  "\nArgs:\n    predictions (list of int): Predicted labels, as returned by a model.\n    references (list of int): Ground truth labels.\n    sample_weight (list of int, float, or bool): Sample weights. Defaults to `None`.\nReturns:\n    matthews_correlation (dict containing float): Matthews correlation.\nExamples:\n    Example 1, a basic example with only predictions and references as inputs:\n        >>> matthews_metric = datasets.load_metric(\"matthews_correlation\")\n        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],\n        ...                                     predictions=[1, 2, 2, 0, 3, 3])\n        >>> print(round(results['matthews_correlation'], 2))\n        0.54\n\n    Example 2, the same example as above, but also including sample weights:\n        >>> matthews_metric = datasets.load_metric(\"matthews_correlation\")\n        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],\n        ...                                     predictions=[1, 2, 2, 0, 3, 3],\n        ...                                     sample_weight=[0.5, 3, 1, 1, 1, 2])\n        >>> print(round(results['matthews_correlation'], 2))\n        0.1\n\n    Example 3, the same example as above, but with sample weights that cause a negative correlation:\n        >>> matthews_metric = datasets.load_metric(\"matthews_correlation\")\n        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],\n        ...                                     predictions=[1, 2, 2, 0, 3, 3],\n        ...                                     sample_weight=[0.5, 1, 0, 0, 0, 1])\n        >>> print(round(results['matthews_correlation'], 2))\n        -0.25\n"

UpperCAmelCase_		:					Dict          =  "\\n@article{scikit-learn,\n  title={Scikit-learn: Machine Learning in {P}ython},\n  author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.\n         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.\n         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and\n         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},\n  journal={Journal of Machine Learning Research},\n  volume={12},\n  pages={2825--2830},\n  year={2011}\n}\n"




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION       ,    _KWARGS_DESCRIPTION      )
class 		__A   (      datasets.Metric      ):

						def      A__    (		self :List[str]			):
										'''simple docstring'''

										return datasets.MetricInfo(
										    description=_DESCRIPTION     ,			citation=_CITATION     ,			inputs_description=_KWARGS_DESCRIPTION     ,			features=datasets.Features(
										        {
										            """predictions""": datasets.Value("""int32"""			),
										            """references""": datasets.Value("""int32"""			),
										        }			)     ,			reference_urls=[
										        """https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html"""
										    ]     ,			)


						def      A__    (		self :Tuple     ,			__snake_case :str     ,			__snake_case :Tuple     ,			__snake_case :List[str]=None			):
										'''simple docstring'''

										return {
										    "matthews_correlation": float(matthews_corrcoef(__snake_case     ,			__snake_case     ,			sample_weight=__snake_case			)			),
										}











"""simple docstring"""






import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation

__UpperCAmelCase						=			logging.get_logger(__name__)


__UpperCAmelCase						=			{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

__UpperCAmelCase						=			{
    '''vocab_file''': {'''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json'''},
    '''merges_file''': {'''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt'''},
    '''tokenizer_config_file''': {
        '''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json'''
    },
}

__UpperCAmelCase						=			{'''facebook/blenderbot-3B''': 128}






@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def      lowercase__ (					)    ->			Any:



			'''simple docstring'''
			a__  :	Any		 =			(
			    list(range(ord("!"			)							,			ord("~"			) + 1			)			) + list(range(ord("¡"			)							,			ord("¬"			) + 1			)			) + list(range(ord("®"			)							,			ord("ÿ"			) + 1			)			)
			)
			a__  :	Any		 =			bs[:]
			a__  :	int		 =			0
			for b in range(2**8			):
						if b not in bs:
									bs.append(lowerCAmelCase__			)
									cs.append(2**8 + n			)
									n += 1
			a__  :	Tuple		 =			[chr(lowerCAmelCase__			) for n in cs]
			return dict(zip(lowerCAmelCase__							,			lowerCAmelCase__			)			)






def      lowercase__ (					lowerCAmelCase__   :			List[str]			)    ->			Any:



			'''simple docstring'''
			a__  :	Tuple		 =			set()
			a__  :	Union[str, Any]		 =			word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)			)
						a__  :	Tuple		 =			char
			return pairs





class 					__UpperCAmelCase							(						_UpperCamelCase  ):
				__lowerCamelCase				:	int				     =				VOCAB_FILES_NAMES
				__lowerCamelCase				:	List[Any]				     =				PRETRAINED_VOCAB_FILES_MAP
				__lowerCamelCase				:	Optional[int]				     =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__lowerCamelCase				:	List[str]				     =				["input_ids", "attention_mask"]
				def __init__(		self    :			str	, a_    :			Optional[Any]	, a_    :			Union[str, Any]	, a_    :			int="replace"	, a_    :			Optional[Any]="<s>"	, a_    :			Optional[Any]="</s>"	, a_    :			Any="</s>"	, a_    :			Dict="<s>"	, a_    :			Optional[int]="<unk>"	, a_    :			Optional[int]="<pad>"	, a_    :			Any="<mask>"	, a_    :			Optional[Any]=False	, **a_    :			str	, )	->	Tuple:



							'''simple docstring'''



							a__  :	Optional[int]		 =			AddedToken(a_	, lstrip=a_	, rstrip=a_    ) if isinstance(a_	, a_    ) else bos_token
							a__  :	Any		 =			AddedToken(a_	, lstrip=a_	, rstrip=a_    ) if isinstance(a_	, a_    ) else eos_token
							a__  :	List[str]		 =			AddedToken(a_	, lstrip=a_	, rstrip=a_    ) if isinstance(a_	, a_    ) else sep_token
							a__  :	str		 =			AddedToken(a_	, lstrip=a_	, rstrip=a_    ) if isinstance(a_	, a_    ) else cls_token
							a__  :	Optional[Any]		 =			AddedToken(a_	, lstrip=a_	, rstrip=a_    ) if isinstance(a_	, a_    ) else unk_token
							a__  :	Tuple		 =			AddedToken(a_	, lstrip=a_	, rstrip=a_    ) if isinstance(a_	, a_    ) else pad_token

							# Mask token behave like a normal word, i.e. include the space before it
							a__  :	Dict		 =			AddedToken(a_	, lstrip=a_	, rstrip=a_    ) if isinstance(a_	, a_    ) else mask_token

							super().__init__(
							    errors=a_	, bos_token=a_	, eos_token=a_	, unk_token=a_	, sep_token=a_	, cls_token=a_	, pad_token=a_	, mask_token=a_	, add_prefix_space=a_	, **a_	, )

							with open(a_	, encoding="utf-8"    ) as vocab_handle:
										a__  :	List[Any]		 =			json.load(a_    )
							a__  :	str		 =			{v: k for k, v in self.encoder.items()}
							a__  :	int		 =			errors  # how to handle errors in decoding
							a__  :	int		 =			bytes_to_unicode()
							a__  :	List[Any]		 =			{v: k for k, v in self.byte_encoder.items()}
							with open(a_	, encoding="utf-8"    ) as merges_handle:
										a__  :	Dict		 =			merges_handle.read().split("\n"    )[1:-1]
							a__  :	Dict		 =			[tuple(merge.split()    ) for merge in bpe_merges]
							a__  :	str		 =			dict(zip(a_	, range(len(a_    )    )    )    )
							a__  :	Tuple		 =			{}
							a__  :	Optional[int]		 =			add_prefix_space

							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
							a__  :	List[str]		 =			re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"    )
				@property
				# Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
				def       UpperCAmelCase      (		self    :			Any    )	->	Optional[int]:



							'''simple docstring'''



							return len(self.encoder    )
				def       UpperCAmelCase      (		self    :			Optional[Any]    )	->	List[str]:



							'''simple docstring'''



							return dict(self.encoder	, **self.added_tokens_encoder    )
				def       UpperCAmelCase      (		self    :			Dict	, a_    :			str    )	->	int:



							'''simple docstring'''



							if token in self.cache:
										return self.cache[token]
							a__  :	List[str]		 =			tuple(a_    )
							a__  :	Dict		 =			get_pairs(a_    )

							if not pairs:
										return token

							while True:
										a__  :	Dict		 =			min(a_	, key=lambda a_    : self.bpe_ranks.get(a_	, float("inf"    )    )    )
										if bigram not in self.bpe_ranks:
													break
										a__ ,	a__  :	List[Any]		 =			bigram
										a__  :	int		 =			[]
										a__  :	List[Any]		 =			0
										while i < len(a_    ):
													try:
																a__  :	List[str]		 =			word.index(a_	, a_    )
													except ValueError:
																new_word.extend(word[i:]    )
																break
													else:
																new_word.extend(word[i:j]    )
																a__  :	Dict		 =			j

													if word[i] == first and i < len(a_    ) - 1 and word[i + 1] == second:
																new_word.append(first + second    )
																i += 2
													else:
																new_word.append(word[i]    )
																i += 1
										a__  :	List[Any]		 =			tuple(a_    )
										a__  :	str		 =			new_word
										if len(a_    ) == 1:
													break
										else:
													a__  :	Optional[int]		 =			get_pairs(a_    )
							a__  :	Union[str, Any]		 =			" ".join(a_    )
							a__  :	str		 =			word
							return word
				def       UpperCAmelCase      (		self    :			Optional[int]	, a_    :			Union[str, Any]    )	->	List[str]:



							'''simple docstring'''



							a__  :	List[Any]		 =			[]
							for token in re.findall(self.pat	, a_    ):
										a__  :	Dict		 =			"".join(
										    self.byte_encoder[b] for b in token.encode("utf-8"    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
										bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a_    ).split(" "    )    )
							return bpe_tokens
				def       UpperCAmelCase      (		self    :			Union[str, Any]	, a_    :			Optional[int]    )	->	Optional[int]:



							'''simple docstring'''



							return self.encoder.get(a_	, self.encoder.get(self.unk_token    )    )
				def       UpperCAmelCase      (		self    :			Optional[Any]	, a_    :			Optional[Any]    )	->	Optional[Any]:



							'''simple docstring'''



							return self.decoder.get(a_    )
				def       UpperCAmelCase      (		self    :			str	, a_    :			List[Any]    )	->	Optional[int]:



							'''simple docstring'''



							a__  :	str		 =			"".join(a_    )
							a__  :	List[str]		 =			bytearray([self.byte_decoder[c] for c in text]    ).decode("utf-8"	, errors=self.errors    )
							return text
				def       UpperCAmelCase      (		self    :			Any	, a_    :			str	, a_    :			Optional[str] = None    )	->	Tuple[str]:



							'''simple docstring'''



							if not os.path.isdir(a_    ):
										logger.error(F"Vocabulary path ({save_directory}) should be a directory"    )
										return
							a__  :	Dict		 =			os.path.join(
							    a_	, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )
							a__  :	List[str]		 =			os.path.join(
							    a_	, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]    )

							with open(a_	, "w"	, encoding="utf-8"    ) as f:
										f.write(json.dumps(self.encoder	, indent=2	, sort_keys=a_	, ensure_ascii=a_    ) + "\n"    )

							a__  :	Dict		 =			0
							with open(a_	, "w"	, encoding="utf-8"    ) as writer:
										writer.write("#version: 0.2\n"    )
										for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	, key=lambda a_    : kv[1]    ):
													if index != token_index:
																logger.warning(
																    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
																    " Please check that the tokenizer is not corrupted!"    )
																a__  :	Optional[Any]		 =			token_index
													writer.write(" ".join(a_    ) + "\n"    )
													index += 1

							return vocab_file, merge_file
				def       UpperCAmelCase      (		self    :			List[Any]	, a_    :			List[int]	, a_    :			Optional[List[int]] = None	, a_    :			bool = False    )	->	List[int]:



							'''simple docstring'''



							if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=a_	, token_ids_a=a_	, already_has_special_tokens=a_    )

							if token_ids_a is None:
										return [1] + ([0] * len(a_    )) + [1]
							return [1] + ([0] * len(a_    )) + [1, 1] + ([0] * len(a_    )) + [1]
				def       UpperCAmelCase      (		self    :			Optional[Any]	, a_    :			List[int]	, a_    :			Optional[List[int]] = None    )	->	List[int]:



							'''simple docstring'''



							a__  :	Dict		 =			[self.sep_token_id]
							a__  :	Tuple		 =			[self.cls_token_id]

							if token_ids_a is None:
										return len(cls + token_ids_a + sep    ) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]
				def       UpperCAmelCase      (		self    :			Any	, a_    :			Optional[Any]	, a_    :			Union[str, Any]=False	, **a_    :			Dict    )	->	Dict:



							'''simple docstring'''



							a__  :	int		 =			kwargs.pop("add_prefix_space"	, self.add_prefix_space    )
							if (is_split_into_words or add_prefix_space) and (len(a_    ) > 0 and not text[0].isspace()):
										a__  :	int		 =			" " + text
							return (text, kwargs)
				def       UpperCAmelCase      (		self    :			str	, a_    :			List[int]	, a_    :			Optional[List[int]] = None    )	->	Optional[int]:



							'''simple docstring'''



							return token_ids_a + [self.eos_token_id]






				def       UpperCAmelCase      (		self    :			str	, a_    :			"Conversation"    )	->	List[int]:



							'''simple docstring'''



							a__  :	Union[str, Any]		 =			[]
							for is_user, text in conversation.iter_texts():
										if is_user:
													# We need to space prefix as it's being done within blenderbot
													inputs.append(" " + text    )
										else:
													# Generated responses should contain them already.
													inputs.append(a_    )

							a__  :	Dict		 =			"  ".join(a_    )
							a__  :	List[str]		 =			self.encode(a_    )
							if len(a_    ) > self.model_max_length:
										a__  :	Union[str, Any]		 =			input_ids[-self.model_max_length :]
										logger.warning(F"Trimmed input from conversation as it was longer than {self.model_max_length} tokens."    )
							return input_ids