





"""simple docstring"""






import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


__UpperCAmelCase						=			'''<<<<<<< This should probably be modified because it mentions: '''

__UpperCAmelCase						=			'''=======
>>>>>>>
'''

__UpperCAmelCase						=			[
    '''TextEncoderConfig''',
    '''ByteTextEncoder''',
    '''SubwordTextEncoder''',
    '''encoder_config''',
    '''maybe_build_from_corpus''',
    '''manual_dir''',
]

__UpperCAmelCase						=			[
    # (pattern, replacement)
    # Order is important here for some replacements
    (R'''tfds\.core''', R'''datasets'''),
    (R'''tf\.io\.gfile\.GFile''', R'''open'''),
    (R'''tf\.([\w\d]+)''', R'''datasets.Value(\'\1\')'''),
    (R'''tfds\.features\.Text\(\)''', R'''datasets.Value(\'string\')'''),
    (R'''tfds\.features\.Text\(''', R'''datasets.Value(\'string\'),'''),
    (R'''features\s*=\s*tfds.features.FeaturesDict\(''', R'''features=datasets.Features('''),
    (R'''tfds\.features\.FeaturesDict\(''', R'''dict('''),
    (R'''The TensorFlow Datasets Authors''', R'''The TensorFlow Datasets Authors and the HuggingFace Datasets Authors'''),
    (R'''tfds\.''', R'''datasets.'''),
    (R'''dl_manager\.manual_dir''', R'''self.config.data_dir'''),
    (R'''self\.builder_config''', R'''self.config'''),
]






def      lowercase__ (					lowerCAmelCase__   :			Namespace			)    ->			int:



			'''simple docstring'''
			return ConvertCommand(args.tfds_path							,			args.datasets_directory			)





class 					__UpperCAmelCase							(						_UpperCamelCase  ):
				@staticmethod
				def       UpperCAmelCase      (		a_    :			ArgumentParser    )	->	List[str]:



							'''simple docstring'''



							a__  :	Optional[Any]		 =			parser.add_parser(
							    "convert"	, help="Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset."	, )
							train_parser.add_argument(
							    "--tfds_path"	, type=a_	, required=a_	, help="Path to a TensorFlow Datasets folder to convert or a single tfds file to convert."	, )
							train_parser.add_argument(
							    "--datasets_directory"	, type=a_	, required=a_	, help="Path to the HuggingFace Datasets folder."    )
							train_parser.set_defaults(func=a_    )
				def __init__(		self    :			Optional[int]	, a_    :			str	, a_    :			str	, *a_    :			List[str]    )	->	int:



							'''simple docstring'''



							a__  :	Dict		 =			get_logger("datasets-cli/converting"    )

							a__  :	Tuple		 =			tfds_path
							a__  :	List[str]		 =			datasets_directory






				def       UpperCAmelCase      (		self    :			Any    )	->	Union[str, Any]:



							'''simple docstring'''



							if os.path.isdir(self._tfds_path    ):
										a__  :	int		 =			os.path.abspath(self._tfds_path    )
							elif os.path.isfile(self._tfds_path    ):
										a__  :	Dict		 =			os.path.dirname(self._tfds_path    )
							else:
										raise ValueError("--tfds_path is neither a directory nor a file. Please check path."    )

							a__  :	int		 =			os.path.abspath(self._datasets_directory    )

							self._logger.info(F"Converting datasets from {abs_tfds_path} to {abs_datasets_path}"    )

							a__  :	Any		 =			[]
							a__  :	Optional[Any]		 =			[]
							a__  :	Dict		 =			{}

							if os.path.isdir(self._tfds_path    ):
										a__  :	List[Any]		 =			os.listdir(a_    )
							else:
										a__  :	List[str]		 =			[os.path.basename(self._tfds_path    )]

							for f_name in file_names:
										self._logger.info(F"Looking at file {f_name}"    )
										a__  :	List[Any]		 =			os.path.join(a_	, a_    )
										a__  :	Union[str, Any]		 =			os.path.join(a_	, a_    )

										if not os.path.isfile(a_    ) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
													self._logger.info("Skipping file"    )
													continue

										with open(a_	, encoding="utf-8"    ) as f:
													a__  :	Optional[int]		 =			f.readlines()

										a__  :	Union[str, Any]		 =			[]
										a__  :	str		 =			False
										a__  :	Dict		 =			False
										a__  :	Union[str, Any]		 =			[]
										for line in lines:
													a__  :	Dict		 =			line

													# Convert imports
													if "import tensorflow.compat.v2 as tf" in out_line:
																continue
													elif "@tfds.core" in out_line:
																continue
													elif "builder=self" in out_line:
																continue
													elif "import tensorflow_datasets.public_api as tfds" in out_line:
																a__  :	Tuple		 =			"import datasets\n"
													elif "import tensorflow" in out_line:
																# order is important here
																a__  :	List[Any]		 =			""
																continue
													elif "from absl import logging" in out_line:
																a__  :	List[Any]		 =			"from datasets import logging\n"
													elif "getLogger" in out_line:
																a__  :	Optional[int]		 =			out_line.replace("getLogger"	, "get_logger"    )
													elif any(expression in out_line for expression in TO_HIGHLIGHT    ):
																a__  :	Dict		 =			True
																a__  :	str		 =			list(filter(lambda a_    : e in out_line	, a_    )    )
																out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(a_    ) + "\n"    )
																out_lines.append(a_    )
																out_lines.append(a_    )
																continue
													else:
																for pattern, replacement in TO_CONVERT:
																			a__  :	Union[str, Any]		 =			re.sub(a_	, a_	, a_    )

                # Take care of saving utilities (to later move them together with main script)
													if "tensorflow_datasets" in out_line:
																a__  :	Optional[int]		 =			re.match(R"from\stensorflow_datasets.*import\s([^\.\r\n]+)"	, a_    )
																tfds_imports.extend(imp.strip() for imp in match.group(1    ).split(","    )    )
																a__  :	Optional[int]		 =			"from . import " + match.group(1    )

													# Check we have not forget anything
													if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
																raise ValueError(F"Error converting {out_line.strip()}"    )

													if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
																a__  :	Dict		 =			True
													out_lines.append(a_    )

										if is_builder or "wmt" in f_name:
													# We create a new directory for each dataset
													a__  :	str		 =			f_name.replace(".py"	, ""    )
													a__  :	List[str]		 =			os.path.join(a_	, a_    )
													a__  :	List[str]		 =			os.path.join(a_	, a_    )
													os.makedirs(a_	, exist_ok=a_    )
													self._logger.info(F"Adding directory {output_dir}"    )
													imports_to_builder_map.update({imp: output_dir for imp in tfds_imports}    )
										else:
													# Utilities will be moved at the end
													utils_files.append(a_    )

										if needs_manual_update:
													with_manual_update.append(a_    )

										with open(a_	, "w"	, encoding="utf-8"    ) as f:
													f.writelines(a_    )
										self._logger.info(F"Converted in {output_file}"    )

							for utils_file in utils_files:
										try:
													a__  :	int		 =			os.path.basename(a_    )
													a__  :	Dict		 =			imports_to_builder_map[f_name.replace(".py"	, ""    )]
													self._logger.info(F"Moving {dest_folder} to {utils_file}"    )
													shutil.copy(a_	, a_    )
										except KeyError:
													self._logger.error(F"Cannot find destination folder for {utils_file}. Please copy manually."    )

							if with_manual_update:
										for file_path in with_manual_update:
													self._logger.warning(
													    F"You need to manually update file {file_path} to remove configurations using 'TextEncoderConfig'."    )






"""simple docstring"""






from sklearn.metrics import matthews_corrcoef

import datasets


__UpperCAmelCase						=			'''
Compute the Matthews correlation coefficient (MCC)

The Matthews correlation coefficient is used in machine learning as a
measure of the quality of binary and multiclass classifications. It takes
into account true and false positives and negatives and is generally
regarded as a balanced measure which can be used even if the classes are of
very different sizes. The MCC is in essence a correlation coefficient value
between -1 and +1. A coefficient of +1 represents a perfect prediction, 0
an average random prediction and -1 an inverse prediction.  The statistic
is also known as the phi coefficient. [source: Wikipedia]
'''

__UpperCAmelCase						=			'''
Args:
    predictions (list of int): Predicted labels, as returned by a model.
    references (list of int): Ground truth labels.
    sample_weight (list of int, float, or bool): Sample weights. Defaults to `None`.
Returns:
    matthews_correlation (dict containing float): Matthews correlation.
Examples:
    Example 1, a basic example with only predictions and references as inputs:
        >>> matthews_metric = datasets.load_metric("matthews_correlation")
        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],
        ...                                     predictions=[1, 2, 2, 0, 3, 3])
        >>> print(round(results[\'matthews_correlation\'], 2))
        0.54

    Example 2, the same example as above, but also including sample weights:
        >>> matthews_metric = datasets.load_metric("matthews_correlation")
        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],
        ...                                     predictions=[1, 2, 2, 0, 3, 3],
        ...                                     sample_weight=[0.5, 3, 1, 1, 1, 2])
        >>> print(round(results[\'matthews_correlation\'], 2))
        0.1

    Example 3, the same example as above, but with sample weights that cause a negative correlation:
        >>> matthews_metric = datasets.load_metric("matthews_correlation")
        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],
        ...                                     predictions=[1, 2, 2, 0, 3, 3],
        ...                                     sample_weight=[0.5, 1, 0, 0, 0, 1])
        >>> print(round(results[\'matthews_correlation\'], 2))
        -0.25
'''

__UpperCAmelCase						=			'''\
@article{scikit-learn,
  title={Scikit-learn: Machine Learning in {P}ython},
  author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
  journal={Journal of Machine Learning Research},
  volume={12},
  pages={2825--2830},
  year={2011}
}
'''



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						,  _KWARGS_DESCRIPTION  )
class 					__UpperCAmelCase							(						datasets.Metric  ):
				def       UpperCAmelCase      (		self    :			Optional[int]    )	->	str:



							'''simple docstring'''



							return datasets.MetricInfo(
							    description=_DESCRIPTION	, citation=_CITATION	, inputs_description=_KWARGS_DESCRIPTION	, features=datasets.Features(
							        {
							            "predictions": datasets.Value("int32"    ),
							            "references": datasets.Value("int32"    ),
							        }    )	, reference_urls=[
							        "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html"
							    ]	, )






				def       UpperCAmelCase      (		self    :			Tuple	, a_    :			Optional[Any]	, a_    :			Optional[Any]	, a_    :			Any=None    )	->	Union[str, Any]:



							'''simple docstring'''



							return {
							    "matthews_correlation": float(matthews_corrcoef(a_	, a_	, sample_weight=a_    )    ),
							}