






"""simple docstring"""






import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


UpperCAmelCase_   : Union[str, Any]   			= """<<<<<<< This should probably be modified because it mentions: """

UpperCAmelCase_   : Optional[Any]   			= """=======
>>>>>>>
"""

UpperCAmelCase_   : Union[str, Any]   			= [
    """TextEncoderConfig""",
    """ByteTextEncoder""",
    """SubwordTextEncoder""",
    """encoder_config""",
    """maybe_build_from_corpus""",
    """manual_dir""",
]

UpperCAmelCase_   : List[str]   			= [
    # (pattern, replacement)
    # Order is important here for some replacements
    (r"""tfds\.core""", r"""datasets"""),
    (r"""tf\.io\.gfile\.GFile""", r"""open"""),
    (r"""tf\.([\w\d]+)""", r"""datasets.Value('\1')"""),
    (r"""tfds\.features\.Text\(\)""", r"""datasets.Value('string')"""),
    (r"""tfds\.features\.Text\(""", r"""datasets.Value('string'),"""),
    (r"""features\s*=\s*tfds.features.FeaturesDict\(""", r"""features=datasets.Features("""),
    (r"""tfds\.features\.FeaturesDict\(""", r"""dict("""),
    (r"""The TensorFlow Datasets Authors""", r"""The TensorFlow Datasets Authors and the HuggingFace Datasets Authors"""),
    (r"""tfds\.""", r"""datasets."""),
    (r"""dl_manager\.manual_dir""", r"""self.config.data_dir"""),
    (r"""self\.builder_config""", r"""self.config"""),
]



def 					_A			(__a							) ->       Optional[int]:
    """simple docstring"""






    return ConvertCommand(args.tfds_path    ,				args.datasets_directory							)





class 							lowerCAmelCase__				(			UpperCAmelCase__ ):






      '''simple docstring'''

      @staticmethod
      def _SCREAMING_SNAKE_CASE (						lowercase_     :  ArgumentParser):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		parser.add_parser(
              '''convert'''						,	help='''Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.'''						,	)
          train_parser.add_argument(
              '''--tfds_path'''						,	type=lowercase_						,	required=lowercase_						,	help='''Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.'''						,	)
          train_parser.add_argument(
              '''--datasets_directory'''						,	type=lowercase_						,	required=lowercase_						,	help='''Path to the HuggingFace Datasets folder.''')
          train_parser.set_defaults(func=lowercase_)
      def __init__(						self     :  Union[str, Any]						,	lowercase_     :  str						,	lowercase_     :  str						,	*lowercase_     :  Union[str, Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Dict    						=		get_logger('''datasets-cli/converting''')

          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		tfds_path
          SCREAMING_SNAKE_CASE_      :  Tuple    						=		datasets_directory



      def _SCREAMING_SNAKE_CASE (						self     :  str):




          '''simple docstring'''


          if os.path.isdir(self._tfds_path):
              SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		os.path.abspath(self._tfds_path)
          elif os.path.isfile(self._tfds_path):
              SCREAMING_SNAKE_CASE_      :  List[str]    						=		os.path.dirname(self._tfds_path)
          else:
              raise ValueError('''--tfds_path is neither a directory nor a file. Please check path.''')

          SCREAMING_SNAKE_CASE_      :  Any    						=		os.path.abspath(self._datasets_directory)

          self._logger.info(F'Converting datasets from {abs_tfds_path} to {abs_datasets_path}')

          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		[]
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		[]
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		{}

          if os.path.isdir(self._tfds_path):
              SCREAMING_SNAKE_CASE_      :  List[str]    						=		os.listdir(lowercase_)
          else:
              SCREAMING_SNAKE_CASE_      :  Any    						=		[os.path.basename(self._tfds_path)]

          for f_name in file_names:
              self._logger.info(F'Looking at file {f_name}')
              SCREAMING_SNAKE_CASE_      :  Tuple    						=		os.path.join(lowercase_						,	lowercase_)
              SCREAMING_SNAKE_CASE_      :  Any    						=		os.path.join(lowercase_						,	lowercase_)

              if not os.path.isfile(lowercase_) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
                  self._logger.info('''Skipping file''')
                  continue

              with open(lowercase_						,	encoding='''utf-8''') as f:
                  SCREAMING_SNAKE_CASE_      :  Any    						=		f.readlines()

              SCREAMING_SNAKE_CASE_      :  List[str]    						=		[]
              SCREAMING_SNAKE_CASE_      :  Dict    						=		False
              SCREAMING_SNAKE_CASE_      :  Any    						=		False
              SCREAMING_SNAKE_CASE_      :  List[str]    						=		[]
              for line in lines:
                  SCREAMING_SNAKE_CASE_      :  List[Any]    						=		line

                  # Convert imports
                  if "import tensorflow.compat.v2 as tf" in out_line:
                      continue
                  elif "@tfds.core" in out_line:
                      continue
                  elif "builder=self" in out_line:
                      continue
                  elif "import tensorflow_datasets.public_api as tfds" in out_line:
                      SCREAMING_SNAKE_CASE_      :  List[str]    						=		'''import datasets\n'''
                  elif "import tensorflow" in out_line:
                      # order is important here
                      SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		''''''
                      continue
                  elif "from absl import logging" in out_line:
                      SCREAMING_SNAKE_CASE_      :  str    						=		'''from datasets import logging\n'''
                  elif "getLogger" in out_line:
                      SCREAMING_SNAKE_CASE_      :  List[str]    						=		out_line.replace('''getLogger'''						,	'''get_logger''')
                  elif any(expression in out_line for expression in TO_HIGHLIGHT):
                      SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		True
                      SCREAMING_SNAKE_CASE_      :  Dict    						=		list(filter(lambda lowercase_: e in out_line						,	lowercase_))
                      out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(lowercase_) + '''\n''')
                      out_lines.append(lowercase_)
                      out_lines.append(lowercase_)
                      continue
                  else:
                      for pattern, replacement in TO_CONVERT:
                          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		re.sub(lowercase_						,	lowercase_						,	lowercase_)

                # Take care of saving utilities (to later move them together with main script)
                  if "tensorflow_datasets" in out_line:
                      SCREAMING_SNAKE_CASE_      :  Any    						=		re.match(r'''from\stensorflow_datasets.*import\s([^\.\r\n]+)'''						,	lowercase_)
                      tfds_imports.extend(imp.strip() for imp in match.group(1).split(''','''))
                      SCREAMING_SNAKE_CASE_      :  str    						=		'''from . import ''' + match.group(1)

                  # Check we have not forget anything
                  if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
                      raise ValueError(F'Error converting {out_line.strip()}')

                  if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
                      SCREAMING_SNAKE_CASE_      :  List[str]    						=		True
                  out_lines.append(lowercase_)

              if is_builder or "wmt" in f_name:
                  # We create a new directory for each dataset
                  SCREAMING_SNAKE_CASE_      :  List[Any]    						=		f_name.replace('''.py'''						,	'''''')
                  SCREAMING_SNAKE_CASE_      :  Any    						=		os.path.join(lowercase_						,	lowercase_)
                  SCREAMING_SNAKE_CASE_      :  str    						=		os.path.join(lowercase_						,	lowercase_)
                  os.makedirs(lowercase_						,	exist_ok=lowercase_)
                  self._logger.info(F'Adding directory {output_dir}')
                  imports_to_builder_map.update({imp: output_dir for imp in tfds_imports})
              else:
                  # Utilities will be moved at the end
                  utils_files.append(lowercase_)

              if needs_manual_update:
                  with_manual_update.append(lowercase_)

              with open(lowercase_						,	'''w'''						,	encoding='''utf-8''') as f:
                  f.writelines(lowercase_)
              self._logger.info(F'Converted in {output_file}')

          for utils_file in utils_files:
              try:
                  SCREAMING_SNAKE_CASE_      :  Tuple    						=		os.path.basename(lowercase_)
                  SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		imports_to_builder_map[f_name.replace('''.py'''						,	'''''')]
                  self._logger.info(F'Moving {dest_folder} to {utils_file}')
                  shutil.copy(lowercase_						,	lowercase_)
              except KeyError:
                  self._logger.error(F'Cannot find destination folder for {utils_file}. Please copy manually.')

          if with_manual_update:
              for file_path in with_manual_update:
                  self._logger.warning(
                      F'You need to manually update file {file_path} to remove configurations using \'TextEncoderConfig\'.')










"""simple docstring"""






import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


UpperCAmelCase_   : Optional[int]   			= logging.get_logger(__name__)

UpperCAmelCase_   : Dict   			= {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase_   : Dict   			= {
    """vocab_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/vocab.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/vocab.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/vocab.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/vocab.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/merges.txt""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/merges.txt""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/merges.txt""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/merges.txt""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/tokenizer.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/tokenizer.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/tokenizer.json""",
    },
}

UpperCAmelCase_   : List[str]   			= {
    """gpt2""": 1024,
    """gpt2-medium""": 1024,
    """gpt2-large""": 1024,
    """gpt2-xl""": 1024,
    """distilgpt2""": 1024,
}




class 							lowerCAmelCase__				(			UpperCAmelCase__ ):






      '''simple docstring'''

      __UpperCamelCase	 =  VOCAB_FILES_NAMES
      __UpperCamelCase	 =  PRETRAINED_VOCAB_FILES_MAP
      __UpperCamelCase	 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __UpperCamelCase	 =  ["input_ids", "attention_mask"]
      __UpperCamelCase	 =  GPTaTokenizer
      def __init__(						self     :  Optional[int]						,	lowercase_     :  int=None						,	lowercase_     :  List[str]=None						,	lowercase_     :  Union[str, Any]=None						,	lowercase_     :  Tuple="<|endoftext|>"						,	lowercase_     :  str="<|endoftext|>"						,	lowercase_     :  Dict="<|endoftext|>"						,	lowercase_     :  Tuple=False						,	**lowercase_     :  Optional[int]						,	):




          '''simple docstring'''


          super().__init__(
              lowercase_						,	lowercase_						,	tokenizer_file=lowercase_						,	unk_token=lowercase_						,	bos_token=lowercase_						,	eos_token=lowercase_						,	add_prefix_space=lowercase_						,	**lowercase_						,	)

          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		kwargs.pop('''add_bos_token'''						,	lowercase_)

          SCREAMING_SNAKE_CASE_      :  Dict    						=		json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
          if pre_tok_state.get('''add_prefix_space'''						,	lowercase_) != add_prefix_space:
              SCREAMING_SNAKE_CASE_      :  int    						=		getattr(lowercase_						,	pre_tok_state.pop('''type'''))
              SCREAMING_SNAKE_CASE_      :  str    						=		add_prefix_space
              SCREAMING_SNAKE_CASE_      :  Dict    						=		pre_tok_class(**lowercase_)

          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		add_prefix_space
      def _SCREAMING_SNAKE_CASE (						self     :  str						,	*lowercase_     :  List[Any]						,	**lowercase_     :  Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		kwargs.get('''is_split_into_words'''						,	lowercase_)
          assert self.add_prefix_space or not is_split_into_words, (
              F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
              "to use it with pretokenized inputs."
          )

          return super()._batch_encode_plus(*lowercase_						,	**lowercase_)
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]						,	*lowercase_     :  List[str]						,	**lowercase_     :  List[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		kwargs.get('''is_split_into_words'''						,	lowercase_)

          assert self.add_prefix_space or not is_split_into_words, (
              F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
              "to use it with pretokenized inputs."
          )

          return super()._encode_plus(*lowercase_						,	**lowercase_)
      def _SCREAMING_SNAKE_CASE (						self     :  int						,	lowercase_     :  str						,	lowercase_     :  Optional[str] = None):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		self._tokenizer.model.save(lowercase_						,	name=lowercase_)
          return tuple(lowercase_)



      def _SCREAMING_SNAKE_CASE (						self     :  int						,	lowercase_     :  "Conversation"):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Dict    						=		[]
          for is_user, text in conversation.iter_texts():
              input_ids.extend(self.encode(lowercase_						,	add_special_tokens=lowercase_) + [self.eos_token_id])

          if len(lowercase_) > self.model_max_length:
              SCREAMING_SNAKE_CASE_      :  Any    						=		input_ids[-self.model_max_length :]
          return input_ids


