





'''simple docstring'''
import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


a__							:							Optional[int] ="<<<<<<< This should probably be modified because it mentions: "

a__							:							List[str] ="=======\n>>>>>>>\n"

a__							:							Tuple =[
    "TextEncoderConfig",
    "ByteTextEncoder",
    "SubwordTextEncoder",
    "encoder_config",
    "maybe_build_from_corpus",
    "manual_dir",
]

a__							:							Optional[int] =[
    # (pattern, replacement)
    # Order is important here for some replacements
    (R"tfds\.core", R"datasets"),
    (R"tf\.io\.gfile\.GFile", R"open"),
    (R"tf\.([\w\d]+)", R"datasets.Value('\1')"),
    (R"tfds\.features\.Text\(\)", R"datasets.Value('string')"),
    (R"tfds\.features\.Text\(", R"datasets.Value('string'),"),
    (R"features\s*=\s*tfds.features.FeaturesDict\(", R"features=datasets.Features("),
    (R"tfds\.features\.FeaturesDict\(", R"dict("),
    (R"The TensorFlow Datasets Authors", R"The TensorFlow Datasets Authors and the HuggingFace Datasets Authors"),
    (R"tfds\.", R"datasets."),
    (R"dl_manager\.manual_dir", R"self.config.data_dir"),
    (R"self\.builder_config", R"self.config"),
]
def 				lowercase__   (   __lowercase						:  List[str]     ) ->      Optional[Any]:



       """simple docstring"""






       return ConvertCommand(args.tfds_path				,  args.datasets_directory     )


class      snake_case     (    __lowerCamelCase       ):




 """simple docstring"""



 @staticmethod
 def        _lowerCamelCase					(       __A       :							ArgumentParser	):
        __UpperCamelCase       =  parser.add_parser(
            'convert'							,	help='Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.'							,	)
        train_parser.add_argument(
            '--tfds_path'							,	type=__A							,	required=__A							,	help='Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.'							,	)
        train_parser.add_argument(
            '--datasets_directory'							,	type=__A							,	required=__A							,	help='Path to the HuggingFace Datasets folder.'	)
        train_parser.set_defaults(func=__A	)






 def __init__(       self       :							Any							,	__A       :							str							,	__A       :							str							,	*__A       :							List[str]	):
        __UpperCamelCase       =  get_logger('datasets-cli/converting'	)

        __UpperCamelCase       =  tfds_path
        __UpperCamelCase       =  datasets_directory






 def        _lowerCamelCase					(       self       :							Dict	):
        if os.path.isdir(self._tfds_path	):
               __UpperCamelCase       =  os.path.abspath(self._tfds_path	)
        elif os.path.isfile(self._tfds_path	):
               __UpperCamelCase       =  os.path.dirname(self._tfds_path	)
        else:
               raise ValueError('--tfds_path is neither a directory nor a file. Please check path.'	)

        __UpperCamelCase       =  os.path.abspath(self._datasets_directory	)

        self._logger.info(f'''Converting datasets from {abs_tfds_path} to {abs_datasets_path}'''	)

        __UpperCamelCase       =  []
        __UpperCamelCase       =  []
        __UpperCamelCase       =  {}

        if os.path.isdir(self._tfds_path	):
               __UpperCamelCase       =  os.listdir(__A	)
        else:
               __UpperCamelCase       =  [os.path.basename(self._tfds_path	)]

        for f_name in file_names:
               self._logger.info(f'''Looking at file {f_name}'''	)
               __UpperCamelCase       =  os.path.join(__A							,	__A	)
               __UpperCamelCase       =  os.path.join(__A							,	__A	)

               if not os.path.isfile(__A	) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
                      self._logger.info('Skipping file'	)
                      continue

               with open(__A							,	encoding='utf-8'	) as f:
                      __UpperCamelCase       =  f.readlines()

               __UpperCamelCase       =  []
               __UpperCamelCase       =  False
               __UpperCamelCase       =  False
               __UpperCamelCase       =  []
               for line in lines:
                      __UpperCamelCase       =  line

                      # Convert imports
                      if "import tensorflow.compat.v2 as tf" in out_line:
                             continue
                      elif "@tfds.core" in out_line:
                             continue
                      elif "builder=self" in out_line:
                             continue
                      elif "import tensorflow_datasets.public_api as tfds" in out_line:
                             __UpperCamelCase       =  """import datasets\n"""
                      elif "import tensorflow" in out_line:
                             # order is important here
                             __UpperCamelCase       =  """"""
                             continue
                      elif "from absl import logging" in out_line:
                             __UpperCamelCase       =  """from datasets import logging\n"""
                      elif "getLogger" in out_line:
                             __UpperCamelCase       =  out_line.replace('getLogger'							,	'get_logger'	)
                      elif any(expression in out_line for expression in TO_HIGHLIGHT	):
                             __UpperCamelCase       =  True
                             __UpperCamelCase       =  list(filter(lambda __A	: e in out_line							,	__A	)	)
                             out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(__A	) + '\n'	)
                             out_lines.append(__A	)
                             out_lines.append(__A	)
                             continue
                      else:
                             for pattern, replacement in TO_CONVERT:
                                    __UpperCamelCase       =  re.sub(__A							,	__A							,	__A	)

                              # Take care of saving utilities (to later move them together with main script)
                      if "tensorflow_datasets" in out_line:
                             __UpperCamelCase       =  re.match(R'from\stensorflow_datasets.*import\s([^\.\r\n]+)'							,	__A	)
                             tfds_imports.extend(imp.strip() for imp in match.group(1	).split(','	)	)
                             __UpperCamelCase       =  """from . import """ + match.group(1	)

                      # Check we have not forget anything
                      if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
                             raise ValueError(f'''Error converting {out_line.strip()}'''	)

                      if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
                             __UpperCamelCase       =  True
                      out_lines.append(__A	)

               if is_builder or "wmt" in f_name:
                      # We create a new directory for each dataset
                      __UpperCamelCase       =  f_name.replace('.py'							,	''	)
                      __UpperCamelCase       =  os.path.join(__A							,	__A	)
                      __UpperCamelCase       =  os.path.join(__A							,	__A	)
                      os.makedirs(__A							,	exist_ok=__A	)
                      self._logger.info(f'''Adding directory {output_dir}'''	)
                      imports_to_builder_map.update({imp: output_dir for imp in tfds_imports}	)
               else:
                      # Utilities will be moved at the end
                      utils_files.append(__A	)

               if needs_manual_update:
                      with_manual_update.append(__A	)

               with open(__A							,	'w'							,	encoding='utf-8'	) as f:
                      f.writelines(__A	)
               self._logger.info(f'''Converted in {output_file}'''	)

        for utils_file in utils_files:
               try:
                      __UpperCamelCase       =  os.path.basename(__A	)
                      __UpperCamelCase       =  imports_to_builder_map[f_name.replace('.py'							,	''	)]
                      self._logger.info(f'''Moving {dest_folder} to {utils_file}'''	)
                      shutil.copy(__A							,	__A	)
               except KeyError:
                      self._logger.error(f'''Cannot find destination folder for {utils_file}. Please copy manually.'''	)

        if with_manual_update:
               for file_path in with_manual_update:
                      self._logger.warning(
                          f'''You need to manually update file {file_path} to remove configurations using \'TextEncoderConfig\'.'''	)







import argparse
import glob
import logging
import os
from argparse import Namespace
from importlib import import_module

import numpy as np
import torch
from lightning_base import BaseTransformer, add_generic_args, generic_train
from seqeval.metrics import accuracy_score, fa_score, precision_score, recall_score
from torch.nn import CrossEntropyLoss
from torch.utils.data import DataLoader, TensorDataset
from utils_ner import TokenClassificationTask


__UpperCAmelCase       :   Union[str, Any]				       =							logging.getLogger(__name__)







class     __snake_case       (							__lowerCamelCase			):





       '''simple docstring'''







       lowerCAmelCase__      		=     """token-classification"""




       def __init__(  self :    Union[str, Any]							,				A :    List[Any]		):
         if type(A		) == dict:
           __snake_case:						str   =						Namespace(**A		)
         __snake_case:						str   =						import_module("""tasks"""		)
         try:
           __snake_case:						Tuple   =						getattr(A							,				hparams.task_type		)
           __snake_case:						TokenClassificationTask   =						token_classification_task_clazz()
         except AttributeError:
           raise ValueError(
               f'''Task {hparams.task_type} needs to be defined as a TokenClassificationTask subclass in {module}. '''
               f'''Available tasks classes are: {TokenClassificationTask.__subclasses__()}'''		)
         __snake_case:						Optional[int]   =						self.token_classification_task.get_labels(hparams.labels		)
         __snake_case:						str   =						CrossEntropyLoss().ignore_index
         super().__init__(A							,				len(self.labels		)							,				self.mode		)




       def 							UpperCAmelCase__ (  self :    Union[str, Any]							,				**A :    Union[str, Any]		):
         return self.model(**A		)




       def 							UpperCAmelCase__ (  self :    Any							,				A :    Optional[int]							,				A :    str		):
         __snake_case:						str   =						{"""input_ids""": batch[0], """attention_mask""": batch[1], """labels""": batch[3]}
         if self.config.model_type != "distilbert":
           __snake_case:						List[str]   =						(
               batch[2] if self.config.model_type in ["""bert""", """xlnet"""] else None
           )  # XLM and RoBERTa don"t use token_type_ids

         __snake_case:						Any   =						self(**A		)
         __snake_case:						Any   =						outputs[0]
         # tensorboard_logs = {"loss": loss, "rate": self.lr_scheduler.get_last_lr()[-1]}
         return {"loss": loss}




       def 							UpperCAmelCase__ (  self :    List[str]		):
         __snake_case:						Tuple   =						self.hparams
         for mode in ["train", "dev", "test"]:
           __snake_case:						Union[str, Any]   =						self._feature_file(A		)
           if os.path.exists(A		) and not args.overwrite_cache:
             logger.info("""Loading features from cached file %s"""							,				A		)
             __snake_case:						Tuple   =						torch.load(A		)
           else:
             logger.info("""Creating features from dataset file at %s"""							,				args.data_dir		)
             __snake_case:						List[Any]   =						self.token_classification_task.read_examples_from_file(args.data_dir							,				A		)
             __snake_case:						Optional[int]   =						self.token_classification_task.convert_examples_to_features(
                 A							,				self.labels							,				args.max_seq_length							,				self.tokenizer							,				cls_token_at_end=bool(self.config.model_type in ["""xlnet"""]		)							,				cls_token=self.tokenizer.cls_token							,				cls_token_segment_id=2 if self.config.model_type in ["""xlnet"""] else 0							,				sep_token=self.tokenizer.sep_token							,				sep_token_extra=A							,				pad_on_left=bool(self.config.model_type in ["""xlnet"""]		)							,				pad_token=self.tokenizer.pad_token_id							,				pad_token_segment_id=self.tokenizer.pad_token_type_id							,				pad_token_label_id=self.pad_token_label_id							,				)
             logger.info("""Saving features into cached file %s"""							,				A		)
             torch.save(A							,				A		)




       def 							UpperCAmelCase__ (  self :    List[str]							,				A :    int							,				A :    int							,				A :    bool = False		):
         __snake_case:						List[str]   =						self._feature_file(A		)
         logger.info("""Loading features from cached file %s"""							,				A		)
         __snake_case:						int   =						torch.load(A		)
         __snake_case:						Optional[int]   =						torch.tensor([f.input_ids for f in features]							,				dtype=torch.long		)
         __snake_case:						Optional[Any]   =						torch.tensor([f.attention_mask for f in features]							,				dtype=torch.long		)
         if features[0].token_type_ids is not None:
           __snake_case:						Dict   =						torch.tensor([f.token_type_ids for f in features]							,				dtype=torch.long		)
         else:
           __snake_case:						Tuple   =						torch.tensor([0 for f in features]							,				dtype=torch.long		)
           # HACK(we will not use this anymore soon)
         __snake_case:						Optional[int]   =						torch.tensor([f.label_ids for f in features]							,				dtype=torch.long		)
         return DataLoader(
             TensorDataset(A							,				A							,				A							,				A		)							,				batch_size=A		)




       def 							UpperCAmelCase__ (  self :    Tuple							,				A :    Optional[Any]							,				A :    int		):
         """Compute validation""" ""
         __snake_case:						List[str]   =						{"""input_ids""": batch[0], """attention_mask""": batch[1], """labels""": batch[3]}
         if self.config.model_type != "distilbert":
           __snake_case:						Union[str, Any]   =						(
               batch[2] if self.config.model_type in ["""bert""", """xlnet"""] else None
           )  # XLM and RoBERTa don"t use token_type_ids
         __snake_case:						List[str]   =						self(**A		)
         __snake_case  ,   __snake_case:						int   =						outputs[:2]
         __snake_case:						List[str]   =						logits.detach().cpu().numpy()
         __snake_case:						Any   =						inputs["""labels"""].detach().cpu().numpy()
         return {"val_loss": tmp_eval_loss.detach().cpu(), "pred": preds, "target": out_label_ids}




       def 							UpperCAmelCase__ (  self :    Dict							,				A :    Dict		):
         __snake_case:						Union[str, Any]   =						torch.stack([x["""val_loss"""] for x in outputs]		).mean()
         __snake_case:						Tuple   =						np.concatenate([x["""pred"""] for x in outputs]							,				axis=0		)
         __snake_case:						List[str]   =						np.argmax(A							,				axis=2		)
         __snake_case:						Optional[Any]   =						np.concatenate([x["""target"""] for x in outputs]							,				axis=0		)

         __snake_case:						Tuple   =						dict(enumerate(self.labels		)		)
         __snake_case:						Dict   =						[[] for _ in range(out_label_ids.shape[0]		)]
         __snake_case:						Tuple   =						[[] for _ in range(out_label_ids.shape[0]		)]

         for i in range(out_label_ids.shape[0]		):
           for j in range(out_label_ids.shape[1]		):
             if out_label_ids[i, j] != self.pad_token_label_id:
               out_label_list[i].append(label_map[out_label_ids[i][j]]		)
               preds_list[i].append(label_map[preds[i][j]]		)

         __snake_case:						Dict   =						{
             """val_loss""": val_loss_mean,
             """accuracy_score""": accuracy_score(A							,				A		),
             """precision""": precision_score(A							,				A		),
             """recall""": recall_score(A							,				A		),
             """f1""": fa_score(A							,				A		),
         }

         __snake_case:						Dict   =						dict(results.items()		)
         __snake_case:						Dict   =						results
         return ret, preds_list, out_label_list




       def 							UpperCAmelCase__ (  self :    Tuple							,				A :    int		):
         # when stable
         __snake_case  ,   __snake_case  ,   __snake_case:						List[str]   =						self._eval_end(A		)
         __snake_case:						int   =						ret["""log"""]
         return {"val_loss": logs["val_loss"], "log": logs, "progress_bar": logs}




       def 							UpperCAmelCase__ (  self :    Dict							,				A :    Optional[int]		):
         # updating to test_epoch_end instead of deprecated test_end
         __snake_case  ,   __snake_case  ,   __snake_case:						int   =						self._eval_end(A		)

         # Converting to the dict required by pl
         # https://github.com/PyTorchLightning/pytorch-lightning/blob/master/\
         # pytorch_lightning/trainer/logging.py#L139
         __snake_case:						List[Any]   =						ret["""log"""]
         # `val_loss` is the key returned by `self._eval_end()` but actually refers to `test_loss`
         return {"avg_test_loss": logs["val_loss"], "log": logs, "progress_bar": logs}






       @staticmethod
       def 							UpperCAmelCase__ (  A :    Any							,				A :    int		):
         # Add NER specific options
         BaseTransformer.add_model_specific_args(A							,				A		)
         parser.add_argument(
             """--task_type"""							,				default="""NER"""							,				type=A							,				help="""Task type to fine tune in training (e.g. NER, POS, etc)"""		)
         parser.add_argument(
             """--max_seq_length"""							,				default=128							,				type=A							,				help=(
                 """The maximum total input sequence length after tokenization. Sequences longer """
                 """than this will be truncated, sequences shorter will be padded."""
             )							,				)

         parser.add_argument(
             """--labels"""							,				default=""""""							,				type=A							,				help="""Path to a file containing all labels. If not specified, CoNLL-2003 labels are used."""							,				)
         parser.add_argument(
             """--gpus"""							,				default=0							,				type=A							,				help="""The number of GPUs allocated for this, it is by default 0 meaning none"""							,				)

         parser.add_argument(
             """--overwrite_cache"""							,				action="""store_true"""							,				help="""Overwrite the cached training and evaluation sets"""		)

         return parser


if __name__ == "__main__":
      __UpperCAmelCase       :   Tuple				       =							argparse.ArgumentParser()
      add_generic_args(parser, os.getcwd())
      __UpperCAmelCase       :   Tuple				       =							NERTransformer.add_model_specific_args(parser, os.getcwd())
      __UpperCAmelCase       :   List[str]				       =							parser.parse_args()
      __UpperCAmelCase       :   Optional[int]				       =							NERTransformer(args)
      __UpperCAmelCase       :   Tuple				       =							generic_train(model, args)

      if args.do_predict:
            # See https://github.com/huggingface/transformers/issues/3159
            # pl use this default format to create a checkpoint:
            # https://github.com/PyTorchLightning/pytorch-lightning/blob/master\
            # /pytorch_lightning/callbacks/model_checkpoint.py#L322
            __UpperCAmelCase       :   int				       =							sorted(glob.glob(os.path.join(args.output_dir, "checkpoint-epoch=*.ckpt"), recursive=True))
            __UpperCAmelCase       :   Any				       =							model.load_from_checkpoint(checkpoints[-1])
            trainer.test(model)






