






"""simple docstring"""






import logging
import os
from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

from filelock import FileLock

from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available


UpperCAmelCase_   : Tuple   			= logging.getLogger(__name__)




@dataclass
class 							lowerCAmelCase__				:






      '''simple docstring'''

      __UpperCamelCase	 =  42
      __UpperCamelCase	 =  42
      __UpperCamelCase	 =  42




@dataclass
class 							lowerCAmelCase__				:






      '''simple docstring'''

      __UpperCamelCase	 =  42
      __UpperCamelCase	 =  42
      __UpperCamelCase	 =  None
      __UpperCamelCase	 =  None




class 							lowerCAmelCase__				(			UpperCAmelCase__ ):






      '''simple docstring'''

      __UpperCamelCase	 =  "train"
      __UpperCamelCase	 =  "dev"
      __UpperCamelCase	 =  "test"




class 							lowerCAmelCase__				:






      '''simple docstring'''

      @staticmethod
      def _SCREAMING_SNAKE_CASE (						lowercase_     :  Any						,	lowercase_     :  Union[Split, str]):




          '''simple docstring'''


          raise NotImplementedError
      @staticmethod
      def _SCREAMING_SNAKE_CASE (						lowercase_     :  str):




          '''simple docstring'''


          raise NotImplementedError



      @staticmethod
      def _SCREAMING_SNAKE_CASE (						lowercase_     :  List[InputExample]						,	lowercase_     :  List[str]						,	lowercase_     :  int						,	lowercase_     :  PreTrainedTokenizer						,	lowercase_     :  str=False						,	lowercase_     :  Dict="[CLS]"						,	lowercase_     :  List[Any]=1						,	lowercase_     :  Optional[int]="[SEP]"						,	lowercase_     :  Optional[Any]=False						,	lowercase_     :  Optional[Any]=False						,	lowercase_     :  str=0						,	lowercase_     :  Dict=0						,	lowercase_     :  Union[str, Any]=-100						,	lowercase_     :  List[str]=0						,	lowercase_     :  Any=True						,	):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Dict    						=		{label: i for i, label in enumerate(lowercase_)}

          SCREAMING_SNAKE_CASE_      :  int    						=		[]
          for ex_index, example in enumerate(lowercase_):
              if ex_index % 10000 == 0:
                  logger.info('''Writing example %d of %d'''						,	lowercase_						,	len(lowercase_))

              SCREAMING_SNAKE_CASE_      :  Tuple    						=		[]
              SCREAMING_SNAKE_CASE_      :  List[str]    						=		[]
              for word, label in zip(example.words						,	example.labels):
                  SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		tokenizer.tokenize(lowercase_)

                  # bert-base-multilingual-cased sometimes output "nothing ([]) when calling tokenize with just a space.
                  if len(lowercase_) > 0:
                      tokens.extend(lowercase_)
                      # Use the real label id for the first token of the word, and padding ids for the remaining tokens
                      label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(lowercase_) - 1))

            # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa.
              SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		tokenizer.num_special_tokens_to_add()
              if len(lowercase_) > max_seq_length - special_tokens_count:
                  SCREAMING_SNAKE_CASE_      :  Dict    						=		tokens[: (max_seq_length - special_tokens_count)]
                  SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		label_ids[: (max_seq_length - special_tokens_count)]

              # The convention in BERT is:
              # (a) For sequence pairs:
              #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
              #  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1
              # (b) For single sequences:
              #  tokens:   [CLS] the dog is hairy . [SEP]
              #  type_ids:   0   0   0   0  0     0   0
              #
              # Where "type_ids" are used to indicate whether this is the first
              # sequence or the second sequence. The embedding vectors for `type=0` and
              # `type=1` were learned during pre-training and are added to the wordpiece
              # embedding vector (and position vector). This is not *strictly* necessary
              # since the [SEP] token unambiguously separates the sequences, but it makes
              # it easier for the model to learn the concept of sequences.
              #
              # For classification tasks, the first vector (corresponding to [CLS]) is
              # used as the "sentence vector". Note that this only makes sense because
              # the entire model is fine-tuned.
              tokens += [sep_token]
              label_ids += [pad_token_label_id]
              if sep_token_extra:
                  # roberta uses an extra separator b/w pairs of sentences
                  tokens += [sep_token]
                  label_ids += [pad_token_label_id]
              SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		[sequence_a_segment_id] * len(lowercase_)

              if cls_token_at_end:
                  tokens += [cls_token]
                  label_ids += [pad_token_label_id]
                  segment_ids += [cls_token_segment_id]
              else:
                  SCREAMING_SNAKE_CASE_      :  int    						=		[cls_token] + tokens
                  SCREAMING_SNAKE_CASE_      :  Any    						=		[pad_token_label_id] + label_ids
                  SCREAMING_SNAKE_CASE_      :  Tuple    						=		[cls_token_segment_id] + segment_ids

              SCREAMING_SNAKE_CASE_      :  List[str]    						=		tokenizer.convert_tokens_to_ids(lowercase_)

              # The mask has 1 for real tokens and 0 for padding tokens. Only real
              # tokens are attended to.
              SCREAMING_SNAKE_CASE_      :  Dict    						=		[1 if mask_padding_with_zero else 0] * len(lowercase_)

              # Zero-pad up to the sequence length.
              SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		max_seq_length - len(lowercase_)
              if pad_on_left:
                  SCREAMING_SNAKE_CASE_      :  int    						=		([pad_token] * padding_length) + input_ids
                  SCREAMING_SNAKE_CASE_      :  int    						=		([0 if mask_padding_with_zero else 1] * padding_length) + input_mask
                  SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		([pad_token_segment_id] * padding_length) + segment_ids
                  SCREAMING_SNAKE_CASE_      :  str    						=		([pad_token_label_id] * padding_length) + label_ids
              else:
                  input_ids += [pad_token] * padding_length
                  input_mask += [0 if mask_padding_with_zero else 1] * padding_length
                  segment_ids += [pad_token_segment_id] * padding_length
                  label_ids += [pad_token_label_id] * padding_length

              assert len(lowercase_) == max_seq_length
              assert len(lowercase_) == max_seq_length
              assert len(lowercase_) == max_seq_length
              assert len(lowercase_) == max_seq_length

              if ex_index < 5:
                  logger.info('''*** Example ***''')
                  logger.info('''guid: %s'''						,	example.guid)
                  logger.info('''tokens: %s'''						,	''' '''.join([str(lowercase_) for x in tokens]))
                  logger.info('''input_ids: %s'''						,	''' '''.join([str(lowercase_) for x in input_ids]))
                  logger.info('''input_mask: %s'''						,	''' '''.join([str(lowercase_) for x in input_mask]))
                  logger.info('''segment_ids: %s'''						,	''' '''.join([str(lowercase_) for x in segment_ids]))
                  logger.info('''label_ids: %s'''						,	''' '''.join([str(lowercase_) for x in label_ids]))

              if "token_type_ids" not in tokenizer.model_input_names:
                  SCREAMING_SNAKE_CASE_      :  Dict    						=		None

              features.append(
                  InputFeatures(
                      input_ids=lowercase_						,	attention_mask=lowercase_						,	token_type_ids=lowercase_						,	label_ids=lowercase_))
          return features


if is_torch_available():
  import torch
  from torch import nn
  from torch.utils.data import Dataset




  class 							lowerCAmelCase__				(			UpperCAmelCase__ ):






        '''simple docstring'''

        __UpperCamelCase	 =  42
        __UpperCamelCase	 =  nn.CrossEntropyLoss().ignore_index
        def __init__(						self     :  List[str]						,	lowercase_     :  TokenClassificationTask						,	lowercase_     :  str						,	lowercase_     :  PreTrainedTokenizer						,	lowercase_     :  List[str]						,	lowercase_     :  str						,	lowercase_     :  Optional[int] = None						,	lowercase_     :  Tuple=False						,	lowercase_     :  Split = Split.train						,	):




            '''simple docstring'''


            SCREAMING_SNAKE_CASE_      :  str    						=		os.path.join(
                lowercase_						,	'''cached_{}_{}_{}'''.format(mode.value						,	tokenizer.__class__.__name__						,	str(lowercase_))						,	)

            # Make sure only the first process in distributed training processes the dataset,
            # and the others will use the cache.
            SCREAMING_SNAKE_CASE_      :  Tuple    						=		cached_features_file + '''.lock'''
            with FileLock(lowercase_):
                if os.path.exists(lowercase_) and not overwrite_cache:
                    logger.info(F'Loading features from cached file {cached_features_file}')
                    SCREAMING_SNAKE_CASE_      :  Tuple    						=		torch.load(lowercase_)
                else:
                    logger.info(F'Creating features from dataset file at {data_dir}')
                    SCREAMING_SNAKE_CASE_      :  List[Any]    						=		token_classification_task.read_examples_from_file(lowercase_						,	lowercase_)
                    # TODO clean up all this to leverage built-in features of tokenizers
                    SCREAMING_SNAKE_CASE_      :  Dict    						=		token_classification_task.convert_examples_to_features(
                        lowercase_						,	lowercase_						,	lowercase_						,	lowercase_						,	cls_token_at_end=bool(model_type in ['''xlnet'''])						,	cls_token=tokenizer.cls_token						,	cls_token_segment_id=2 if model_type in ['''xlnet'''] else 0						,	sep_token=tokenizer.sep_token						,	sep_token_extra=lowercase_						,	pad_on_left=bool(tokenizer.padding_side == '''left''')						,	pad_token=tokenizer.pad_token_id						,	pad_token_segment_id=tokenizer.pad_token_type_id						,	pad_token_label_id=self.pad_token_label_id						,	)
                    logger.info(F'Saving features into cached file {cached_features_file}')
                    torch.save(self.features						,	lowercase_)
        def __len__(						self     :  List[str]):




            '''simple docstring'''


            return len(self.features)



        def __getitem__(						self     :  List[Any]						,	lowercase_     :  str):




            '''simple docstring'''


            return self.features[i]


if is_tf_available():
  import tensorflow as tf




  class 							lowerCAmelCase__				:






        '''simple docstring'''

        __UpperCamelCase	 =  42
        __UpperCamelCase	 =  -1_0_0
        def __init__(						self     :  List[str]						,	lowercase_     :  TokenClassificationTask						,	lowercase_     :  str						,	lowercase_     :  PreTrainedTokenizer						,	lowercase_     :  List[str]						,	lowercase_     :  str						,	lowercase_     :  Optional[int] = None						,	lowercase_     :  str=False						,	lowercase_     :  Split = Split.train						,	):




            '''simple docstring'''


            SCREAMING_SNAKE_CASE_      :  List[str]    						=		token_classification_task.read_examples_from_file(lowercase_						,	lowercase_)
            # TODO clean up all this to leverage built-in features of tokenizers
            SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		token_classification_task.convert_examples_to_features(
                lowercase_						,	lowercase_						,	lowercase_						,	lowercase_						,	cls_token_at_end=bool(model_type in ['''xlnet'''])						,	cls_token=tokenizer.cls_token						,	cls_token_segment_id=2 if model_type in ['''xlnet'''] else 0						,	sep_token=tokenizer.sep_token						,	sep_token_extra=lowercase_						,	pad_on_left=bool(tokenizer.padding_side == '''left''')						,	pad_token=tokenizer.pad_token_id						,	pad_token_segment_id=tokenizer.pad_token_type_id						,	pad_token_label_id=self.pad_token_label_id						,	)

            def gen():
                for ex in self.features:
                    if ex.token_type_ids is None:
                        yield (
                            {"input_ids": ex.input_ids, "attention_mask": ex.attention_mask},
                            ex.label_ids,
                        )
                    else:
                        yield (
                            {
                                "input_ids": ex.input_ids,
                                "attention_mask": ex.attention_mask,
                                "token_type_ids": ex.token_type_ids,
                            },
                            ex.label_ids,
                        )

            if "token_type_ids" not in tokenizer.model_input_names:
                SCREAMING_SNAKE_CASE_      :  Tuple    						=		tf.data.Dataset.from_generator(
                    lowercase_						,	({'''input_ids''': tf.intaa, '''attention_mask''': tf.intaa}, tf.intaa)						,	(
                        {'''input_ids''': tf.TensorShape([None]), '''attention_mask''': tf.TensorShape([None])},
                        tf.TensorShape([None]),
                    )						,	)
            else:
                SCREAMING_SNAKE_CASE_      :  str    						=		tf.data.Dataset.from_generator(
                    lowercase_						,	({'''input_ids''': tf.intaa, '''attention_mask''': tf.intaa, '''token_type_ids''': tf.intaa}, tf.intaa)						,	(
                        {
                            '''input_ids''': tf.TensorShape([None]),
                            '''attention_mask''': tf.TensorShape([None]),
                            '''token_type_ids''': tf.TensorShape([None]),
                        },
                        tf.TensorShape([None]),
                    )						,	)
        def _SCREAMING_SNAKE_CASE (						self     :  List[str]):




            '''simple docstring'''


            SCREAMING_SNAKE_CASE_      :  Tuple    						=		self.dataset.apply(tf.data.experimental.assert_cardinality(len(self.features)))

            return self.dataset
        def __len__(						self     :  Optional[Any]):




            '''simple docstring'''


            return len(self.features)



        def __getitem__(						self     :  Tuple						,	lowercase_     :  str):




            '''simple docstring'''


            return self.features[i]










"""simple docstring"""






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_   : Dict   			= logging.get_logger(__name__)

UpperCAmelCase_   : Tuple   			= """▁"""

UpperCAmelCase_   : Optional[Any]   			= {"""vocab_file""": """sentencepiece.bpe.model"""}

UpperCAmelCase_   : str   			= {
    """vocab_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model""",
    }
}

UpperCAmelCase_   : str   			= {
    """facebook/xglm-564M""": 2048,
}




class 							lowerCAmelCase__				(			UpperCAmelCase__ ):






      '''simple docstring'''

      __UpperCamelCase	 =  VOCAB_FILES_NAMES
      __UpperCamelCase	 =  PRETRAINED_VOCAB_FILES_MAP
      __UpperCamelCase	 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __UpperCamelCase	 =  ["input_ids", "attention_mask"]
      def __init__(						self     :  List[Any]						,	lowercase_     :  str						,	lowercase_     :  Tuple="<s>"						,	lowercase_     :  Any="</s>"						,	lowercase_     :  Optional[int]="</s>"						,	lowercase_     :  List[Any]="<s>"						,	lowercase_     :  Union[str, Any]="<unk>"						,	lowercase_     :  Union[str, Any]="<pad>"						,	lowercase_     :  Optional[Dict[str, Any]] = None						,	**lowercase_     :  Tuple						,	):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Any    						=		{} if sp_model_kwargs is None else sp_model_kwargs

          # Compatibility with the original tokenizer
          SCREAMING_SNAKE_CASE_      :  List[str]    						=		7
          SCREAMING_SNAKE_CASE_      :  Tuple    						=		[F'<madeupword{i}>' for i in range(self.num_madeup_words)]

          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		kwargs.get('''additional_special_tokens'''						,	[])
          kwargs["additional_special_tokens"] += [
              word for word in madeup_words if word not in kwargs["additional_special_tokens"]
          ]

          super().__init__(
              bos_token=lowercase_						,	eos_token=lowercase_						,	unk_token=lowercase_						,	sep_token=lowercase_						,	cls_token=lowercase_						,	pad_token=lowercase_						,	sp_model_kwargs=self.sp_model_kwargs						,	**lowercase_						,	)

          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		spm.SentencePieceProcessor(**self.sp_model_kwargs)
          self.sp_model.Load(str(lowercase_))
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		vocab_file

          # Original fairseq vocab and spm vocab must be "aligned":
          # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
          # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
          # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
          # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

          # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		1

          # Mimic fairseq token-to-id alignment for the first 4 token
          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		{'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		len(self.sp_model)
          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		{F'<madeupword{i}>': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words)}
          self.fairseq_tokens_to_ids.update(lowercase_)

          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		{v: k for k, v in self.fairseq_tokens_to_ids.items()}
      def __getstate__(						self     :  Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Any    						=		self.__dict__.copy()
          SCREAMING_SNAKE_CASE_      :  str    						=		None
          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		self.sp_model.serialized_model_proto()
          return state
      def __setstate__(						self     :  Tuple						,	lowercase_     :  str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  int    						=		d

          # for backward compatibility
          if not hasattr(self						,	'''sp_model_kwargs'''):
              SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		{}

          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		spm.SentencePieceProcessor(**self.sp_model_kwargs)
          self.sp_model.LoadFromSerializedProto(self.sp_model_proto)
      def _SCREAMING_SNAKE_CASE (						self     :  Union[str, Any]						,	lowercase_     :  List[int]						,	lowercase_     :  Optional[List[int]] = None):




          '''simple docstring'''


          if token_ids_a is None:
              return [self.sep_token_id] + token_ids_a
          SCREAMING_SNAKE_CASE_      :  Dict    						=		[self.sep_token_id]
          return sep + token_ids_a + sep + sep + token_ids_a
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]						,	lowercase_     :  List[int]						,	lowercase_     :  Optional[List[int]] = None						,	lowercase_     :  bool = False):




          '''simple docstring'''


          if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=lowercase_						,	token_ids_a=lowercase_						,	already_has_special_tokens=lowercase_)

          if token_ids_a is None:
              return [1] + ([0] * len(lowercase_))
          return [1] + ([0] * len(lowercase_)) + [1, 1] + ([0] * len(lowercase_))
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[Any]						,	lowercase_     :  List[int]						,	lowercase_     :  Optional[List[int]] = None):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		[self.sep_token_id]

          if token_ids_a is None:
              return len(sep + token_ids_a) * [0]
          return len(sep + token_ids_a + sep + sep + token_ids_a) * [0]
      @property
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[Any]):




          '''simple docstring'''


          return len(self.sp_model) + self.fairseq_offset + self.num_madeup_words
      def _SCREAMING_SNAKE_CASE (						self     :  Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		{self.convert_ids_to_tokens(lowercase_): i for i in range(self.vocab_size)}
          vocab.update(self.added_tokens_encoder)
          return vocab
      def _SCREAMING_SNAKE_CASE (						self     :  Dict						,	lowercase_     :  str):




          '''simple docstring'''


          return self.sp_model.encode(lowercase_						,	out_type=lowercase_)
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[Any]						,	lowercase_     :  Union[str, Any]):




          '''simple docstring'''


          if token in self.fairseq_tokens_to_ids:
              return self.fairseq_tokens_to_ids[token]
          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		self.sp_model.PieceToId(lowercase_)

          # Need to return unknown token if the SP model returned 0
          return spm_id + self.fairseq_offset if spm_id else self.unk_token_id
      def _SCREAMING_SNAKE_CASE (						self     :  int						,	lowercase_     :  Optional[Any]):




          '''simple docstring'''


          if index in self.fairseq_ids_to_tokens:
              return self.fairseq_ids_to_tokens[index]
          return self.sp_model.IdToPiece(index - self.fairseq_offset)
      def _SCREAMING_SNAKE_CASE (						self     :  int						,	lowercase_     :  Tuple):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		''''''.join(lowercase_).replace(lowercase_						,	''' ''').strip()
          return out_string



      def _SCREAMING_SNAKE_CASE (						self     :  str						,	lowercase_     :  str						,	lowercase_     :  Optional[str] = None):




          '''simple docstring'''


          if not os.path.isdir(lowercase_):
              logger.error(F'Vocabulary path ({save_directory}) should be a directory')
              return
          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		os.path.join(
              lowercase_						,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

          if os.path.abspath(self.vocab_file) != os.path.abspath(lowercase_) and os.path.isfile(self.vocab_file):
              copyfile(self.vocab_file						,	lowercase_)
          elif not os.path.isfile(self.vocab_file):
              with open(lowercase_						,	'''wb''') as fi:
                  SCREAMING_SNAKE_CASE_      :  int    						=		self.sp_model.serialized_model_proto()
                  fi.write(lowercase_)

          return (out_vocab_file,)


