


'''simple docstring'''




import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


__snake_case    						=logging.getLogger(__name__)

@dataclass(frozen=__lowercase		)
class 		UpperCAmelCase_      :
     lowerCamelCase				:	str
     lowerCamelCase				:	str
     lowerCamelCase				:	Optional[str]		=     None
     lowerCamelCase				:	Optional[str]		=     None
     lowerCamelCase				:	Optional[str]		=     None




@dataclass(frozen=__lowercase		)
class 		UpperCAmelCase_      :
     lowerCamelCase				:	List[int]
     lowerCamelCase				:	Optional[List[int]]		=     None
     lowerCamelCase				:	Optional[List[int]]		=     None
     lowerCamelCase				:	Optional[Union[int, float]]		=     None
     lowerCamelCase				:	Optional[int]		=     None


if is_torch_available():
       import torch
       from torch.utils.data import Dataset

       class 		UpperCAmelCase_      (  __lowercase		):
            lowerCamelCase				:	List[InputFeatures]





            def __init__( self       :      Union[str, Any]    ,      UpperCAmelCase__       :      str    ,      UpperCAmelCase__       :      PreTrainedTokenizer    ,      UpperCAmelCase__       :      str    ,      UpperCAmelCase__       :      Optional[int] = None    ,      UpperCAmelCase__       :      Dict=False    ,      UpperCAmelCase__       :      bool = False    ,      )				->  int:
               lowerCAmelCase     =			hans_processors[task]()

               lowerCAmelCase     =			os.path.join(
                   UpperCAmelCase__    ,      'cached_{}_{}_{}_{}'.format(
                       'dev' if evaluate else 'train'    ,      tokenizer.__class__.__name__    ,      str(UpperCAmelCase__    )    ,      UpperCAmelCase__    ,      )    ,      )
               lowerCAmelCase     =			processor.get_labels()
               if tokenizer.__class__ in (
                   RobertaTokenizer,
                   RobertaTokenizerFast,
                   XLMRobertaTokenizer,
                   BartTokenizer,
                   BartTokenizerFast,
               ):
                  # HACK(label indices are swapped in RoBERTa pretrained model)
                  lowerCAmelCase     ,  lowerCAmelCase     =			label_list[2], label_list[1]
               lowerCAmelCase     =			label_list

               # Make sure only the first process in distributed training processes the dataset,
               # and the others will use the cache.
               lowerCAmelCase     =			cached_features_file + '.lock'
               with FileLock(UpperCAmelCase__    ):
                  if os.path.exists(UpperCAmelCase__    ) and not overwrite_cache:
                     logger.info(F'''Loading features from cached file {cached_features_file}'''    )
                     lowerCAmelCase     =			torch.load(UpperCAmelCase__    )
                  else:
                     logger.info(F'''Creating features from dataset file at {data_dir}'''    )

                     lowerCAmelCase     =			(
                         processor.get_dev_examples(UpperCAmelCase__    ) if evaluate else processor.get_train_examples(UpperCAmelCase__    )
                     )

                     logger.info('Training examples: %s'    ,      len(UpperCAmelCase__    )    )
                     lowerCAmelCase     =			hans_convert_examples_to_features(UpperCAmelCase__    ,      UpperCAmelCase__    ,      UpperCAmelCase__    ,      UpperCAmelCase__    )
                     logger.info('Saving features into cached file %s'    ,      UpperCAmelCase__    )
                     torch.save(self.features    ,      UpperCAmelCase__    )





            def __len__( self       :      List[str]    )				->  Optional[int]:
               return len(self.features    )





            def __getitem__( self       :      Optional[int]    ,      UpperCAmelCase__       :      Optional[int]    )				->  InputFeatures:
               return self.features[i]





            def 							__UpperCAmelCase      ( self       :      Any    )				->  Any:
               return self.label_list


if is_tf_available():
       import tensorflow as tf

       class 		UpperCAmelCase_      :
            lowerCamelCase				:	List[InputFeatures]





            def __init__( self       :      str    ,      UpperCAmelCase__       :      str    ,      UpperCAmelCase__       :      PreTrainedTokenizer    ,      UpperCAmelCase__       :      str    ,      UpperCAmelCase__       :      Optional[int] = 1_2_8    ,      UpperCAmelCase__       :      str=False    ,      UpperCAmelCase__       :      bool = False    ,      )				->  int:
               lowerCAmelCase     =			hans_processors[task]()
               lowerCAmelCase     =			processor.get_labels()
               if tokenizer.__class__ in (
                   RobertaTokenizer,
                   RobertaTokenizerFast,
                   XLMRobertaTokenizer,
                   BartTokenizer,
                   BartTokenizerFast,
               ):
                  # HACK(label indices are swapped in RoBERTa pretrained model)
                  lowerCAmelCase     ,  lowerCAmelCase     =			label_list[2], label_list[1]
               lowerCAmelCase     =			label_list

               lowerCAmelCase     =			processor.get_dev_examples(UpperCAmelCase__    ) if evaluate else processor.get_train_examples(UpperCAmelCase__    )
               lowerCAmelCase     =			hans_convert_examples_to_features(UpperCAmelCase__    ,      UpperCAmelCase__    ,      UpperCAmelCase__    ,      UpperCAmelCase__    )

               def gen():
                  for ex_index, ex in tqdm.tqdm(enumerate(self.features    )    ,      desc='convert examples to features'    ):
                     if ex_index % 1_0_0_0_0 == 0:
                        logger.info('Writing example %d of %d' % (ex_index, len(UpperCAmelCase__    ))    )

                     yield (
                         {
                             "example_id": 0,
                             "input_ids": ex.input_ids,
                             "attention_mask": ex.attention_mask,
                             "token_type_ids": ex.token_type_ids,
                         },
                         ex.label,
                     )

               lowerCAmelCase     =			tf.data.Dataset.from_generator(
                   UpperCAmelCase__    ,      (
                       {
                           'example_id': tf.intaa,
                           'input_ids': tf.intaa,
                           'attention_mask': tf.intaa,
                           'token_type_ids': tf.intaa,
                       },
                       tf.intaa,
                   )    ,      (
                       {
                           'example_id': tf.TensorShape([]    ),
                           'input_ids': tf.TensorShape([None, None]    ),
                           'attention_mask': tf.TensorShape([None, None]    ),
                           'token_type_ids': tf.TensorShape([None, None]    ),
                       },
                       tf.TensorShape([]    ),
                   )    ,      )





            def 							__UpperCAmelCase      ( self       :      Dict    )				->  Any:
               return self.dataset





            def __len__( self       :      List[Any]    )				->  List[str]:
               return len(self.features    )





            def __getitem__( self       :      int    ,      UpperCAmelCase__       :      int    )				->  InputFeatures:
               return self.features[i]





            def 							__UpperCAmelCase      ( self       :      Tuple    )				->  int:
               return self.label_list

class 		UpperCAmelCase_      (  __lowercase		):





     def 							__UpperCAmelCase      ( self       :      Optional[int]    ,      UpperCAmelCase__       :      Dict    )				->  Any:
        return self._create_examples(self._read_tsv(os.path.join(UpperCAmelCase__    ,      'heuristics_train_set.txt'    )    )    ,      'train'    )





     def 							__UpperCAmelCase      ( self       :      List[Any]    ,      UpperCAmelCase__       :      List[Any]    )				->  List[Any]:
        return self._create_examples(self._read_tsv(os.path.join(UpperCAmelCase__    ,      'heuristics_evaluation_set.txt'    )    )    ,      'dev'    )





     def 							__UpperCAmelCase      ( self       :      List[str]    )				->  Any:
        return ["contradiction", "entailment", "neutral"]





     def 							__UpperCAmelCase      ( self       :      Dict    ,      UpperCAmelCase__       :      List[Any]    ,      UpperCAmelCase__       :      Optional[int]    )				->  Tuple:
        lowerCAmelCase     =			[]
        for i, line in enumerate(UpperCAmelCase__    ):
           if i == 0:
              continue
           lowerCAmelCase     =			'%s-%s' % (set_type, line[0])
           lowerCAmelCase     =			line[5]
           lowerCAmelCase     =			line[6]
           lowerCAmelCase     =			line[7][2:] if line[7].startswith('ex'    ) else line[7]
           lowerCAmelCase     =			line[0]
           examples.append(InputExample(guid=UpperCAmelCase__    ,      text_a=UpperCAmelCase__    ,      text_b=UpperCAmelCase__    ,      label=UpperCAmelCase__    ,      pairID=UpperCAmelCase__    )    )
        return examples





def 	a_					( lowerCamelCase	:     List[InputExample]	, lowerCamelCase	:     List[str]	, lowerCamelCase	:     int	, lowerCamelCase	:     PreTrainedTokenizer	, ):
   lowerCAmelCase     =			{label: i for i, label in enumerate(lowerCamelCase				)}

   lowerCAmelCase     =			[]
   for ex_index, example in tqdm.tqdm(enumerate(lowerCamelCase				)	, desc='convert examples to features'				):
      if ex_index % 10000 == 0:
         logger.info('Writing example %d' % (ex_index)				)

      lowerCAmelCase     =			tokenizer(
          example.text_a	, example.text_b	, add_special_tokens=lowerCamelCase	, max_length=lowerCamelCase	, padding='max_length'	, truncation=lowerCamelCase	, return_overflowing_tokens=lowerCamelCase	, )

      lowerCAmelCase     =			label_map[example.label] if example.label in label_map else 0

      lowerCAmelCase     =			int(example.pairID				)

      features.append(InputFeatures(**lowerCamelCase	, label=lowerCamelCase	, pairID=lowerCamelCase				)				)

   for i, example in enumerate(examples[:5]				):
      logger.info('*** Example ***'				)
      logger.info(f'''guid: {example}'''				)
      logger.info(f'''features: {features[i]}'''				)

   return features


__snake_case    						={
    """hans""": 3,
}




__snake_case    						={
    """hans""": HansProcessor,
}








'''simple docstring'''






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__											=	logging.get_logger(__name__)

lowerCAmelCase__											=	{'''vocab_file''': '''spiece.model'''}

lowerCAmelCase__											=	{
    '''vocab_file''': {
        '''bert_for_seq_generation''': (
            '''https://huggingface.co/google/bert_for_seq_generation_L-24_bbc_encoder/resolve/main/spiece.model'''
        ),
    }
}

lowerCAmelCase__											=	{'''bert_for_seq_generation''': 512}






class 			lowercase_	(lowerCamelCase__      ):




  """simple docstring"""




  SCREAMING_SNAKE_CASE    :  List[Any]							=			VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE    :  Dict							=			PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE    :  Optional[int]							=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE    :  List[int]							=			[]
  SCREAMING_SNAKE_CASE    :  Any							=			['input_ids', 'attention_mask']






  def __init__(      self   :     Optional[Any]					,lowercase__   :     Tuple					,lowercase__   :     Tuple="<s>"					,lowercase__   :     Union[str, Any]="</s>"					,lowercase__   :     str="<unk>"					,lowercase__   :     Tuple="<pad>"					,lowercase__   :     Union[str, Any]="<::::>"					,lowercase__   :     Optional[Dict[str, Any]] = None					,**lowercase__   :     Any					,):
   __lowercase							  = {} if sp_model_kwargs is None else sp_model_kwargs

   # Add extra_ids to the special token list
   super().__init__(
       bos_token=lowercase__					,eos_token=lowercase__					,unk_token=lowercase__					,pad_token=lowercase__					,sep_token=lowercase__					,sp_model_kwargs=self.sp_model_kwargs					,**lowercase__					,)

   __lowercase							  = vocab_file

   __lowercase							  = spm.SentencePieceProcessor(**self.sp_model_kwargs			)
   self.sp_model.Load(lowercase__			)






  @property
  def 					SCREAMING_SNAKE_CASE      (      self   :     Optional[int]			):
   return self.sp_model.get_piece_size()






  def 					SCREAMING_SNAKE_CASE      (      self   :     Union[str, Any]			):
   __lowercase							  = {self.convert_ids_to_tokens(lowercase__			): i for i in range(self.vocab_size			)}
   vocab.update(self.added_tokens_encoder			)
   return vocab






  def __getstate__(      self   :     str			):
   __lowercase							  = self.__dict__.copy()
   __lowercase							  = None
   return state






  def __setstate__(      self   :     Optional[int]					,lowercase__   :     Optional[Any]			):
   __lowercase							  = d

   # for backward compatibility
   if not hasattr(self					,'''sp_model_kwargs'''			):
    __lowercase							  = {}

   __lowercase							  = spm.SentencePieceProcessor(**self.sp_model_kwargs			)
   self.sp_model.Load(self.vocab_file			)






  def 					SCREAMING_SNAKE_CASE      (      self   :     str					,lowercase__   :     str			):
   return self.sp_model.encode(lowercase__					,out_type=lowercase__			)






  def 					SCREAMING_SNAKE_CASE      (      self   :     Tuple					,lowercase__   :     Union[str, Any]			):
   return self.sp_model.piece_to_id(lowercase__			)






  def 					SCREAMING_SNAKE_CASE      (      self   :     Any					,lowercase__   :     Tuple			):
   __lowercase							  = self.sp_model.IdToPiece(lowercase__			)
   return token






  def 					SCREAMING_SNAKE_CASE      (      self   :     Optional[int]					,lowercase__   :     int			):
   __lowercase							  = []
   __lowercase							  = ''''''
   for token in tokens:
    # make sure that special tokens are not decoded using sentencepiece model
    if token in self.all_special_tokens:
     out_string += self.sp_model.decode(lowercase__			) + token
     __lowercase							  = []
    else:
     current_sub_tokens.append(lowercase__			)
   out_string += self.sp_model.decode(lowercase__			)
   return out_string.strip()






  def 					SCREAMING_SNAKE_CASE      (      self   :     int					,lowercase__   :     str					,lowercase__   :     Optional[str] = None			):
   if not os.path.isdir(lowercase__			):
    logger.error(F"Vocabulary path ({save_directory}) should be a directory"			)
    return
   __lowercase							  = os.path.join(
       lowercase__					,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)

   if os.path.abspath(self.vocab_file			) != os.path.abspath(lowercase__			) and os.path.isfile(self.vocab_file			):
    copyfile(self.vocab_file					,lowercase__			)
   elif not os.path.isfile(self.vocab_file			):
    with open(lowercase__					,'''wb'''			) as fi:
     __lowercase							  = self.sp_model.serialized_model_proto()
     fi.write(lowercase__			)

   return (out_vocab_file,)

