



import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE__   			=   logging.getLogger(__name__)


@dataclass(frozen=UpperCAmelCase_   )
class   __lowerCAmelCase  :
					"""simple docstring"""






					A__					:		str
					A__					:		str
					A__					:		Optional[str]      							=     None
					A__					:		Optional[str]      							=     None
					A__					:		Optional[str]      							=     None







@dataclass(frozen=UpperCAmelCase_   )
class   __lowerCAmelCase  :
					"""simple docstring"""






					A__					:		List[int]
					A__					:		Optional[List[int]]      							=     None
					A__					:		Optional[List[int]]      							=     None
					A__					:		Optional[Union[int, float]]      							=     None
					A__					:		Optional[int]      							=     None


if is_torch_available():
							import torch
							from torch.utils.data import Dataset


							class   __lowerCAmelCase  (		UpperCAmelCase_   ):
												"""simple docstring"""






												A__					:		List[InputFeatures]



												def __init__(  self       :							Optional[Any]					,       _snake_case       :							str					,       _snake_case       :							PreTrainedTokenizer					,       _snake_case       :							str					,       _snake_case       :							Optional[int] = None					,       _snake_case       :							Tuple=False					,       _snake_case       :							bool = False					,       ):






															"""simple docstring"""


															A__	  =     hans_processors[task]()

															A__	  =     os.path.join(
															    _snake_case					,       'cached_{}_{}_{}_{}'.format(
															        'dev' if evaluate else 'train'					,       tokenizer.__class__.__name__					,       str(_snake_case       )					,       _snake_case					,       )					,       )
															A__	  =     processor.get_labels()
															if tokenizer.__class__ in (
															    RobertaTokenizer,
															    RobertaTokenizerFast,
															    XLMRobertaTokenizer,
															    BartTokenizer,
															    BartTokenizerFast,
															):
																		# HACK(label indices are swapped in RoBERTa pretrained model)
																		A__  ,			A__	  =     label_list[2], label_list[1]
															A__	  =     label_list

															# Make sure only the first process in distributed training processes the dataset,
															# and the others will use the cache.
															A__	  =     cached_features_file + '.lock'
															with FileLock(_snake_case       ):
																		if os.path.exists(_snake_case       ) and not overwrite_cache:
																					logger.info(F'''Loading features from cached file {cached_features_file}'''       )
																					A__	  =     torch.load(_snake_case       )
																		else:
																					logger.info(F'''Creating features from dataset file at {data_dir}'''       )

																					A__	  =     (
																					    processor.get_dev_examples(_snake_case       ) if evaluate else processor.get_train_examples(_snake_case       )
																					)

																					logger.info('Training examples: %s'					,       len(_snake_case       )       )
																					A__	  =     hans_convert_examples_to_features(_snake_case					,       _snake_case					,       _snake_case					,       _snake_case       )
																					logger.info('Saving features into cached file %s'					,       _snake_case       )
																					torch.save(self.features					,       _snake_case       )



												def __len__(  self       :							Dict       ):






															"""simple docstring"""


															return len(self.features       )



												def __getitem__(  self       :							Any					,       _snake_case       :							Dict       ):






															"""simple docstring"""


															return self.features[i]



												def      _a				(  self       :							str       ):






															"""simple docstring"""


															return self.label_list


if is_tf_available():
							import tensorflow as tf


							class   __lowerCAmelCase  :
												"""simple docstring"""






												A__					:		List[InputFeatures]



												def __init__(  self       :							Union[str, Any]					,       _snake_case       :							str					,       _snake_case       :							PreTrainedTokenizer					,       _snake_case       :							str					,       _snake_case       :							Optional[int] = 1_28					,       _snake_case       :							List[Any]=False					,       _snake_case       :							bool = False					,       ):






															"""simple docstring"""


															A__	  =     hans_processors[task]()
															A__	  =     processor.get_labels()
															if tokenizer.__class__ in (
															    RobertaTokenizer,
															    RobertaTokenizerFast,
															    XLMRobertaTokenizer,
															    BartTokenizer,
															    BartTokenizerFast,
															):
																		# HACK(label indices are swapped in RoBERTa pretrained model)
																		A__  ,			A__	  =     label_list[2], label_list[1]
															A__	  =     label_list

															A__	  =     processor.get_dev_examples(_snake_case       ) if evaluate else processor.get_train_examples(_snake_case       )
															A__	  =     hans_convert_examples_to_features(_snake_case					,       _snake_case					,       _snake_case					,       _snake_case       )

															def gen():
																		for ex_index, ex in tqdm.tqdm(enumerate(self.features       )					,       desc='convert examples to features'       ):
																					if ex_index % 1_00_00 == 0:
																								logger.info('Writing example %d of %d' % (ex_index, len(_snake_case       ))       )

																					yield (
																					    {
																					        "example_id": 0,
																					        "input_ids": ex.input_ids,
																					        "attention_mask": ex.attention_mask,
																					        "token_type_ids": ex.token_type_ids,
																					    },
																					    ex.label,
																					)

															A__	  =     tf.data.Dataset.from_generator(
															    _snake_case					,       (
															        {
															            'example_id': tf.intaa,
															            'input_ids': tf.intaa,
															            'attention_mask': tf.intaa,
															            'token_type_ids': tf.intaa,
															        },
															        tf.intaa,
															    )					,       (
															        {
															            'example_id': tf.TensorShape([]       ),
															            'input_ids': tf.TensorShape([None, None]       ),
															            'attention_mask': tf.TensorShape([None, None]       ),
															            'token_type_ids': tf.TensorShape([None, None]       ),
															        },
															        tf.TensorShape([]       ),
															    )					,       )



												def      _a				(  self       :							Tuple       ):






															"""simple docstring"""


															return self.dataset



												def __len__(  self       :							Optional[int]       ):






															"""simple docstring"""


															return len(self.features       )



												def __getitem__(  self       :							str					,       _snake_case       :							Optional[int]       ):






															"""simple docstring"""


															return self.features[i]



												def      _a				(  self       :							Any       ):






															"""simple docstring"""


															return self.label_list


class   __lowerCAmelCase  (		UpperCAmelCase_   ):
					"""simple docstring"""






					def      _a				(  self       :							List[str]					,       _snake_case       :							int       ):






								"""simple docstring"""


								return self._create_examples(self._read_tsv(os.path.join(_snake_case					,       'heuristics_train_set.txt'       )       )					,       'train'       )



					def      _a				(  self       :							List[Any]					,       _snake_case       :							Union[str, Any]       ):






								"""simple docstring"""


								return self._create_examples(self._read_tsv(os.path.join(_snake_case					,       'heuristics_evaluation_set.txt'       )       )					,       'dev'       )



					def      _a				(  self       :							Optional[Any]       ):






								"""simple docstring"""


								return ["contradiction", "entailment", "neutral"]



					def      _a				(  self       :							str					,       _snake_case       :							Tuple					,       _snake_case       :							Any       ):






								"""simple docstring"""


								A__	  =     []
								for i, line in enumerate(_snake_case       ):
											if i == 0:
														continue
											A__	  =     '%s-%s' % (set_type, line[0])
											A__	  =     line[5]
											A__	  =     line[6]
											A__	  =     line[7][2:] if line[7].startswith('ex'       ) else line[7]
											A__	  =     line[0]
											examples.append(InputExample(guid=_snake_case					,       text_a=_snake_case					,       text_b=_snake_case					,       label=_snake_case					,       pairID=_snake_case       )       )
								return examples







def A    (							__UpperCamelCase			,						__UpperCamelCase			,						__UpperCamelCase			,						__UpperCamelCase			,						)    ->   str:
			A__	  =     {label: i for i, label in enumerate(__UpperCamelCase       )}

			A__	  =     []
			for ex_index, example in tqdm.tqdm(enumerate(__UpperCamelCase       )			,						desc='convert examples to features'       ):
						if ex_index % 10_000 == 0:
									logger.info('Writing example %d' % (ex_index)       )

						A__	  =     tokenizer(
						    example.text_a			,						example.text_b			,						add_special_tokens=__UpperCamelCase			,						max_length=__UpperCamelCase			,						padding='max_length'			,						truncation=__UpperCamelCase			,						return_overflowing_tokens=__UpperCamelCase			,						)

						A__	  =     label_map[example.label] if example.label in label_map else 0

						A__	  =     int(example.pairID       )

						features.append(InputFeatures(**__UpperCamelCase			,						label=__UpperCamelCase			,						pairID=__UpperCamelCase       )       )

			for i, example in enumerate(examples[:5]       ):
						logger.info('*** Example ***'       )
						logger.info(f'''guid: {example}'''       )
						logger.info(f'''features: {features[i]}'''       )

			return features


SCREAMING_SNAKE_CASE__   			=   {
    '''hans''': 3,
}






SCREAMING_SNAKE_CASE__   			=   {
    '''hans''': HansProcessor,
}







"""simple docstring"""


import os
import unittest

from transformers.models.bartpho.tokenization_bartpho import VOCAB_FILES_NAMES, BartphoTokenizer
from transformers.testing_utils import get_tests_dir

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase													=get_tests_dir("fixtures/test_sentencepiece_bpe.model")





class lowerCamelCase__   (				SCREAMING_SNAKE_CASE     ,   unittest.TestCase    ):



    '''simple docstring'''




    _lowerCamelCase					=   BartphoTokenizer
    _lowerCamelCase					=   False
    _lowerCamelCase					=   True




    def UpperCamelCase__		( self    )    ->      str:
          super().setUp()

          A				  =		["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]
          A				  =		dict(zip(lowerCamelCase_ ,range(len(lowerCamelCase_    )    )    )    )
          A				  =		{"""unk_token""": """<unk>"""}

          A				  =		os.path.join(self.tmpdirname ,VOCAB_FILES_NAMES["""monolingual_vocab_file"""]    )
          with open(self.monolingual_vocab_file ,"""w""" ,encoding="""utf-8"""    ) as fp:
                for token in vocab_tokens:
                      fp.write(f'{token} {vocab_tokens[token]}\n'    )

          A				  =		BartphoTokenizer(lowerCamelCase_ ,self.monolingual_vocab_file ,**self.special_tokens_map    )
          tokenizer.save_pretrained(self.tmpdirname    )




    def UpperCamelCase__		( self ,**lowerCamelCase_    )    ->      Tuple:
          kwargs.update(self.special_tokens_map    )
          return BartphoTokenizer.from_pretrained(self.tmpdirname ,**lowerCamelCase_    )




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      Optional[Any]:
          A				  =		"""This is a là test"""
          A				  =		"""This is a<unk><unk> test"""
          return input_text, output_text




    def UpperCamelCase__		( self    )    ->      int:
          A				  =		BartphoTokenizer(lowerCamelCase_ ,self.monolingual_vocab_file ,**self.special_tokens_map    )
          A				  =		"""This is a là test"""
          A				  =		"""▁This ▁is ▁a ▁l à ▁t est""".split()
          A				  =		tokenizer.tokenize(lowerCamelCase_    )
          self.assertListEqual(lowerCamelCase_ ,lowerCamelCase_    )

          A				  =		tokens + [tokenizer.unk_token]
          A				  =		[4, 5, 6, 3, 3, 7, 8, 3]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_    ) ,lowerCamelCase_    )




