
'''simple docstring'''
import json
import os
from datetime import date
from pathlib import Path

from tabulate import DataRow, TableFormat, tabulate


lowercase__						       =      TableFormat(
    lineabove=None,
    linebelowheader=None,
    linebetweenrows=None,
    linebelow=None,
    headerrow=DataRow("", "|", "|"),
    datarow=DataRow("", "|", "|"),
    padding=1,
    with_header_hide=None,
)


lowercase__						       =      []
lowercase__						       =      []

lowercase__						       =      {"type": "section", "text": {"type": "plain_text", "text": "No failed tests! 🤗", "emoji": True}}

lowercase__						       =      [
    {
        "type": "header",
        "text": {
            "type": "plain_text",
            "text": f"🤗 Accelerate nightly {os.environ.get('TEST_TYPE', '')} test results",
            "emoji": True,
        },
    }
]

lowercase__						       =      0
for log in Path().glob("*.log"):
						lowercase__						       =      0
						with open(log, "r") as f:
												for line in f:
																		lowercase__						       =      json.loads(line)
																		if line.get("nodeid", "") != "":
																								lowercase__						       =      line["nodeid"]
																								if line.get("duration", None) is not None:
																														lowercase__						       =      f"{line['duration']:.4f}"
																														if line.get("outcome", "") == "failed":
																																				section_num_failed += 1
																																				failed.append([test, duration, log.name.split("_")[0]])
																																				total_num_failed += 1
						group_info.append([str(log), section_num_failed, failed])
						lowercase__						       =      []
						log.unlink()

lowercase__						       =      ""
lowercase__						       =      []
if total_num_failed > 0:
						for name, num_failed, failed_tests in group_info:
												if num_failed > 0:
																		if num_failed == 1:
																								message += f"*{name[1:]}: {num_failed} failed test*\n"
																		else:
																								message += f"*{name[1:]}: {num_failed} failed tests*\n"
																		lowercase__						       =      []
																		lowercase__						       =      {}
																		for test in failed_tests:
																								lowercase__						       =      test[0].split("::")
																								lowercase__						       =      data[0].split("/")[-1]
																								if data[0] not in filesafailed:
																														lowercase__						       =      [data[1:]]
																								else:
																														filesafailed[data[0]] += [data[1:]]
																								failed_table.append(data)

																		lowercase__						       =      [test[0] for test in failed_table]
																		lowercase__						       =      list(set(files))
																		# Count number of instances in failed_tests
																		lowercase__						       =      []
																		for file in individual_files:
																								table.append([file, len(filesafailed[file])])

																		lowercase__						       =      tabulate(
																		    table,
																		    headers=["Test Location", "Num Failed"],
																		    tablefmt=hf_table_format,
																		    stralign="right",
																		)
																		message += f"\n```\n{failed_table}\n```"
																		all_filesafailed.append(filesafailed)
						if len(message) > 3_0_0_0:
												lowercase__						       =      "Too many failed tests, please see the full report in the Action results."
												lowercase__						       =      len(err) + 1_0
												lowercase__						       =      message[: 3_0_0_0 - offset] + f"\n...\n```\n{err}"
						print(f"### {message}")
else:
						lowercase__						       =      "No failed tests! 🤗"
						print(f"## {message}")
						payload.append(no_error_payload)

if os.environ.get("TEST_TYPE", "") != "":
						from slack_sdk import WebClient

						lowercase__						       =      WebClient(token=os.environ["SLACK_API_TOKEN"])
						if message != "No failed tests! 🤗":
												lowercase__						       =      {
												    "type": "section",
												    "text": {
												        "type": "mrkdwn",
												        "text": message,
												    },
												}
												payload.append(md_report)
												lowercase__						       =      {
												    "type": "section",
												    "text": {
												        "type": "mrkdwn",
												        "text": "*For more details:*",
												    },
												    "accessory": {
												        "type": "button",
												        "text": {
												            "type": "plain_text",
												            "text": "Check Action results",
												            "emoji": True,
												        },
												        "url": f"https://github.com/{os.environ['GITHUB_REPOSITORY']}/actions/runs/{os.environ['GITHUB_RUN_ID']}",
												    },
												}
												payload.append(action_button)
												lowercase__						       =      {
												    "type": "context",
												    "elements": [
												        {
												            "type": "plain_text",
												            "text": f"Nightly {os.environ.get('TEST_TYPE')} test results for {date.today()}",
												        }
												    ],
												}
												payload.append(date_report)
						lowercase__						       =      client.chat_postMessage(channel="#accelerate-ci-daily", text=message, blocks=payload)
						lowercase__						       =      response.data["ts"]
						for failed_file in all_filesafailed:
												for test_location, test_failures in failed_file.items():
																		# Keep only the first instance of the test name
																		lowercase__						       =      ""
																		for i, row in enumerate(test_failures):
																								if row[0] != test_class:
																														lowercase__						       =      row[0]
																								else:
																														lowercase__						       =      ""

																		lowercase__						       =      {
																		    "type": "section",
																		    "text": {
																		        "type": "mrkdwn",
																		        "text": f"Test location: {test_location}\n```\n{tabulate(test_failures, headers=['Class', 'Test'], tablefmt=hf_table_format, stralign='right')}\n```",
																		    },
																		}

																		client.chat_postMessage(
																		    channel="#accelerate-ci-daily",
																		    thread_ts=ts,
																		    blocks=[payload],
																		)




'''simple docstring'''
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
						from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
						lowercase__						       =      get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
						from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowercase__						       =      1_2_8_0_2_2
lowercase__						       =      1_2_8_0_2_8






@require_sentencepiece
class 	snake_case__       (      __SCREAMING_SNAKE_CASE    ,					unittest.TestCase					):







		"""simple docstring"""

		lowerCamelCase     							=						MaMaaaTokenizer
		lowerCamelCase     							=						False
		lowerCamelCase     							=						False
		lowerCamelCase     							=						True



		def       lowerCAmelCase				(      self				:			Dict      )						->			Optional[Any]:






				"""simple docstring"""







				super().setUp()

				snake_case  :       int     			=						['''</s>''', '''<unk>''', '''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''', '''\u0120''', '''<pad>''']
				snake_case  :       int     			=						dict(zip(UpperCamelCase__  ,	range(len(UpperCamelCase__      )      )      )      )
				snake_case  :       Optional[int]     			=						Path(self.tmpdirname      )
				save_json(UpperCamelCase__  ,	save_dir / VOCAB_FILES_NAMES['''vocab_file''']      )
				if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
						copyfile(UpperCamelCase__  ,	save_dir / VOCAB_FILES_NAMES['''spm_file''']      )

				snake_case  :       Optional[int]     			=						MaMaaaTokenizer.from_pretrained(self.tmpdirname      )
				tokenizer.save_pretrained(self.tmpdirname      )



		def       lowerCAmelCase				(      self				:			Dict  ,	**UpperCamelCase__				:			Optional[int]      )						->			Any:






				"""simple docstring"""







				return MaMaaaTokenizer.from_pretrained(self.tmpdirname  ,	**UpperCamelCase__      )



		def       lowerCAmelCase				(      self				:			Tuple  ,	UpperCamelCase__				:			Tuple      )						->			str:






				"""simple docstring"""







				return (
				    "This is a test",
				    "This is a test",
				)



		def       lowerCAmelCase				(      self				:			List[str]      )						->			Optional[Any]:






				"""simple docstring"""







				snake_case  :       Optional[int]     			=						'''</s>'''
				snake_case  :       str     			=						0

				self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase__      )  ,	UpperCamelCase__      )
				self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase__      )  ,	UpperCamelCase__      )



		def       lowerCAmelCase				(      self				:			Dict      )						->			int:






				"""simple docstring"""







				snake_case  :       Tuple     			=						self.get_tokenizer()
				snake_case  :       Dict     			=						list(tokenizer.get_vocab().keys()      )

				self.assertEqual(vocab_keys[0]  ,	'''</s>'''      )
				self.assertEqual(vocab_keys[1]  ,	'''<unk>'''      )
				self.assertEqual(vocab_keys[-1]  ,	'''<s>'''      )
				self.assertEqual(len(UpperCamelCase__      )  ,	tokenizer.vocab_size + len(tokenizer.get_added_vocab()      )      )



		@unittest.skip('''Skip this test while all models are still to be uploaded.'''      )
		def       lowerCAmelCase				(      self				:			str      )						->			List[str]:






				"""simple docstring"""







				pass



		def       lowerCAmelCase				(      self				:			int      )						->			Union[str, Any]:






				"""simple docstring"""







				snake_case  :       Optional[Any]     			=						self.get_tokenizer()

				snake_case  :       Optional[Any]     			=						tokenizer.tokenize('''This is a test'''      )
				self.assertListEqual(UpperCamelCase__  ,	['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']      )

				self.assertListEqual(
				    tokenizer.convert_tokens_to_ids(UpperCamelCase__      )  ,	[2, 3, 4, 5, 6]  ,	)

				snake_case  :       Any     			=						tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6]      )
				self.assertListEqual(UpperCamelCase__  ,	['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']      )

				snake_case  :       Optional[int]     			=						tokenizer.convert_tokens_to_string(UpperCamelCase__      )
				self.assertEqual(UpperCamelCase__  ,	'''This is a test'''      )



		@slow
		def       lowerCAmelCase				(      self				:			Union[str, Any]      )						->			List[Any]:






				"""simple docstring"""







				snake_case  :       Tuple     			=						{'''input_ids''': [[12_8022, 11_0108, 397, 11, 3_8272, 2247, 12_4811, 285, 1_8105, 1586, 207, 7, 3_9534, 4428, 397, 1019, 1_8105, 1586, 207, 7, 4_1337, 1_6786, 241, 7, 2_0214, 17, 12_5690, 1_0398, 7, 4_4378, 5_8069, 6_8342, 7798, 7343, 11, 299, 3_3310, 4, 158, 3_7350, 9_4077, 4569, 299, 3_3310, 90, 4, 5_2840, 290, 4, 3_1270, 112, 299, 682, 4, 5_2840, 3_9953, 1_4079, 193, 5_2519, 9_0894, 1_7894, 12_0697, 11, 4_0445, 551, 17, 1019, 5_2519, 9_0894, 1_7756, 963, 11, 4_0445, 480, 17, 9792, 1120, 5173, 1393, 6240, 1_6786, 241, 12_0996, 28, 1245, 1393, 11_8240, 1_1123, 1019, 9_3612, 2691, 1_0618, 9_8058, 12_0409, 1928, 279, 4, 4_0683, 367, 178, 207, 1019, 103, 10_3121, 506, 6_5296, 5, 2], [12_8022, 2_1217, 367, 117, 12_5450, 128, 719, 7, 7308, 40, 9_3612, 1_2669, 1116, 1_6704, 71, 1_7785, 3699, 1_5592, 35, 144, 9584, 241, 1_1943, 713, 950, 799, 2247, 8_8427, 150, 149, 11_8813, 12_0706, 1019, 10_6906, 8_1518, 28, 1224, 2_2799, 397, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [12_8022, 1658, 12_3311, 5155, 5578, 4722, 279, 1_4947, 2366, 1120, 1197, 14, 1348, 9232, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
				# fmt: on

				self.tokenizer_integration_test_util(
				    expected_encoding=UpperCamelCase__  ,	model_name='''facebook/m2m100_418M'''  ,	revision='''c168bae485c864188cf9aa0e4108b0b6934dc91e'''  ,	)







@require_torch
@require_sentencepiece
@require_tokenizers
class 	snake_case__       (      unittest.TestCase					):







		"""simple docstring"""

		lowerCamelCase     							=						"""facebook/m2m100_418M"""
		lowerCamelCase     							=						[
		    """In my opinion, there are two levels of response from the French government.""",
		    """NSA Affair Emphasizes Complete Lack of Debate on Intelligence""",
		]
		lowerCamelCase     							=						[
		    """Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.""",
		    """L'affaire NSA souligne l'absence totale de débat sur le renseignement""",
		]

		# fmt: off
		lowerCamelCase     							=						[EN_CODE, 593, 1949, 115781, 4, 71586, 4234, 60633, 126233, 432, 123808, 15592, 1197, 117132, 120618, 5, 2]



		@classmethod
		def       lowerCAmelCase				(      cls				:			List[Any]      )						->			int:






				"""simple docstring"""







				snake_case  :       MaMaaaTokenizer     			=						MaMaaaTokenizer.from_pretrained(
				    cls.checkpoint_name  ,	src_lang='''en'''  ,	tgt_lang='''fr'''      )
				snake_case  :       List[str]     			=						1
				return cls



		def       lowerCAmelCase				(      self				:			Union[str, Any]      )						->			str:






				"""simple docstring"""







				self.assertEqual(self.tokenizer.get_lang_id('''ar'''      )  ,	12_8006      )
				self.assertEqual(self.tokenizer.get_lang_id('''en'''      )  ,	12_8022      )
				self.assertEqual(self.tokenizer.get_lang_id('''ro'''      )  ,	12_8076      )
				self.assertEqual(self.tokenizer.get_lang_id('''mr'''      )  ,	12_8063      )



		def       lowerCAmelCase				(      self				:			Any      )						->			Dict:






				"""simple docstring"""







				snake_case  :       List[str]     			=						self.tokenizer.get_vocab()
				self.assertEqual(len(UpperCamelCase__      )  ,	self.tokenizer.vocab_size      )
				self.assertEqual(vocab['''<unk>''']  ,	3      )
				self.assertIn(self.tokenizer.get_lang_token('''en'''      )  ,	UpperCamelCase__      )



		def       lowerCAmelCase				(      self				:			Optional[Any]      )						->			List[Any]:






				"""simple docstring"""







				snake_case  :       Dict     			=						'''en'''
				snake_case  :       Dict     			=						self.tokenizer.batch_encode_plus(self.src_text      ).input_ids[0]
				self.assertListEqual(self.expected_src_tokens  ,	UpperCamelCase__      )



		def       lowerCAmelCase				(      self				:			List[str]      )						->			List[Any]:






				"""simple docstring"""







				self.assertIn(UpperCamelCase__  ,	self.tokenizer.all_special_ids      )
				# fmt: off
				snake_case  :       str     			=						[FR_CODE, 5364, 82, 8642, 4, 294, 47, 8, 1_4028, 136, 3286, 9706, 6, 9_0797, 6, 14_4012, 162, 8_8128, 3_0061, 5, 2]
				# fmt: on
				snake_case  :       int     			=						self.tokenizer.decode(UpperCamelCase__  ,	skip_special_tokens=UpperCamelCase__      )
				snake_case  :       List[Any]     			=						self.tokenizer.decode(generated_ids[1:]  ,	skip_special_tokens=UpperCamelCase__      )
				self.assertEqual(UpperCamelCase__  ,	UpperCamelCase__      )
				self.assertNotIn(self.tokenizer.eos_token  ,	UpperCamelCase__      )



		def       lowerCAmelCase				(      self				:			Optional[int]      )						->			Tuple:






				"""simple docstring"""







				snake_case  :       Union[str, Any]     			=						tempfile.mkdtemp()
				snake_case  :       Union[str, Any]     			=						self.tokenizer.lang_token_to_id
				self.tokenizer.save_pretrained(UpperCamelCase__      )
				snake_case  :       Optional[int]     			=						MaMaaaTokenizer.from_pretrained(UpperCamelCase__      )
				self.assertDictEqual(new_tok.lang_token_to_id  ,	UpperCamelCase__      )



		@require_torch
		def       lowerCAmelCase				(      self				:			Optional[Any]      )						->			Union[str, Any]:






				"""simple docstring"""







				snake_case  :       Optional[Any]     			=						'''en'''
				snake_case  :       int     			=						'''fr'''

				snake_case  :       Dict     			=						self.tokenizer(self.src_text  ,	text_target=self.tgt_text  ,	padding=UpperCamelCase__  ,	return_tensors='''pt'''      )

				snake_case  :       List[str]     			=						shift_tokens_right(
				    batch['''labels''']  ,	self.tokenizer.pad_token_id  ,	self.tokenizer.eos_token_id      )

				for k in batch:
						snake_case  :       Tuple     			=						batch[k].tolist()
				# batch = {k: v.tolist() for k,v in batch.items()}
				# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
				# batch.decoder_inputs_ids[0][0] ==
				assert batch.input_ids[1][0] == EN_CODE
				assert batch.input_ids[1][-1] == 2
				assert batch.labels[1][0] == FR_CODE
				assert batch.labels[1][-1] == 2
				assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]



		@require_torch
		def       lowerCAmelCase				(      self				:			int      )						->			Union[str, Any]:






				"""simple docstring"""







				snake_case  :       List[str]     			=						'''mr'''
				self.assertListEqual(self.tokenizer.prefix_tokens  ,	[self.tokenizer.get_lang_id('''mr'''      )]      )
				self.assertListEqual(self.tokenizer.suffix_tokens  ,	[self.tokenizer.eos_token_id]      )

				snake_case  :       Optional[int]     			=						'''zh'''
				self.assertListEqual(self.tokenizer.prefix_tokens  ,	[self.tokenizer.get_lang_id('''zh'''      )]      )
				self.assertListEqual(self.tokenizer.suffix_tokens  ,	[self.tokenizer.eos_token_id]      )



		@require_torch
		def       lowerCAmelCase				(      self				:			Union[str, Any]      )						->			Dict:






				"""simple docstring"""







				snake_case  :       List[Any]     			=						'''mr'''
				self.tokenizer._switch_to_target_mode()
				self.assertListEqual(self.tokenizer.prefix_tokens  ,	[self.tokenizer.get_lang_id('''mr'''      )]      )
				self.assertListEqual(self.tokenizer.suffix_tokens  ,	[self.tokenizer.eos_token_id]      )
				self.tokenizer._switch_to_input_mode()
				self.assertListEqual(self.tokenizer.prefix_tokens  ,	[self.tokenizer.get_lang_id(self.tokenizer.src_lang      )]      )

				snake_case  :       List[str]     			=						'''zh'''
				self.tokenizer._switch_to_target_mode()
				self.assertListEqual(self.tokenizer.prefix_tokens  ,	[self.tokenizer.get_lang_id('''zh'''      )]      )
				self.assertListEqual(self.tokenizer.suffix_tokens  ,	[self.tokenizer.eos_token_id]      )
				self.tokenizer._switch_to_input_mode()
				self.assertListEqual(self.tokenizer.prefix_tokens  ,	[self.tokenizer.get_lang_id(self.tokenizer.src_lang      )]      )



		@require_torch
		def       lowerCAmelCase				(      self				:			Tuple      )						->			str:






				"""simple docstring"""







				snake_case  :       int     			=						self.tokenizer._build_translation_inputs('''A test'''  ,	return_tensors='''pt'''  ,	src_lang='''en'''  ,	tgt_lang='''ar'''      )

				self.assertEqual(
				    nested_simplify(UpperCamelCase__      )  ,	{
				        # en_XX, A, test, EOS
				        '''input_ids''': [[12_8022, 58, 4183, 2]],
				        '''attention_mask''': [[1, 1, 1, 1]],
				        # ar_AR
				        '''forced_bos_token_id''': 12_8006,
				    }  ,	)


