



"""simple docstring"""







import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
lowerCAmelCase__									=logging.getLogger(__name__)






def     _a (	)						->    Tuple:
      __SCREAMING_SNAKE_CASE       		=   argparse.ArgumentParser(
          description='''Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'''     )
      parser.add_argument('''--file_path'''     ,	type=lowerCamelCase__     ,	default='''data/dump.txt'''     ,	help='''The path to the data.'''     )
      parser.add_argument('''--tokenizer_type'''     ,	type=lowerCamelCase__     ,	default='''bert'''     ,	choices=['''bert''', '''roberta''', '''gpt2''']     )
      parser.add_argument('''--tokenizer_name'''     ,	type=lowerCamelCase__     ,	default='''bert-base-uncased'''     ,	help='''The tokenizer to use.'''     )
      parser.add_argument('''--dump_file'''     ,	type=lowerCamelCase__     ,	default='''data/dump'''     ,	help='''The dump file prefix.'''     )
      __SCREAMING_SNAKE_CASE       		=   parser.parse_args()

      logger.info(f"""Loading Tokenizer ({args.tokenizer_name})"""     )
      if args.tokenizer_type == "bert":
            __SCREAMING_SNAKE_CASE       		=   BertTokenizer.from_pretrained(args.tokenizer_name     )
            __SCREAMING_SNAKE_CASE       		=   tokenizer.special_tokens_map['''cls_token''']  # `[CLS]`
            __SCREAMING_SNAKE_CASE       		=   tokenizer.special_tokens_map['''sep_token''']  # `[SEP]`
      elif args.tokenizer_type == "roberta":
            __SCREAMING_SNAKE_CASE       		=   RobertaTokenizer.from_pretrained(args.tokenizer_name     )
            __SCREAMING_SNAKE_CASE       		=   tokenizer.special_tokens_map['''cls_token''']  # `<s>`
            __SCREAMING_SNAKE_CASE       		=   tokenizer.special_tokens_map['''sep_token''']  # `</s>`
      elif args.tokenizer_type == "gpt2":
            __SCREAMING_SNAKE_CASE       		=   GPTaTokenizer.from_pretrained(args.tokenizer_name     )
            __SCREAMING_SNAKE_CASE       		=   tokenizer.special_tokens_map['''bos_token''']  # `<|endoftext|>`
            __SCREAMING_SNAKE_CASE       		=   tokenizer.special_tokens_map['''eos_token''']  # `<|endoftext|>`

      logger.info(f"""Loading text from {args.file_path}"""     )
      with open(args.file_path     ,	'''r'''     ,	encoding='''utf8'''     ) as fp:
            __SCREAMING_SNAKE_CASE       		=   fp.readlines()

      logger.info('''Start encoding'''     )
      logger.info(f"""{len(lowerCamelCase__     )} examples to process."""     )

      __SCREAMING_SNAKE_CASE       		=   []
      __SCREAMING_SNAKE_CASE       		=   0
      __SCREAMING_SNAKE_CASE       		=   1_00_00
      __SCREAMING_SNAKE_CASE       		=   time.time()
      for text in data:
            __SCREAMING_SNAKE_CASE       		=   f"""{bos} {text.strip()} {sep}"""
            __SCREAMING_SNAKE_CASE       		=   tokenizer.encode(lowerCamelCase__     ,	add_special_tokens=lowerCamelCase__     )
            rslt.append(lowerCamelCase__     )

            iter += 1
            if iter % interval == 0:
                  __SCREAMING_SNAKE_CASE       		=   time.time()
                  logger.info(f"""{iter} examples processed. - {(end-start):.2f}s/{interval}expl"""     )
                  __SCREAMING_SNAKE_CASE       		=   time.time()
      logger.info('''Finished binarization'''     )
      logger.info(f"""{len(lowerCamelCase__     )} examples processed."""     )

      __SCREAMING_SNAKE_CASE       		=   f"""{args.dump_file}.{args.tokenizer_name}.pickle"""
      __SCREAMING_SNAKE_CASE       		=   tokenizer.vocab_size
      if vocab_size < (1 << 16):
            __SCREAMING_SNAKE_CASE       		=   [np.uintaa(lowerCamelCase__     ) for d in rslt]
      else:
            __SCREAMING_SNAKE_CASE       		=   [np.intaa(lowerCamelCase__     ) for d in rslt]
      random.shuffle(rslt_     )
      logger.info(f"""Dump to {dp_file}"""     )
      with open(lowerCamelCase__     ,	'''wb'''     ) as handle:
            pickle.dump(rslt_     ,	lowerCamelCase__     ,	protocol=pickle.HIGHEST_PROTOCOL     )


if __name__ == "__main__":
       main()

"""simple docstring"""




from itertools import permutations







def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		if num[3] % 2 != 0:
				return False

		if (num[2] + num[3] + num[4]) % 3 != 0:
				return False

		if num[5] % 5 != 0:
				return False

		lowerCAmelCase__   = [7, 11, 13, 17]
		for i, test in enumerate(lowerCamelCase__  ):
				if (num[i + 4] * 100 + num[i + 5] * 10 + num[i + 6]) % test != 0:
						return False
		return True







def 				_UpperCAmelCase		(							lowerCamelCase__ = 10  ):





		"""simple docstring"""




		return sum(
		    int("""""".join(map(lowerCamelCase__      ,	lowerCamelCase__  )  )  )
		    for num in permutations(range(lowerCamelCase__  )  )
		    if is_substring_divisible(lowerCamelCase__  )  )


if __name__ == "__main__":
			print(F"{solution() = }")




