



'''simple docstring'''

import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO
)
UpperCAmelCase      						=       logging.getLogger(__name__)






def 						_snake_case						(   )     ->		List[str]:







	"""simple docstring"""
	lowerCAmelCase       =      argparse.ArgumentParser(
	    description="""Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids)."""				)
	parser.add_argument("""--file_path"""				,						type=_lowerCAmelCase				,						default="""data/dump.txt"""				,						help="""The path to the data."""				)
	parser.add_argument("""--tokenizer_type"""				,						type=_lowerCAmelCase				,						default="""bert"""				,						choices=["""bert""", """roberta""", """gpt2"""]				)
	parser.add_argument("""--tokenizer_name"""				,						type=_lowerCAmelCase				,						default="""bert-base-uncased"""				,						help="""The tokenizer to use."""				)
	parser.add_argument("""--dump_file"""				,						type=_lowerCAmelCase				,						default="""data/dump"""				,						help="""The dump file prefix."""				)
	lowerCAmelCase       =      parser.parse_args()

	logger.info(f'Loading Tokenizer ({args.tokenizer_name})'				)
	if args.tokenizer_type == "bert":
		lowerCAmelCase       =      BertTokenizer.from_pretrained(args.tokenizer_name				)
		lowerCAmelCase       =      tokenizer.special_tokens_map["cls_token"]  # `[CLS]`
		lowerCAmelCase       =      tokenizer.special_tokens_map["sep_token"]  # `[SEP]`
	elif args.tokenizer_type == "roberta":
		lowerCAmelCase       =      RobertaTokenizer.from_pretrained(args.tokenizer_name				)
		lowerCAmelCase       =      tokenizer.special_tokens_map["cls_token"]  # `<s>`
		lowerCAmelCase       =      tokenizer.special_tokens_map["sep_token"]  # `</s>`
	elif args.tokenizer_type == "gpt2":
		lowerCAmelCase       =      GPTaTokenizer.from_pretrained(args.tokenizer_name				)
		lowerCAmelCase       =      tokenizer.special_tokens_map["bos_token"]  # `<|endoftext|>`
		lowerCAmelCase       =      tokenizer.special_tokens_map["eos_token"]  # `<|endoftext|>`

	logger.info(f'Loading text from {args.file_path}'				)
	with open(args.file_path				,						"""r"""				,						encoding="""utf8"""				) as fp:
		lowerCAmelCase       =      fp.readlines()

	logger.info("""Start encoding"""				)
	logger.info(f'{len(_lowerCAmelCase				)} examples to process.'				)

	lowerCAmelCase       =      []
	lowerCAmelCase       =      0
	lowerCAmelCase       =      10_000
	lowerCAmelCase       =      time.time()
	for text in data:
		lowerCAmelCase       =      f'{bos} {text.strip()} {sep}'
		lowerCAmelCase       =      tokenizer.encode(_lowerCAmelCase				,						add_special_tokens=_lowerCAmelCase				)
		rslt.append(_lowerCAmelCase				)

		iter += 1
		if iter % interval == 0:
			lowerCAmelCase       =      time.time()
			logger.info(f'{iter} examples processed. - {(end-start):.2f}s/{interval}expl'				)
			lowerCAmelCase       =      time.time()
	logger.info("""Finished binarization"""				)
	logger.info(f'{len(_lowerCAmelCase				)} examples processed.'				)

	lowerCAmelCase       =      f'{args.dump_file}.{args.tokenizer_name}.pickle'
	lowerCAmelCase       =      tokenizer.vocab_size
	if vocab_size < (1 << 16):
		lowerCAmelCase       =      [np.uintaa(_lowerCAmelCase				) for d in rslt]
	else:
		lowerCAmelCase       =      [np.intaa(_lowerCAmelCase				) for d in rslt]
	random.shuffle(rslt_				)
	logger.info(f'Dump to {dp_file}'				)
	with open(_lowerCAmelCase				,						"""wb"""				) as handle:
		pickle.dump(rslt_				,						_lowerCAmelCase				,						protocol=pickle.HIGHEST_PROTOCOL				)


if __name__ == "__main__":
				main()






import os



def 						A_							(	)		->					Union[str, Any]:
 with open(os.path.dirname(_lowerCAmelCase	) + "/grid.txt"	) as f:
  UpperCamelCase      :   Optional[Any]			      =  []  # noqa: E741
  for _ in range(20	):
   l.append([int(_lowerCAmelCase	) for x in f.readline().split()]	)

  UpperCamelCase      :   str			      =  0

  # right
  for i in range(20	):
   for j in range(17	):
    UpperCamelCase      :   int			      =  l[i][j] * l[i][j + 1] * l[i][j + 2] * l[i][j + 3]
    if temp > maximum:
     UpperCamelCase      :   List[Any]			      =  temp

        # down
  for i in range(17	):
   for j in range(20	):
    UpperCamelCase      :   List[str]			      =  l[i][j] * l[i + 1][j] * l[i + 2][j] * l[i + 3][j]
    if temp > maximum:
     UpperCamelCase      :   List[str]			      =  temp

        # diagonal 1
  for i in range(17	):
   for j in range(17	):
    UpperCamelCase      :   Any			      =  l[i][j] * l[i + 1][j + 1] * l[i + 2][j + 2] * l[i + 3][j + 3]
    if temp > maximum:
     UpperCamelCase      :   Tuple			      =  temp

        # diagonal 2
  for i in range(17	):
   for j in range(3				,  20	):
    UpperCamelCase      :   Tuple			      =  l[i][j] * l[i + 1][j - 1] * l[i + 2][j - 2] * l[i + 3][j - 3]
    if temp > maximum:
     UpperCamelCase      :   List[Any]			      =  temp
  return maximum


if __name__ == "__main__":
    print(solution())
