





'''simple docstring'''
import json
import logging
import os
import socket

import git
import numpy as np
import torch


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - PID: %(process)d -  %(message)s''',
    datefmt='''%m/%d/%Y %H:%M:%S''',
    level=logging.INFO,
)
a__							:							Tuple =logging.getLogger(__name__)
def 				lowercase__   (   __lowercase						:  str     ) ->      str:



       """simple docstring"""






       __UpperCamelCase       =  git.Repo(search_parent_directories=__lowercase     )
       __UpperCamelCase       =  {
           'repo_id': str(__lowercase     ),
           'repo_sha': str(repo.head.object.hexsha     ),
           'repo_branch': str(repo.active_branch     ),
       }

       with open(os.path.join(__lowercase				,  'git_log.json'     )				,  'w'     ) as f:
              json.dump(__lowercase				,  __lowercase				,  indent=4     )
def 				lowercase__   (   __lowercase						:  int     ) ->      List[Any]:



       """simple docstring"""






       if params.n_gpu <= 0:
              __UpperCamelCase       =  0
              __UpperCamelCase       =  -1
              __UpperCamelCase       =  True
              __UpperCamelCase       =  False
              return

       assert torch.cuda.is_available()

       logger.info('Initializing GPUs'     )
       if params.n_gpu > 1:
              assert params.local_rank != -1

              __UpperCamelCase       =  int(os.environ['WORLD_SIZE']     )
              __UpperCamelCase       =  int(os.environ['N_GPU_NODE']     )
              __UpperCamelCase       =  int(os.environ['RANK']     )

              # number of nodes / node ID
              __UpperCamelCase       =  params.world_size // params.n_gpu_per_node
              __UpperCamelCase       =  params.global_rank // params.n_gpu_per_node
              __UpperCamelCase       =  True

              assert params.n_nodes == int(os.environ['N_NODES']     )
              assert params.node_id == int(os.environ['NODE_RANK']     )

       # local job (single GPU)
       else:
              assert params.local_rank == -1

              __UpperCamelCase       =  1
              __UpperCamelCase       =  0
              __UpperCamelCase       =  0
              __UpperCamelCase       =  0
              __UpperCamelCase       =  1
              __UpperCamelCase       =  1
              __UpperCamelCase       =  False

       # sanity checks
       assert params.n_nodes >= 1
       assert 0 <= params.node_id < params.n_nodes
       assert 0 <= params.local_rank <= params.global_rank < params.world_size
       assert params.world_size == params.n_nodes * params.n_gpu_per_node

       # define whether this is the master process / if we are in multi-node distributed mode
       __UpperCamelCase       =  params.node_id == 0 and params.local_rank == 0
       __UpperCamelCase       =  params.n_nodes > 1

       # summary
       __UpperCamelCase       =  F'''--- Global rank: {params.global_rank} - '''
       logger.info(PREFIX + 'Number of nodes: %i' % params.n_nodes     )
       logger.info(PREFIX + 'Node ID        : %i' % params.node_id     )
       logger.info(PREFIX + 'Local rank     : %i' % params.local_rank     )
       logger.info(PREFIX + 'World size     : %i' % params.world_size     )
       logger.info(PREFIX + 'GPUs per node  : %i' % params.n_gpu_per_node     )
       logger.info(PREFIX + 'Master         : %s' % str(params.is_master     )     )
       logger.info(PREFIX + 'Multi-node     : %s' % str(params.multi_node     )     )
       logger.info(PREFIX + 'Multi-GPU      : %s' % str(params.multi_gpu     )     )
       logger.info(PREFIX + 'Hostname       : %s' % socket.gethostname()     )

       # set GPU device
       torch.cuda.set_device(params.local_rank     )

       # initialize multi-GPU
       if params.multi_gpu:
              logger.info('Initializing PyTorch distributed'     )
              torch.distributed.init_process_group(
                  init_method='env://'				,  backend='nccl'				,  )


def 				lowercase__   (   __lowercase						:  List[Any]     ) ->      Any:



       """simple docstring"""






       np.random.seed(args.seed     )
       torch.manual_seed(args.seed     )
       if args.n_gpu > 0:
              torch.cuda.manual_seed_all(args.seed     )







"""simple docstring"""




import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
a						: Union[str, Any]     	=			logging.getLogger(__name__)






def       _SCREAMING_SNAKE_CASE		(	) ->Tuple:






	'''simple docstring'''






	a      :		Dict	=				argparse.ArgumentParser(
	    description="Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids)."					)
	parser.add_argument("--file_path"    ,       type=_lowercase    ,       default="data/dump.txt"    ,       help="The path to the data."					)
	parser.add_argument("--tokenizer_type"    ,       type=_lowercase    ,       default="bert"    ,       choices=["bert", "roberta", "gpt2"]					)
	parser.add_argument("--tokenizer_name"    ,       type=_lowercase    ,       default="bert-base-uncased"    ,       help="The tokenizer to use."					)
	parser.add_argument("--dump_file"    ,       type=_lowercase    ,       default="data/dump"    ,       help="The dump file prefix."					)
	a      :		Dict	=				parser.parse_args()

	logger.info(F"""Loading Tokenizer ({args.tokenizer_name})"""					)
	if args.tokenizer_type == "bert":
		a      :		Optional[Any]	=				BertTokenizer.from_pretrained(args.tokenizer_name					)
		a      :		str	=				tokenizer.special_tokens_map["cls_token"]  # `[CLS]`
		a      :		List[str]	=				tokenizer.special_tokens_map["sep_token"]  # `[SEP]`
	elif args.tokenizer_type == "roberta":
		a      :		Tuple	=				RobertaTokenizer.from_pretrained(args.tokenizer_name					)
		a      :		Union[str, Any]	=				tokenizer.special_tokens_map["cls_token"]  # `<s>`
		a      :		str	=				tokenizer.special_tokens_map["sep_token"]  # `</s>`
	elif args.tokenizer_type == "gpt2":
		a      :		List[Any]	=				GPTaTokenizer.from_pretrained(args.tokenizer_name					)
		a      :		Optional[int]	=				tokenizer.special_tokens_map["bos_token"]  # `<|endoftext|>`
		a      :		List[Any]	=				tokenizer.special_tokens_map["eos_token"]  # `<|endoftext|>`

	logger.info(F"""Loading text from {args.file_path}"""					)
	with open(args.file_path    ,       "r"    ,       encoding="utf8"					) as fp:
		a      :		List[Any]	=				fp.readlines()

	logger.info("Start encoding"					)
	logger.info(F"""{len(_lowercase					)} examples to process."""					)

	a      :		Optional[Any]	=				[]
	a      :		Optional[Any]	=				0
	a      :		int	=				1_0000
	a      :		Dict	=				time.time()
	for text in data:
		a      :		List[Any]	=				F"""{bos} {text.strip()} {sep}"""
		a      :		Optional[int]	=				tokenizer.encode(_lowercase    ,       add_special_tokens=_lowercase					)
		rslt.append(_lowercase					)

		iter += 1
		if iter % interval == 0:
			a      :		Optional[Any]	=				time.time()
			logger.info(F"""{iter} examples processed. - {(end-start):.2f}s/{interval}expl"""					)
			a      :		Optional[Any]	=				time.time()
	logger.info("Finished binarization"					)
	logger.info(F"""{len(_lowercase					)} examples processed."""					)

	a      :		Optional[int]	=				F"""{args.dump_file}.{args.tokenizer_name}.pickle"""
	a      :		Tuple	=				tokenizer.vocab_size
	if vocab_size < (1 << 16):
		a      :		Optional[int]	=				[np.uintaa(_lowercase					) for d in rslt]
	else:
		a      :		Optional[Any]	=				[np.intaa(_lowercase					) for d in rslt]
	random.shuffle(rslt_					)
	logger.info(F"""Dump to {dp_file}"""					)
	with open(_lowercase    ,       "wb"					) as handle:
		pickle.dump(rslt_    ,       _lowercase    ,       protocol=pickle.HIGHEST_PROTOCOL					)


if __name__ == "__main__":
		main()


