






"""simple docstring"""
import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


lowerCamelCase_					=      logging.get_logger(__name__)

def 				snake_case      (   ):
			# Get the sagemaker specific mp parameters from smp_options variable.
			UpperCAmelCase_     : Optional[Any]			     =					os.getenv("SM_HP_MP_PARAMETERS"	,"{}"						)
			try:
						# Parse it and check the field "partitions" is included, it is required for model parallel.
						UpperCAmelCase_     : int			     =					json.loads(A__						)
						if "partitions" not in smp_options:
									return False
			except json.JSONDecodeError:
						return False

			# Get the sagemaker specific framework parameters from mpi_options variable.
			UpperCAmelCase_     : Union[str, Any]			     =					os.getenv("SM_FRAMEWORK_PARAMS"	,"{}"						)
			try:
						# Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
						UpperCAmelCase_     : Dict			     =					json.loads(A__						)
						if not mpi_options.get("sagemaker_mpi_enabled"	,A__						):
									return False
			except json.JSONDecodeError:
						return False
			# Lastly, check if the `smdistributed` module is present.
			return importlib.util.find_spec("smdistributed"						) is not None


if is_sagemaker_model_parallel_available():
			import smdistributed.modelparallel.torch as smp

			smp.init()







@dataclass
class 			UpperCamelCase_       (__A     ):
	__magic_name__	      =				field(
	    default=''''''				,						metadata={'''help''': '''Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer'''}				,						)





	def      _SCREAMING_SNAKE_CASE      (      self					: int						)						->			Dict:
				super().__post_init__()
				warnings.warn(
				    "`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use "
				    "`TrainingArguments` instead."	,	lowerCAmelCase_	,	)





	@cached_property
	def      _SCREAMING_SNAKE_CASE      (      self					: Dict						)						->			"torch.device":
				logger.info("PyTorch: setting up devices"						)
				if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
							logger.warning(
							    "torch.distributed process group is initialized, but local_rank == -1. "
							    "In order to use Torch DDP, launch your script with `python -m torch.distributed.launch"						)
				if self.no_cuda:
							UpperCAmelCase_     : str			     =					torch.device("cpu"						)
							UpperCAmelCase_     : int			     =					0
				elif is_sagemaker_model_parallel_available():
							UpperCAmelCase_     : Optional[Any]			     =					smp.local_rank()
							UpperCAmelCase_     : List[str]			     =					torch.device("cuda"	,	lowerCAmelCase_						)
							UpperCAmelCase_     : List[str]			     =					1
				elif is_sagemaker_dp_enabled():
							import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

							torch.distributed.init_process_group(backend="smddp"	,	timeout=self.ddp_timeout_delta						)
							UpperCAmelCase_     : Optional[Any]			     =					int(os.getenv("SMDATAPARALLEL_LOCAL_RANK"						)						)
							UpperCAmelCase_     : int			     =					torch.device("cuda"	,	self.local_rank						)
							UpperCAmelCase_     : Optional[int]			     =					1
				elif self.local_rank == -1:
							# if n_gpu is > 1 we'll use nn.DataParallel.
							# If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
							# Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
							# trigger an error that a device index is missing. Index 0 takes into account the
							# GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
							# will use the first GPU in that env, i.e. GPU#1
							UpperCAmelCase_     : Optional[Any]			     =					torch.device("cuda:0" if torch.cuda.is_available() else "cpu"						)
							# Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
							# the default value.
							UpperCAmelCase_     : List[str]			     =					torch.cuda.device_count()
				else:
							# Here, we'll use torch.distributed.
							# Initializes the distributed backend which will take care of synchronizing nodes/GPUs
							if not torch.distributed.is_initialized():
										torch.distributed.init_process_group(backend="nccl"	,	timeout=self.ddp_timeout_delta						)
							UpperCAmelCase_     : int			     =					torch.device("cuda"	,	self.local_rank						)
							UpperCAmelCase_     : List[str]			     =					1

				if device.type == "cuda":
							torch.cuda.set_device(lowerCAmelCase_						)

				return device





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]						)						->			str:
				if is_sagemaker_model_parallel_available():
							return smp.dp_size()

				return super().world_size





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple						)						->			List[str]:
				return not is_sagemaker_model_parallel_available()





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: int						)						->			List[str]:
				return False











"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase_					=      logging.get_logger(__name__)

lowerCamelCase_					=      {
    '''RWKV/rwkv-4-169m-pile''': '''https://huggingface.co/RWKV/rwkv-4-169m-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-430m-pile''': '''https://huggingface.co/RWKV/rwkv-4-430m-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-1b5-pile''': '''https://huggingface.co/RWKV/rwkv-4-1b5-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-3b-pile''': '''https://huggingface.co/RWKV/rwkv-4-3b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-7b-pile''': '''https://huggingface.co/RWKV/rwkv-4-7b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-14b-pile''': '''https://huggingface.co/RWKV/rwkv-4-14b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-raven-1b5''': '''https://huggingface.co/RWKV/rwkv-raven-1b5/resolve/main/config.json''',
    '''RWKV/rwkv-raven-3b''': '''https://huggingface.co/RWKV/rwkv-raven-3b/resolve/main/config.json''',
    '''RWKV/rwkv-raven-7b''': '''https://huggingface.co/RWKV/rwkv-raven-7b/resolve/main/config.json''',
    '''RWKV/rwkv-raven-14b''': '''https://huggingface.co/RWKV/rwkv-raven-14b/resolve/main/config.json''',
}







class 			UpperCamelCase_       (__A     ):
	__magic_name__	      =				'''rwkv'''
	__magic_name__	      =				{'''max_position_embeddings''': '''context_length'''}





	def __init__(      self					: str	,	lowerCAmelCase_					: str=50_277	,	lowerCAmelCase_					: Optional[int]=1_024	,	lowerCAmelCase_					: Optional[int]=4_096	,	lowerCAmelCase_					: Union[str, Any]=32	,	lowerCAmelCase_					: List[Any]=None	,	lowerCAmelCase_					: int=None	,	lowerCAmelCase_					: List[Any]=1e-5	,	lowerCAmelCase_					: str=0	,	lowerCAmelCase_					: Tuple=0	,	lowerCAmelCase_					: int=6	,	lowerCAmelCase_					: Optional[Any]=False	,	lowerCAmelCase_					: Any=True	,	**lowerCAmelCase_					: List[Any]	,	)						->			List[str]:
				UpperCAmelCase_     : Tuple			     =					vocab_size
				UpperCAmelCase_     : List[str]			     =					context_length
				UpperCAmelCase_     : Dict			     =					hidden_size
				UpperCAmelCase_     : Optional[int]			     =					num_hidden_layers
				UpperCAmelCase_     : Optional[int]			     =					attention_hidden_size if attention_hidden_size is not None else hidden_size
				UpperCAmelCase_     : Dict			     =					intermediate_size if intermediate_size is not None else 4 * hidden_size
				UpperCAmelCase_     : Any			     =					layer_norm_epsilon
				UpperCAmelCase_     : List[Any]			     =					rescale_every
				UpperCAmelCase_     : List[str]			     =					use_cache

				UpperCAmelCase_     : List[str]			     =					bos_token_id
				UpperCAmelCase_     : Union[str, Any]			     =					eos_token_id

				super().__init__(
				    tie_word_embeddings=lowerCAmelCase_	,	bos_token_id=lowerCAmelCase_	,	eos_token_id=lowerCAmelCase_	,	**lowerCAmelCase_						)



