

'''simple docstring'''



from typing import Dict, List, Optional, Union

import numpy as np

from .feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from .utils import PaddingStrategy, TensorType, is_tf_tensor, is_torch_tensor, logging, to_numpy


UpperCAmelCase					:			Optional[int]       =	logging.get_logger(__name__)







class 		lowerCAmelCase__  ( a   ):



 """simple docstring"""



 def __init__(						self   :			Optional[int]   ,				__SCREAMING_SNAKE_CASE   :			int   ,				__SCREAMING_SNAKE_CASE   :			int   ,				__SCREAMING_SNAKE_CASE   :			float   ,				**__SCREAMING_SNAKE_CASE   :			Union[str, Any]     )    ->					Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = feature_size
      __SCREAMING_SNAKE_CASE      = sampling_rate
      __SCREAMING_SNAKE_CASE      = padding_value

      __SCREAMING_SNAKE_CASE      = kwargs.pop("""padding_side"""   ,				"""right"""     )
      __SCREAMING_SNAKE_CASE      = kwargs.pop("""return_attention_mask"""   ,				__SCREAMING_SNAKE_CASE     )

      super().__init__(**__SCREAMING_SNAKE_CASE     )
 def  UpperCAmelCase__						(						self   :			Optional[Any]   ,				__SCREAMING_SNAKE_CASE   :			Union[
         BatchFeature,
         List[BatchFeature],
         Dict[str, BatchFeature],
         Dict[str, List[BatchFeature]],
         List[Dict[str, BatchFeature]],
     ]   ,				__SCREAMING_SNAKE_CASE   :			Union[bool, str, PaddingStrategy] = True   ,				__SCREAMING_SNAKE_CASE   :			Optional[int] = None   ,				__SCREAMING_SNAKE_CASE   :			bool = False   ,				__SCREAMING_SNAKE_CASE   :			Optional[int] = None   ,				__SCREAMING_SNAKE_CASE   :			Optional[bool] = None   ,				__SCREAMING_SNAKE_CASE   :			Optional[Union[str, TensorType]] = None   ,				)    ->					BatchFeature:


      """simple docstring"""




      if isinstance(__SCREAMING_SNAKE_CASE   ,				(list, tuple)     ) and isinstance(processed_features[0]   ,				(dict, BatchFeature)     ):
           __SCREAMING_SNAKE_CASE      = {
               key: [example[key] for example in processed_features] for key in processed_features[0].keys()
           }

      # The model's main input name, usually `input_values`, has be passed for padding
      if self.model_input_names[0] not in processed_features:
           raise ValueError(
               """You should supply an instance of `transformers.BatchFeature` or list of `transformers.BatchFeature`"""
               f' to this method that includes {self.model_input_names[0]}, but you provided'
               f' {list(processed_features.keys()     )}'     )

      __SCREAMING_SNAKE_CASE      = processed_features[self.model_input_names[0]]
      __SCREAMING_SNAKE_CASE      = (
          return_attention_mask if return_attention_mask is not None else self.return_attention_mask
      )

      if len(__SCREAMING_SNAKE_CASE     ) == 0:
           if return_attention_mask:
                __SCREAMING_SNAKE_CASE      = []
           return processed_features

      # If we have PyTorch/TF tensors or lists as inputs, we cast them as Numpy arrays
      # and rebuild them afterwards if no return_tensors is specified
      # Note that we lose the specific device the tensor may be on for PyTorch

      __SCREAMING_SNAKE_CASE      = required_input[0]
      if isinstance(__SCREAMING_SNAKE_CASE   ,				(list, tuple)     ):
           # first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.
           __SCREAMING_SNAKE_CASE      = 0
           while len(required_input[index]     ) == 0:
                index += 1
           if index < len(__SCREAMING_SNAKE_CASE     ):
                __SCREAMING_SNAKE_CASE      = required_input[index][0]

      if return_tensors is None:
           if is_tf_tensor(__SCREAMING_SNAKE_CASE     ):
                __SCREAMING_SNAKE_CASE      = """tf"""
           elif is_torch_tensor(__SCREAMING_SNAKE_CASE     ):
                __SCREAMING_SNAKE_CASE      = """pt"""
           elif isinstance(__SCREAMING_SNAKE_CASE   ,				(int, float, list, tuple, np.ndarray)     ):
                __SCREAMING_SNAKE_CASE      = """np"""
           else:
                raise ValueError(
                    f'type of {first_element} unknown: {type(__SCREAMING_SNAKE_CASE     )}. '
                    """Should be one of a python, numpy, pytorch or tensorflow object."""     )

      for key, value in processed_features.items():
           if isinstance(value[0]   ,				(int, float)     ):
                __SCREAMING_SNAKE_CASE      = to_numpy(__SCREAMING_SNAKE_CASE     )
           else:
                __SCREAMING_SNAKE_CASE      = [to_numpy(__SCREAMING_SNAKE_CASE     ) for v in value]

        # Convert padding_strategy in PaddingStrategy
      __SCREAMING_SNAKE_CASE      = self._get_padding_strategies(padding=__SCREAMING_SNAKE_CASE   ,				max_length=__SCREAMING_SNAKE_CASE     )

      __SCREAMING_SNAKE_CASE      = processed_features[self.model_input_names[0]]

      __SCREAMING_SNAKE_CASE      = len(__SCREAMING_SNAKE_CASE     )
      if not all(len(__SCREAMING_SNAKE_CASE     ) == batch_size for v in processed_features.values()     ):
           raise ValueError("""Some items in the output dictionary have a different batch size than others."""     )

      __SCREAMING_SNAKE_CASE      = []
      for i in range(__SCREAMING_SNAKE_CASE     ):
           __SCREAMING_SNAKE_CASE      = {k: v[i] for k, v in processed_features.items()}
           # truncation
           __SCREAMING_SNAKE_CASE      = self._truncate(
               __SCREAMING_SNAKE_CASE   ,				max_length=__SCREAMING_SNAKE_CASE   ,				pad_to_multiple_of=__SCREAMING_SNAKE_CASE   ,				truncation=__SCREAMING_SNAKE_CASE   ,				)
           truncated_inputs.append(__SCREAMING_SNAKE_CASE     )

      if padding_strategy == PaddingStrategy.LONGEST:
           # make sure that `max_length` cannot be longer than the longest truncated length
           __SCREAMING_SNAKE_CASE      = max(len(input_slice[self.model_input_names[0]]     ) for input_slice in truncated_inputs     )
           __SCREAMING_SNAKE_CASE      = PaddingStrategy.MAX_LENGTH

      __SCREAMING_SNAKE_CASE      = {}
      for i in range(__SCREAMING_SNAKE_CASE     ):
           # padding
           __SCREAMING_SNAKE_CASE      = self._pad(
               truncated_inputs[i]   ,				max_length=__SCREAMING_SNAKE_CASE   ,				padding_strategy=__SCREAMING_SNAKE_CASE   ,				pad_to_multiple_of=__SCREAMING_SNAKE_CASE   ,				return_attention_mask=__SCREAMING_SNAKE_CASE   ,				)

           for key, value in outputs.items():
                if key not in batch_outputs:
                     __SCREAMING_SNAKE_CASE      = []
                if value.dtype is np.dtype(np.floataa     ):
                     __SCREAMING_SNAKE_CASE      = value.astype(np.floataa     )
                batch_outputs[key].append(__SCREAMING_SNAKE_CASE     )

      return BatchFeature(__SCREAMING_SNAKE_CASE   ,				tensor_type=__SCREAMING_SNAKE_CASE     )
 def  UpperCAmelCase__						(						self   :			Optional[int]   ,				__SCREAMING_SNAKE_CASE   :			Union[Dict[str, np.ndarray], BatchFeature]   ,				__SCREAMING_SNAKE_CASE   :			Optional[int] = None   ,				__SCREAMING_SNAKE_CASE   :			PaddingStrategy = PaddingStrategy.DO_NOT_PAD   ,				__SCREAMING_SNAKE_CASE   :			Optional[int] = None   ,				__SCREAMING_SNAKE_CASE   :			Optional[bool] = None   ,				)    ->					dict:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = processed_features[self.model_input_names[0]]

      if padding_strategy == PaddingStrategy.LONGEST:
           __SCREAMING_SNAKE_CASE      = len(__SCREAMING_SNAKE_CASE     )

      if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
           __SCREAMING_SNAKE_CASE      = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

      __SCREAMING_SNAKE_CASE      = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(__SCREAMING_SNAKE_CASE     ) < max_length

      if return_attention_mask and "attention_mask" not in processed_features:
           __SCREAMING_SNAKE_CASE      = np.ones(len(__SCREAMING_SNAKE_CASE     )   ,				dtype=np.intaa     )

      if needs_to_be_padded:
           __SCREAMING_SNAKE_CASE      = max_length - len(__SCREAMING_SNAKE_CASE     )
           if self.padding_side == "right":
                if return_attention_mask:
                     __SCREAMING_SNAKE_CASE      = np.pad(
                         processed_features["""attention_mask"""]   ,				(0, difference)     )
                __SCREAMING_SNAKE_CASE      = ((0, difference), (0, 0)) if self.feature_size > 1 else (0, difference)
                __SCREAMING_SNAKE_CASE      = np.pad(
                    __SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE   ,				"""constant"""   ,				constant_values=self.padding_value     )
           elif self.padding_side == "left":
                if return_attention_mask:
                     __SCREAMING_SNAKE_CASE      = np.pad(
                         processed_features["""attention_mask"""]   ,				(difference, 0)     )
                __SCREAMING_SNAKE_CASE      = ((difference, 0), (0, 0)) if self.feature_size > 1 else (difference, 0)
                __SCREAMING_SNAKE_CASE      = np.pad(
                    __SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE   ,				"""constant"""   ,				constant_values=self.padding_value     )
           else:
                raise ValueError("""Invalid padding strategy:""" + str(self.padding_side     )     )

      return processed_features
 def  UpperCAmelCase__						(						self   :			Dict   ,				__SCREAMING_SNAKE_CASE   :			Union[Dict[str, np.ndarray], BatchFeature]   ,				__SCREAMING_SNAKE_CASE   :			Optional[int] = None   ,				__SCREAMING_SNAKE_CASE   :			Optional[int] = None   ,				__SCREAMING_SNAKE_CASE   :			Optional[bool] = None   ,				)    ->					Tuple:


      """simple docstring"""




      if not truncation:
           return processed_features
      elif truncation and max_length is None:
           raise ValueError("""When setting ``truncation=True``, make sure that ``max_length`` is defined."""     )

      __SCREAMING_SNAKE_CASE      = processed_features[self.model_input_names[0]]

      # find `max_length` that fits `pad_to_multiple_of`
      if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
           __SCREAMING_SNAKE_CASE      = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

      __SCREAMING_SNAKE_CASE      = len(__SCREAMING_SNAKE_CASE     ) > max_length

      if needs_to_be_truncated:
           __SCREAMING_SNAKE_CASE      = processed_features[self.model_input_names[0]][:max_length]
           if "attention_mask" in processed_features:
                __SCREAMING_SNAKE_CASE      = processed_features["""attention_mask"""][:max_length]

      return processed_features


 def  UpperCAmelCase__						(						self   :			int   ,				__SCREAMING_SNAKE_CASE   :			str=False   ,				__SCREAMING_SNAKE_CASE   :			int=None     )    ->					str:


      """simple docstring"""




      if padding is not False:
           if padding is True:
                __SCREAMING_SNAKE_CASE      = PaddingStrategy.LONGEST  # Default to pad to the longest sequence in the batch
           elif not isinstance(__SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE     ):
                __SCREAMING_SNAKE_CASE      = PaddingStrategy(__SCREAMING_SNAKE_CASE     )
           elif isinstance(__SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE     ):
                __SCREAMING_SNAKE_CASE      = padding
      else:
           __SCREAMING_SNAKE_CASE      = PaddingStrategy.DO_NOT_PAD

      # Set max length if needed
      if max_length is None:
           if padding_strategy == PaddingStrategy.MAX_LENGTH:
                raise ValueError(
                    f'When setting ``padding={PaddingStrategy.MAX_LENGTH}``, make sure that max_length is defined'     )

        # Test if we have a padding value
      if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.padding_value is None):
           raise ValueError(
               """Asking to pad but the feature_extractor does not have a padding value. Please select a value to use"""
               """ as `padding_value`. For example: `feature_extractor.padding_value = 0.0`."""     )

      return padding_strategy









'''simple docstring'''



import multiprocessing
import time

from arguments import PretokenizationArguments
from datasets import load_dataset

from transformers import AutoTokenizer, HfArgumentParser







def   a__			(    a__		):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE      = {}
     __SCREAMING_SNAKE_CASE      = tokenizer(example["""content"""] ,       truncation=a__		)["""input_ids"""]
     __SCREAMING_SNAKE_CASE      = len(example["""content"""]		) / len(output["""input_ids"""]		)
     return output


UpperCAmelCase					:			Tuple       =	HfArgumentParser(PretokenizationArguments)
UpperCAmelCase					:			Optional[Any]       =	parser.parse_args()
if args.num_workers is None:
  UpperCAmelCase					:			List[Any]       =	multiprocessing.cpu_count()
UpperCAmelCase					:			str       =	AutoTokenizer.from_pretrained(args.tokenizer_dir)

UpperCAmelCase					:			Optional[int]       =	time.time()
UpperCAmelCase					:			Any       =	load_dataset(args.dataset_name, split='train')
print(f"""Dataset loaded in {time.time()-t_start:.2f}s""")

UpperCAmelCase					:			List[str]       =	time.time()
UpperCAmelCase					:			List[str]       =	ds.map(
    tokenize,
    num_proc=args.num_workers,
    remove_columns=[
        'repo_name',
        'path',
        'copies',
        'size',
        'content',
        'license',
        'hash',
        'line_mean',
        'line_max',
        'alpha_frac',
        'autogenerated',
    ],
)
print(f"""Dataset tokenized in {time.time()-t_start:.2f}s""")

UpperCAmelCase					:			List[Any]       =	time.time()
ds.push_to_hub(args.tokenized_data_repo)



print(f"""Data pushed to the hub in {time.time()-t_start:.2f}s""")






