





import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


UpperCamelCase_												=	logging.getLogger(__name__)

UpperCamelCase_												=	tf.data.AUTOTUNE






def 				_UpperCAmelCase			(    ):




   '''simple docstring'''

   UpperCAmelCase__						=argparse.ArgumentParser(description="Train a masked language model on TPU."     )
   parser.add_argument(
       "--pretrained_model_config"				,							type=A				,							default="roberta-base"				,							help="The model config to use. Note that we don't copy the model's weights, only the config!"				,							)
   parser.add_argument(
       "--tokenizer"				,							type=A				,							default="unigram-tokenizer-wikitext"				,							help="The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model's vocab size."				,							)

   parser.add_argument(
       "--per_replica_batch_size"				,							type=A				,							default=8				,							help="Batch size per TPU core."				,							)

   parser.add_argument(
       "--no_tpu"				,							action="store_true"				,							help="If set, run on CPU and don't try to initialize a TPU. Useful for debugging on non-TPU instances."				,							)

   parser.add_argument(
       "--tpu_name"				,							type=A				,							help="Name of TPU resource to initialize. Should be blank on Colab, and 'local' on TPU VMs."				,							default="local"				,							)

   parser.add_argument(
       "--tpu_zone"				,							type=A				,							help="Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes."				,							)

   parser.add_argument(
       "--gcp_project"				,							type=A				,							help="Google cloud project name. Only used for non-Colab TPU nodes."     )

   parser.add_argument(
       "--bfloat16"				,							action="store_true"				,							help="Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU."				,							)

   parser.add_argument(
       "--train_dataset"				,							type=A				,							help="Path to training dataset to load. If the path begins with `gs://`"
       " then the dataset will be loaded from a Google Cloud Storage bucket."				,							)

   parser.add_argument(
       "--shuffle_buffer_size"				,							type=A				,							default=2**18				,							help="Size of the shuffle buffer (in samples)"				,							)

   parser.add_argument(
       "--eval_dataset"				,							type=A				,							help="Path to evaluation dataset to load. If the path begins with `gs://`"
       " then the dataset will be loaded from a Google Cloud Storage bucket."				,							)

   parser.add_argument(
       "--num_epochs"				,							type=A				,							default=1				,							help="Number of epochs to train for."				,							)

   parser.add_argument(
       "--learning_rate"				,							type=A				,							default=1e-4				,							help="Learning rate to use for training."				,							)

   parser.add_argument(
       "--weight_decay_rate"				,							type=A				,							default=1e-3				,							help="Weight decay rate to use for training."				,							)

   parser.add_argument(
       "--max_length"				,							type=A				,							default=512				,							help="Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py"				,							)

   parser.add_argument(
       "--mlm_probability"				,							type=A				,							default=0.15				,							help="Fraction of tokens to mask during training."				,							)

   parser.add_argument("--output_dir"				,							type=A				,							required=A				,							help="Path to save model checkpoints to."     )
   parser.add_argument("--hub_model_id"				,							type=A				,							help="Model ID to upload to on the Hugging Face Hub."     )

   UpperCAmelCase__						=parser.parse_args()
   return args






def 				_UpperCAmelCase			(    A     ):




   '''simple docstring'''

   try:
      if args.tpu_name:
         UpperCAmelCase__						=tf.distribute.cluster_resolver.TPUClusterResolver(
             args.tpu_name				,							zone=args.tpu_zone				,							project=args.gcp_project     )
      else:
         UpperCAmelCase__						=tf.distribute.cluster_resolver.TPUClusterResolver()
   except ValueError:
      raise RuntimeError(
          "Couldn't connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or "
          "--gcp_project. When running on a TPU VM, use --tpu_name local."     )

   tf.config.experimental_connect_to_cluster(A     )
   tf.tpu.experimental.initialize_tpu_system(A     )

   return tpu






def 				_UpperCAmelCase			(    A     ):




   '''simple docstring'''

   UpperCAmelCase__						=0
   for file in file_list:
      UpperCAmelCase__						=file.split("/"     )[-1]
      UpperCAmelCase__						=re.search(R"-\d+-(\d+)\.tfrecord"				,							A     ).group(1     )
      UpperCAmelCase__						=int(A     )
      num_samples += sample_count

   return num_samples






def 				_UpperCAmelCase			(    A				,							A				,							A				,							A				,							A				,							A=None     ):




   '''simple docstring'''

   UpperCAmelCase__						=count_samples(A     )
   UpperCAmelCase__						=tf.data.Dataset.from_tensor_slices(A     )
   if shuffle:
      UpperCAmelCase__						=dataset.shuffle(len(A     )     )
   UpperCAmelCase__						=tf.data.TFRecordDataset(A				,							num_parallel_reads=A     )
   # TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
   UpperCAmelCase__						=dataset.apply(tf.data.experimental.assert_cardinality(A     )     )
   UpperCAmelCase__						=dataset.map(A				,							num_parallel_calls=A     )
   if shuffle:
      assert shuffle_buffer_size is not None
      UpperCAmelCase__						=dataset.shuffle(args.shuffle_buffer_size     )
   UpperCAmelCase__						=dataset.batch(A				,							drop_remainder=A     )
   UpperCAmelCase__						=dataset.map(A				,							num_parallel_calls=A     )
   UpperCAmelCase__						=dataset.prefetch(A     )
   return dataset






def 				_UpperCAmelCase			(    A     ):




   '''simple docstring'''

   if not args.no_tpu:
      UpperCAmelCase__						=initialize_tpu(A     )
      UpperCAmelCase__						=tf.distribute.TPUStrategy(A     )
   else:
      UpperCAmelCase__						=tf.distribute.OneDeviceStrategy(device="/gpu:0"     )

   if args.bfloataa:
      tf.keras.mixed_precision.set_global_policy("mixed_bfloat16"     )

   UpperCAmelCase__						=AutoTokenizer.from_pretrained(args.tokenizer     )
   UpperCAmelCase__						=AutoConfig.from_pretrained(args.pretrained_model_config     )
   UpperCAmelCase__						=tokenizer.vocab_size

   UpperCAmelCase__						=tf.io.gfile.glob(os.path.join(args.train_dataset				,							"*.tfrecord"     )     )
   if not training_records:
      raise ValueError(F"""No .tfrecord files found in {args.train_dataset}."""     )
   UpperCAmelCase__						=tf.io.gfile.glob(os.path.join(args.eval_dataset				,							"*.tfrecord"     )     )
   if not eval_records:
      raise ValueError(F"""No .tfrecord files found in {args.eval_dataset}."""     )

   UpperCAmelCase__						=count_samples(A     )

   UpperCAmelCase__						=num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
   UpperCAmelCase__						=steps_per_epoch * args.num_epochs

   with strategy.scope():
      UpperCAmelCase__						=TFAutoModelForMaskedLM.from_config(A     )
      model(model.dummy_inputs     )  # Pass some dummy inputs through the model to ensure all the weights are built
      UpperCAmelCase__			,							UpperCAmelCase__						=create_optimizer(
          num_train_steps=A				,							num_warmup_steps=total_train_steps // 20				,							init_lr=args.learning_rate				,							weight_decay_rate=args.weight_decay_rate				,							)

      # Transformers models compute the right loss for their task by default when labels are passed, and will
      # use this for training unless you specify your own loss function in compile().
      model.compile(optimizer=A				,							metrics=["accuracy"]     )

   def decode_fn(A     ):
      UpperCAmelCase__						={
          "input_ids": tf.io.FixedLenFeature(dtype=tf.intaa				,							shape=(args.max_length,)     ),
          "attention_mask": tf.io.FixedLenFeature(dtype=tf.intaa				,							shape=(args.max_length,)     ),
      }
      return tf.io.parse_single_example(A				,							A     )

   # Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
   # use their methods in our data pipeline.
   UpperCAmelCase__						=DataCollatorForLanguageModeling(
       tokenizer=A				,							mlm_probability=args.mlm_probability				,							mlm=A				,							return_tensors="tf"     )

   def mask_with_collator(A     ):
      # TF really needs an isin() function
      UpperCAmelCase__						=(
          ~tf.cast(batch["attention_mask"]				,							tf.bool     )
          | (batch["input_ids"] == tokenizer.cls_token_id)
          | (batch["input_ids"] == tokenizer.sep_token_id)
      )
      UpperCAmelCase__			,							UpperCAmelCase__						=data_collator.tf_mask_tokens(
          batch["input_ids"]				,							vocab_size=len(A     )				,							mask_token_id=tokenizer.mask_token_id				,							special_tokens_mask=A				,							)
      return batch

   UpperCAmelCase__						=args.per_replica_batch_size * strategy.num_replicas_in_sync

   UpperCAmelCase__						=prepare_dataset(
       A				,							decode_fn=A				,							mask_fn=A				,							batch_size=A				,							shuffle=A				,							shuffle_buffer_size=args.shuffle_buffer_size				,							)

   UpperCAmelCase__						=prepare_dataset(
       A				,							decode_fn=A				,							mask_fn=A				,							batch_size=A				,							shuffle=A				,							)

   UpperCAmelCase__						=[]
   if args.hub_model_id:
      callbacks.append(
          PushToHubCallback(output_dir=args.output_dir				,							hub_model_id=args.hub_model_id				,							tokenizer=A     )     )

   model.fit(
       A				,							validation_data=A				,							epochs=args.num_epochs				,							callbacks=A				,							)

   model.save_pretrained(args.output_dir     )


if __name__ == "__main__":
    UpperCamelCase_												=	parse_args()
    main(args)










import argparse
import shutil
from pathlib import Path

from tqdm import tqdm

from transformers import AutoTokenizer






def 				_UpperCAmelCase			(    A				,							A				,							A				,							A=1024     ):




   '''simple docstring'''

   UpperCAmelCase__			,							UpperCAmelCase__						=[], []

   UpperCAmelCase__						=list(zip(A				,							A     )     )
   UpperCAmelCase__			,							UpperCAmelCase__						=sorted_examples[0]

   def is_too_big(A     ):
      return tok(A				,							return_tensors="pt"     ).input_ids.shape[1] > max_tokens

   for src, tgt in tqdm(sorted_examples[1:]     ):
      UpperCAmelCase__						=new_src + " " + src
      UpperCAmelCase__						=new_tgt + " " + tgt
      if is_too_big(A     ) or is_too_big(A     ):  # cant fit, finalize example
         finished_src.append(A     )
         finished_tgt.append(A     )
         UpperCAmelCase__			,							UpperCAmelCase__						=src, tgt
      else:  # can fit, keep adding
         UpperCAmelCase__			,							UpperCAmelCase__						=cand_src, cand_tgt

    # cleanup
   if new_src:
      assert new_tgt
      finished_src.append(A     )
      finished_tgt.append(A     )
   return finished_src, finished_tgt






def 				_UpperCAmelCase			(    A				,							A				,							A				,							A     ):




   '''simple docstring'''

   UpperCAmelCase__						=Path(A     )
   save_path.mkdir(exist_ok=A     )
   for split in ["train"]:
      UpperCAmelCase__			,							UpperCAmelCase__						=data_dir / F"""{split}.source""", data_dir / F"""{split}.target"""
      UpperCAmelCase__						=[x.rstrip() for x in Path(A     ).open().readlines()]
      UpperCAmelCase__						=[x.rstrip() for x in Path(A     ).open().readlines()]
      UpperCAmelCase__			,							UpperCAmelCase__						=pack_examples(A				,							A				,							A				,							A     )
      print(F"""packed {split} split from {len(A     )} examples -> {len(A     )}."""     )
      Path(save_path / F"""{split}.source"""     ).open("w"     ).write("\n".join(A     )     )
      Path(save_path / F"""{split}.target"""     ).open("w"     ).write("\n".join(A     )     )
   for split in ["val", "test"]:
      UpperCAmelCase__			,							UpperCAmelCase__						=data_dir / F"""{split}.source""", data_dir / F"""{split}.target"""
      shutil.copyfile(A				,							save_path / F"""{split}.source"""     )
      shutil.copyfile(A				,							save_path / F"""{split}.target"""     )






def 				_UpperCAmelCase			(    ):




   '''simple docstring'''

   UpperCAmelCase__						=argparse.ArgumentParser()
   parser.add_argument("--tok_name"				,							type=A				,							help="like facebook/bart-large-cnn,t5-base, etc."     )
   parser.add_argument("--max_seq_len"				,							type=A				,							default=128     )
   parser.add_argument("--data_dir"				,							type=A     )
   parser.add_argument("--save_path"				,							type=A     )
   UpperCAmelCase__						=parser.parse_args()
   UpperCAmelCase__						=AutoTokenizer.from_pretrained(args.tok_name     )
   return pack_data_dir(A				,							Path(args.data_dir     )				,							args.max_seq_len				,							args.save_path     )


if __name__ == "__main__":
    packer_cli()



