




import argparse
from copy import deepcopy

import numpy as np
from datasets import ClassLabel, DatasetDict, load_dataset
from evaluate import load

from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    DataCollatorWithPadding,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    set_seed,
)




def 					lowerCamelCase__       (						):





       '''simple docstring'''



       __lowerCamelCase   =						argparse.ArgumentParser()
       parser.add_argument("""--model_ckpt"""  ,			type=snake_case__  ,			default="""microsoft/unixcoder-base-nine"""	)
       parser.add_argument("""--num_epochs"""  ,			type=snake_case__  ,			default=5	)
       parser.add_argument("""--batch_size"""  ,			type=snake_case__  ,			default=6	)
       parser.add_argument("""--gradient_accumulation_steps"""  ,			type=snake_case__  ,			default=1	)
       parser.add_argument("""--freeze"""  ,			type=snake_case__  ,			default=snake_case__	)
       parser.add_argument("""--learning_rate"""  ,			type=snake_case__  ,			default=5E-4	)
       parser.add_argument("""--seed"""  ,			type=snake_case__  ,			default=0	)
       parser.add_argument("""--lr_scheduler_type"""  ,			type=snake_case__  ,			default="""cosine"""	)
       parser.add_argument("""--num_warmup_steps"""  ,			type=snake_case__  ,			default=10	)
       parser.add_argument("""--weight_decay"""  ,			type=snake_case__  ,			default=0.01	)
       parser.add_argument("""--output_dir"""  ,			type=snake_case__  ,			default="""./results"""	)
       return parser.parse_args()


UpperCAmelCase_									=	load('accuracy')




def 					lowerCamelCase__       (						A__       :					Optional[Any]	):





       '''simple docstring'''



       __lowerCamelCase,   __lowerCamelCase   =						eval_pred
       __lowerCamelCase   =						np.argmax(snake_case__  ,			axis=1	)
       return metric.compute(predictions=snake_case__  ,			references=snake_case__	)





class     lowerCamelCase__(					UpperCAmelCase__):
      def __init__(				self: int    ,       UpperCamelCase_: List[Any]   ):
             super().__init__()
             __lowerCamelCase   =						trainer




      def 			lowerCAmelCase__						(				self: Union[str, Any]    ,       UpperCamelCase_: str    ,       UpperCamelCase_: List[str]    ,       UpperCamelCase_: Optional[Any]    ,       **UpperCamelCase_: Tuple   ):
             if control.should_evaluate:
                    __lowerCamelCase   =						deepcopy(_a   )
                    self._trainer.evaluate(eval_dataset=self._trainer.train_dataset    ,       metric_key_prefix="""train"""   )
                    return control_copy




def 					lowerCamelCase__       (						):





       '''simple docstring'''



       __lowerCamelCase   =						get_args()
       set_seed(args.seed	)

       __lowerCamelCase   =						load_dataset("""codeparrot/codecomplex"""  ,			split="""train"""	)
       __lowerCamelCase   =						dataset.train_test_split(test_size=0.2	)
       __lowerCamelCase   =						train_test["""test"""].train_test_split(test_size=0.5	)
       __lowerCamelCase   =						DatasetDict(
           {
               """train""": train_test["""train"""],
               """test""": test_validation["""train"""],
               """valid""": test_validation["""test"""],
           }	)

       print("""Loading tokenizer and model"""	)
       __lowerCamelCase   =						AutoTokenizer.from_pretrained(args.model_ckpt	)
       __lowerCamelCase   =						tokenizer.eos_token
       __lowerCamelCase   =						AutoModelForSequenceClassification.from_pretrained(args.model_ckpt  ,			num_labels=7	)
       __lowerCamelCase   =						model.config.eos_token_id

       if args.freeze:
              for param in model.roberta.parameters():
                     __lowerCamelCase   =						False

       __lowerCamelCase   =						ClassLabel(num_classes=7  ,			names=list(set(train_test_validation["""train"""]["""complexity"""]	)	)	)

       def tokenize(A__       :					int	):
              __lowerCamelCase   =						tokenizer(example["""src"""]  ,			truncation=snake_case__  ,			max_length=1024	)
              __lowerCamelCase   =						labels.straint(example["""complexity"""]	)
              return {
                  "input_ids": inputs["input_ids"],
                  "attention_mask": inputs["attention_mask"],
                  "label": label,
              }

       __lowerCamelCase   =						train_test_validation.map(
           snake_case__  ,			batched=snake_case__  ,			remove_columns=train_test_validation["""train"""].column_names  ,			)
       __lowerCamelCase   =						DataCollatorWithPadding(tokenizer=snake_case__	)

       __lowerCamelCase   =						TrainingArguments(
           output_dir=args.output_dir  ,			learning_rate=args.learning_rate  ,			lr_scheduler_type=args.lr_scheduler_type  ,			evaluation_strategy="""epoch"""  ,			save_strategy="""epoch"""  ,			logging_strategy="""epoch"""  ,			per_device_train_batch_size=args.batch_size  ,			per_device_eval_batch_size=args.batch_size  ,			num_train_epochs=args.num_epochs  ,			gradient_accumulation_steps=args.gradient_accumulation_steps  ,			weight_decay=0.01  ,			metric_for_best_model="""accuracy"""  ,			run_name="""complexity-java"""  ,			report_to="""wandb"""  ,			)

       __lowerCamelCase   =						Trainer(
           model=snake_case__  ,			args=snake_case__  ,			train_dataset=tokenized_datasets["""train"""]  ,			eval_dataset=tokenized_datasets["""valid"""]  ,			tokenizer=snake_case__  ,			data_collator=snake_case__  ,			compute_metrics=snake_case__  ,			)

       print("""Training..."""	)
       trainer.add_callback(CustomCallback(snake_case__	)	)
       trainer.train()


if __name__ == "__main__":
       main()







"""simple docstring"""




import numpy as np
import torch
from torch.utils.data import Dataset, IterableDataset

from ..utils.generic import ModelOutput






class     __magic_name__							(       UpperCAmelCase__				):






				'''simple docstring'''





				def __init__(       self		, _a		, _a		, _a ):





							"""simple docstring"""
							lowerCamelCase			      =					dataset
							lowerCamelCase			      =					process
							lowerCamelCase			      =					params





				def __len__(       self ):





							"""simple docstring"""
							return len(self.dataset )





				def __getitem__(       self		, _a ):





							"""simple docstring"""
							lowerCamelCase			      =					self.dataset[i]
							lowerCamelCase			      =					self.process(_a		, **self.params )
							return processed






class     __magic_name__							(       UpperCAmelCase__				):






				'''simple docstring'''





				def __init__(       self		, _a		, _a		, _a		, _a=None ):





							"""simple docstring"""
							lowerCamelCase			      =					loader
							lowerCamelCase			      =					infer
							lowerCamelCase			      =					params
							if loader_batch_size == 1:
										# Let's spare some time by deactivating altogether
										lowerCamelCase			      =					None
							lowerCamelCase			      =					loader_batch_size

							# Internal bookkeeping
							lowerCamelCase			      =					None
							lowerCamelCase			      =					None





				def __len__(       self ):





							"""simple docstring"""
							return len(self.loader )





				def __iter__(       self ):





							"""simple docstring"""
							lowerCamelCase			      =					iter(self.loader )
							return self





				def    _lowerCAmelCase				(       self ):





							"""simple docstring"""
							if isinstance(self._loader_batch_data		, torch.Tensor ):
										# Batch data is simple tensor, just fetch the slice
										lowerCamelCase			      =					self._loader_batch_data[self._loader_batch_index]
							else:
										# Batch data is assumed to be BaseModelOutput (or dict)
										lowerCamelCase			      =					{}
										for k, element in self._loader_batch_data.items():
													if isinstance(_a		, _a ):
																# Convert ModelOutput to tuple first
																lowerCamelCase			      =					element.to_tuple()
																if isinstance(element[0]		, torch.Tensor ):
																			lowerCamelCase			      =					tuple(el[self._loader_batch_index].unsqueeze(0 ) for el in element )
																elif isinstance(element[0]		, np.ndarray ):
																			lowerCamelCase			      =					tuple(np.expand_dims(el[self._loader_batch_index]		, 0 ) for el in element )
																continue
													if k in {"hidden_states", "past_key_values", "attentions"} and isinstance(_a		, _a ):
																# Those are stored as lists of tensors so need specific unbatching.
																if isinstance(element[0]		, torch.Tensor ):
																			lowerCamelCase			      =					tuple(el[self._loader_batch_index].unsqueeze(0 ) for el in element )
																elif isinstance(element[0]		, np.ndarray ):
																			lowerCamelCase			      =					tuple(np.expand_dims(el[self._loader_batch_index]		, 0 ) for el in element )
																continue
													if element is None:
																# This can happen for optional data that get passed around
																lowerCamelCase			      =					None
													elif isinstance(element[self._loader_batch_index]		, torch.Tensor ):
																# Take correct batch data, but make it looked like batch_size=1
																# For compatibility with other methods within transformers

																lowerCamelCase			      =					element[self._loader_batch_index].unsqueeze(0 )
													elif isinstance(element[self._loader_batch_index]		, np.ndarray ):
																# Take correct batch data, but make it looked like batch_size=1
																# For compatibility with other methods within transformers
																lowerCamelCase			      =					np.expand_dims(element[self._loader_batch_index]		, 0 )
													else:
																# This is typically a list, so no need to `unsqueeze`.
																lowerCamelCase			      =					element[self._loader_batch_index]
            # Recreate the element by reusing the original class to make it look
            # batch_size=1
										lowerCamelCase			      =					self._loader_batch_data.__class__(_a )
							self._loader_batch_index += 1
							return result





				def    _lowerCAmelCase				(       self ):





							"""simple docstring"""
							if self._loader_batch_index is not None and self._loader_batch_index < self.loader_batch_size:
										# We are currently unrolling a batch so we just need to return
										# the current item within a batch
										return self.loader_batch_item()

							# We're out of items within a batch
							lowerCamelCase			      =					next(self.iterator )
							lowerCamelCase			      =					self.infer(_a		, **self.params )
							# We now have a batch of "inferred things".
							if self.loader_batch_size is not None:
										# Try to infer the size of the batch
										if isinstance(_a		, torch.Tensor ):
													lowerCamelCase			      =					processed
										else:
													lowerCamelCase			      =					list(processed.keys() )[0]
													lowerCamelCase			      =					processed[key]
										if isinstance(_a		, _a ):
													lowerCamelCase			      =					len(_a )
										else:
													lowerCamelCase			      =					first_tensor.shape[0]
										if 0 < observed_batch_size < self.loader_batch_size:
													# could be last batch so we can't unroll as many
													# elements.
													lowerCamelCase			      =					observed_batch_size
										# Setting internal index to unwrap the batch
										lowerCamelCase			      =					processed
										lowerCamelCase			      =					0
										return self.loader_batch_item()
							else:
										# We're not unrolling batches
										return processed






class     __magic_name__							(       UpperCAmelCase__				):






				'''simple docstring'''





				def __init__(       self		, _a		, _a		, _a		, _a=None ):





							"""simple docstring"""
							super().__init__(_a		, _a		, _a )





				def __iter__(       self ):





							"""simple docstring"""
							lowerCamelCase			      =					iter(self.loader )
							lowerCamelCase			      =					None
							return self





				def    _lowerCAmelCase				(       self ):





							"""simple docstring"""
							if self.subiterator is None:
										lowerCamelCase			      =					self.infer(next(self.iterator )		, **self.params )
							try:
										# Try to return next item
										lowerCamelCase			      =					next(self.subiterator )
							except StopIteration:
										# When a preprocess iterator ends, we can start lookig at the next item
										# ChunkIterator will keep feeding until ALL elements of iterator
										# all have created their subiterator and have been iterating against.
										#
										# Another way to look at it, is we're basically flattening lists of lists
										# into a single list, but with generators
										lowerCamelCase			      =					self.infer(next(self.iterator )		, **self.params )
										lowerCamelCase			      =					next(self.subiterator )
							return processed






class     __magic_name__							(       UpperCAmelCase__				):






				'''simple docstring'''





				def __iter__(       self ):





							"""simple docstring"""
							lowerCamelCase			      =					iter(self.loader )
							return self





				def    _lowerCAmelCase				(       self ):





							"""simple docstring"""
							# Extremely similar to PipelineIterator in its unpacking mechanism
							# BUT, we have an extra required item which is the presence of `is_last`
							# That is because everything is flattened by `PipelineChunkIterator` we
							# need to keep track of how to regroup here in the original `process`
							# boundaries so that `process` and `postprocess` see the same data.

							# This iterator accumulates items (possibly while unbatching) until it
							# its a `is_last` and then just passes it on to the caller.
							lowerCamelCase			      =					False
							lowerCamelCase			      =					[]
							if self._loader_batch_index is not None and self._loader_batch_index < self.loader_batch_size:
										while self._loader_batch_index < self.loader_batch_size:
													lowerCamelCase			      =					self.loader_batch_item()
													lowerCamelCase			      =					item.pop("""is_last""" )
													accumulator.append(_a )
													if is_last:
																return accumulator

							while not is_last:
										lowerCamelCase			      =					self.infer(next(self.iterator )		, **self.params )
										if self.loader_batch_size is not None:
													if isinstance(_a		, torch.Tensor ):
																lowerCamelCase			      =					processed
													else:
																lowerCamelCase			      =					list(processed.keys() )[0]
																lowerCamelCase			      =					processed[key]
													if isinstance(_a		, _a ):
																lowerCamelCase			      =					len(_a )
													else:
																lowerCamelCase			      =					first_tensor.shape[0]
													if 0 < observed_batch_size < self.loader_batch_size:
																# could be last batch so we can't unroll as many
																# elements.
																lowerCamelCase			      =					observed_batch_size
													lowerCamelCase			      =					processed
													lowerCamelCase			      =					0
													while self._loader_batch_index < self.loader_batch_size:
																lowerCamelCase			      =					self.loader_batch_item()
																lowerCamelCase			      =					item.pop("""is_last""" )
																accumulator.append(_a )
																if is_last:
																			return accumulator
										else:
													lowerCamelCase			      =					processed
													lowerCamelCase			      =					item.pop("""is_last""" )
													accumulator.append(_a )
							return accumulator






class     __magic_name__							(       UpperCAmelCase__				):






				'''simple docstring'''





				def __init__(       self		, _a		, _a ):





							"""simple docstring"""
							lowerCamelCase			      =					dataset
							lowerCamelCase			      =					key





				def __len__(       self ):





							"""simple docstring"""
							return len(self.dataset )





				def __getitem__(       self		, _a ):





							"""simple docstring"""
							return self.dataset[i][self.key]






class     __magic_name__							(       UpperCAmelCase__				):






				'''simple docstring'''





				def __init__(       self		, _a		, _a		, _a ):





							"""simple docstring"""
							lowerCamelCase			      =					dataset
							lowerCamelCase			      =					keya
							lowerCamelCase			      =					keya





				def __len__(       self ):





							"""simple docstring"""
							return len(self.dataset )





				def __getitem__(       self		, _a ):





							"""simple docstring"""
							return {"text": self.dataset[i][self.keya], "text_pair": self.dataset[i][self.keya]}



