




import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


__magic_name__  :  List[Any]					=       ["""\nclass""", """\ndef""", """\n#""", """\n@""", """\nprint""", """\nif"""]





class     SCREAMING_SNAKE_CASE__		(_a			):
     def __init__(						self	:		Union[str, Any]       ,						__lowerCamelCase	:		Optional[Any]       ,						__lowerCamelCase	:		Dict       ,						__lowerCamelCase	:		Optional[int]=None       ,						__lowerCamelCase	:		Union[str, Any]=1						):



       """simple docstring"""
       lowerCAmelCase__										=      tokenizer
       lowerCAmelCase__										=      dataset
       lowerCAmelCase__										=      len(__lowerCamelCase						) if n_tasks is None else n_tasks
       lowerCAmelCase__										=      n_copies



     def __iter__(						self	:		Union[str, Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      []
       for task in range(self.n_tasks						):
         # without strip, the model generate commented codes ...
         prompts.append(self.tokenizer.eos_token + self.dataset[task]['''prompt'''].strip()						)
       lowerCAmelCase__										=      self.tokenizer(__lowerCamelCase       ,						padding=__lowerCamelCase       ,						return_tensors='''pt'''						)
       for task in range(self.n_tasks						):
         for _ in range(self.n_copies						):
           yield {
               "ids": outputs.input_ids[task],
               "task_id": task,
               "input_len": outputs.attention_mask[task].sum(),
           }





class     SCREAMING_SNAKE_CASE__		(_a			):
     def __init__(						self	:		List[Any]       ,						__lowerCamelCase	:		List[Any]       ,						__lowerCamelCase	:		Optional[Any]       ,						__lowerCamelCase	:		str						):



       """simple docstring"""
       lowerCAmelCase__										=      start_length
       lowerCAmelCase__										=      eof_strings
       lowerCAmelCase__										=      tokenizer



     def __call__(						self	:		Dict       ,						__lowerCamelCase	:		Optional[int]       ,						__lowerCamelCase	:		Dict       ,						**__lowerCamelCase	:		str						):



       """simple docstring"""
       lowerCAmelCase__										=      self.tokenizer.batch_decode(input_ids[:, self.start_length :]						)
       lowerCAmelCase__										=      []
       for decoded_generation in decoded_generations:
         done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings						)						)
       return all(__lowerCamelCase						)



def 		a_							( __lowerCAmelCase				):
  lowerCAmelCase__										=      re.split('''(%s)''' % '''|'''.join(__lowerCAmelCase				)   , __lowerCAmelCase				)
  # last string should be ""
  return "".join(string_list[:-2]				)
def 		a_							( __lowerCAmelCase   , __lowerCAmelCase   , __lowerCAmelCase   , __lowerCAmelCase   , __lowerCAmelCase   , __lowerCAmelCase=20   , **__lowerCAmelCase				):
  lowerCAmelCase__										=      defaultdict(__lowerCAmelCase				)  # dict of list of generated tokens
  for step, batch in tqdm(enumerate(__lowerCAmelCase				)				):
    with torch.no_grad():
      lowerCAmelCase__										=      batch['''ids'''].shape[-1]
      lowerCAmelCase__										=      accelerator.unwrap_model(__lowerCAmelCase				).generate(
          input_ids=batch['''ids'''][:, : batch['''input_len''']]   , num_return_sequences=__lowerCAmelCase   , **__lowerCAmelCase				)
      # each task is generated batch_size times
      lowerCAmelCase__										=      batch['''task_id'''].repeat(__lowerCAmelCase				)
      lowerCAmelCase__										=      accelerator.pad_across_processes(
          __lowerCAmelCase   , dim=1   , pad_index=tokenizer.pad_token_id				)

      lowerCAmelCase__				,      lowerCAmelCase__										=      accelerator.gather((generated_tokens, generated_tasks)				)
      lowerCAmelCase__										=      generated_tokens.cpu().numpy()
      lowerCAmelCase__										=      generated_tasks.cpu().numpy()

      for task, generated_tokens in zip(__lowerCAmelCase   , __lowerCAmelCase				):
        gen_token_dict[task].append(__lowerCAmelCase				)

  lowerCAmelCase__										=      [[] for _ in range(__lowerCAmelCase				)]
  for task, generated_tokens in gen_token_dict.items():
    for s in generated_tokens:
      lowerCAmelCase__										=      tokenizer.decode(__lowerCAmelCase   , skip_special_tokens=__lowerCAmelCase   , clean_up_tokenization_spaces=__lowerCAmelCase				)
      code_gens[task].append(remove_last_block(__lowerCAmelCase				)				)
  return code_gens
def 		a_							( ):
  # Setup configuration
  lowerCAmelCase__										=      HfArgumentParser(__lowerCAmelCase				)
  lowerCAmelCase__										=      parser.parse_args()

  transformers.logging.set_verbosity_error()
  # enables code execution in code_eval metric
  lowerCAmelCase__										=      args.HF_ALLOW_CODE_EVAL
  # make sure tokenizer plays nice with multiprocessing
  lowerCAmelCase__										=      '''false'''

  if args.num_workers is None:
    lowerCAmelCase__										=      multiprocessing.cpu_count()

  # Use dataset load to feed to accelerate
  lowerCAmelCase__										=      Accelerator()
  set_seed(args.seed   , device_specific=__lowerCAmelCase				)

  # Load model and tokenizer
  lowerCAmelCase__										=      AutoTokenizer.from_pretrained(args.model_ckpt				)
  lowerCAmelCase__										=      tokenizer.eos_token
  lowerCAmelCase__										=      AutoModelForCausalLM.from_pretrained(args.model_ckpt				)

  # Generation settings
  lowerCAmelCase__										=      {
      '''do_sample''': args.do_sample,
      '''temperature''': args.temperature,
      '''max_new_tokens''': args.max_new_tokens,
      '''top_p''': args.top_p,
      '''top_k''': args.top_k,
      '''stopping_criteria''': StoppingCriteriaList([EndOfFunctionCriteria(0   , __lowerCAmelCase   , __lowerCAmelCase				)]				),
  }

  # Load evaluation dataset and metric
  lowerCAmelCase__										=      load_dataset('''openai_humaneval'''				)
  lowerCAmelCase__										=      load_metric('''code_eval'''				)

  lowerCAmelCase__										=      args.num_tasks if args.num_tasks is not None else len(human_eval['''test''']				)
  lowerCAmelCase__										=      args.n_samples // args.batch_size

  lowerCAmelCase__										=      TokenizedDataset(__lowerCAmelCase   , human_eval['''test''']   , n_copies=__lowerCAmelCase   , n_tasks=__lowerCAmelCase				)
  # do not confuse args.batch_size, which is actually the num_return_sequences
  lowerCAmelCase__										=      DataLoader(__lowerCAmelCase   , batch_size=1				)

  # Run a quick test to see if code evaluation is enabled
  try:
    lowerCAmelCase__										=      code_eval_metric.compute(references=['''''']   , predictions=[['''''']]				)
  except ValueError as exception:
    print(
        '''Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL="1"`'''
        ''' flag to enable code evaluation.'''				)
    raise exception

  lowerCAmelCase__				,      lowerCAmelCase__										=      accelerator.prepare(__lowerCAmelCase   , __lowerCAmelCase				)

  lowerCAmelCase__										=      complete_code(
      __lowerCAmelCase   , __lowerCAmelCase   , __lowerCAmelCase   , __lowerCAmelCase   , n_tasks=__lowerCAmelCase   , batch_size=args.batch_size   , **__lowerCAmelCase   , )

  if accelerator.is_main_process:
    lowerCAmelCase__										=      []

    for task in tqdm(range(__lowerCAmelCase				)				):
      lowerCAmelCase__										=      human_eval['''test'''][task]['''test''']
      lowerCAmelCase__										=      F"""check({human_eval['test'][task]['entry_point']})"""
      references.append('''\n''' + test_func + '''\n''' + entry_point				)

    # Evaluate completions with "code_eval" metric
    lowerCAmelCase__				,      lowerCAmelCase__										=      code_eval_metric.compute(
        references=__lowerCAmelCase   , predictions=__lowerCAmelCase   , num_workers=args.num_workers				)
    print(F"""Results: {pass_at_k}"""				)

    # Save results to json file
    with open(args.output_file   , '''w'''				) as fp:
      json.dump(__lowerCAmelCase   , __lowerCAmelCase				)


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
      main()







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__magic_name__  :  List[str]					=       {
    """configuration_m2m_100""": ["""M2M_100_PRETRAINED_CONFIG_ARCHIVE_MAP""", """M2M100Config""", """M2M100OnnxConfig"""],
    """tokenization_m2m_100""": ["""M2M100Tokenizer"""],
}


try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __magic_name__  :  Union[str, Any]					=       [
          """M2M_100_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """M2M100ForConditionalGeneration""",
          """M2M100Model""",
          """M2M100PreTrainedModel""",
      ]


if TYPE_CHECKING:
      from .configuration_mam_aaa import M2M_100_PRETRAINED_CONFIG_ARCHIVE_MAP, MaMaaaConfig, MaMaaaOnnxConfig
      from .tokenization_mam_aaa import MaMaaaTokenizer

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_mam_aaa import (
                M2M_100_PRETRAINED_MODEL_ARCHIVE_LIST,
                MaMaaaForConditionalGeneration,
                MaMaaaModel,
                MaMaaaPreTrainedModel,
            )


else:
      import sys

      __magic_name__  :  List[str]					=       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

