





import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


snake_case__					:							str													=     ['\nclass', '\ndef', '\n#', '\n@', '\nprint', '\nif']





class A_					(			_lowerCamelCase   ):




   def __init__(self	:Optional[int]    ,   _UpperCamelCase	:int    ,   _UpperCamelCase	:int    ,   _UpperCamelCase	:str=None    ,   _UpperCamelCase	:List[str]=1 )->					Optional[Any]:
         __A          =				tokenizer
         __A          =				dataset
         __A          =				len(_UpperCamelCase ) if n_tasks is None else n_tasks
         __A          =				n_copies






   def __iter__(self	:Union[str, Any] )->					Union[str, Any]:
         __A          =				[]
         for task in range(self.n_tasks ):
               # without strip, the model generate commented codes ...
               prompts.append(self.tokenizer.eos_token + self.dataset[task]['''prompt'''].strip() )
         __A          =				self.tokenizer(_UpperCamelCase    ,   padding=_UpperCamelCase    ,   return_tensors='''pt''' )
         for task in range(self.n_tasks ):
               for _ in range(self.n_copies ):
                     yield {
                         "ids": outputs.input_ids[task],
                         "task_id": task,
                         "input_len": outputs.attention_mask[task].sum(),
                     }





class A_					(			_lowerCamelCase   ):




   def __init__(self	:Dict    ,   _UpperCamelCase	:str    ,   _UpperCamelCase	:Optional[Any]    ,   _UpperCamelCase	:int )->					str:
         __A          =				start_length
         __A          =				eof_strings
         __A          =				tokenizer






   def __call__(self	:Dict    ,   _UpperCamelCase	:Any    ,   _UpperCamelCase	:Tuple    ,   **_UpperCamelCase	:Dict )->					List[Any]:
         __A          =				self.tokenizer.batch_decode(input_ids[:, self.start_length :] )
         __A          =				[]
         for decoded_generation in decoded_generations:
               done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings ) )
         return all(_UpperCamelCase )





def _a				(	lowerCamelCase:   List[Any] )	->						Tuple:







      '''simple docstring'''






      __A          =				re.split('''(%s)''' % '''|'''.join(lowerCamelCase )					,				lowerCamelCase )
      # last string should be ""
      return "".join(string_list[:-2] )



def _a				(	lowerCamelCase:   Dict					,				lowerCamelCase:   Any					,				lowerCamelCase:   Tuple					,				lowerCamelCase:   str					,				lowerCamelCase:   Dict					,				lowerCamelCase:   Union[str, Any]=20					,				**lowerCamelCase:   List[Any] )	->						List[str]:







      '''simple docstring'''






      __A          =				defaultdict(lowerCamelCase )  # dict of list of generated tokens
      for step, batch in tqdm(enumerate(lowerCamelCase ) ):
            with torch.no_grad():
                  __A          =				batch['''ids'''].shape[-1]
                  __A          =				accelerator.unwrap_model(lowerCamelCase ).generate(
                      input_ids=batch['''ids'''][:, : batch['''input_len''']]					,				num_return_sequences=lowerCamelCase					,				**lowerCamelCase )
                  # each task is generated batch_size times
                  __A          =				batch['''task_id'''].repeat(lowerCamelCase )
                  __A          =				accelerator.pad_across_processes(
                      lowerCamelCase					,				dim=1					,				pad_index=tokenizer.pad_token_id )

                  __A   ,						__A          =				accelerator.gather((generated_tokens, generated_tasks) )
                  __A          =				generated_tokens.cpu().numpy()
                  __A          =				generated_tasks.cpu().numpy()

                  for task, generated_tokens in zip(lowerCamelCase					,				lowerCamelCase ):
                        gen_token_dict[task].append(lowerCamelCase )

      __A          =				[[] for _ in range(lowerCamelCase )]
      for task, generated_tokens in gen_token_dict.items():
            for s in generated_tokens:
                  __A          =				tokenizer.decode(lowerCamelCase					,				skip_special_tokens=lowerCamelCase					,				clean_up_tokenization_spaces=lowerCamelCase )
                  code_gens[task].append(remove_last_block(lowerCamelCase ) )
      return code_gens



def _a				(	)	->						List[str]:







      '''simple docstring'''






      __A          =				HfArgumentParser(lowerCamelCase )
      __A          =				parser.parse_args()

      transformers.logging.set_verbosity_error()
      # enables code execution in code_eval metric
      __A          =				args.HF_ALLOW_CODE_EVAL
      # make sure tokenizer plays nice with multiprocessing
      __A          =				'''false'''

      if args.num_workers is None:
            __A          =				multiprocessing.cpu_count()

      # Use dataset load to feed to accelerate
      __A          =				Accelerator()
      set_seed(args.seed					,				device_specific=lowerCamelCase )

      # Load model and tokenizer
      __A          =				AutoTokenizer.from_pretrained(args.model_ckpt )
      __A          =				tokenizer.eos_token
      __A          =				AutoModelForCausalLM.from_pretrained(args.model_ckpt )

      # Generation settings
      __A          =				{
          '''do_sample''': args.do_sample,
          '''temperature''': args.temperature,
          '''max_new_tokens''': args.max_new_tokens,
          '''top_p''': args.top_p,
          '''top_k''': args.top_k,
          '''stopping_criteria''': StoppingCriteriaList([EndOfFunctionCriteria(0					,				lowerCamelCase					,				lowerCamelCase )] ),
      }

      # Load evaluation dataset and metric
      __A          =				load_dataset('''openai_humaneval''' )
      __A          =				load_metric('''code_eval''' )

      __A          =				args.num_tasks if args.num_tasks is not None else len(human_eval['''test'''] )
      __A          =				args.n_samples // args.batch_size

      __A          =				TokenizedDataset(lowerCamelCase					,				human_eval['''test''']					,				n_copies=lowerCamelCase					,				n_tasks=lowerCamelCase )
      # do not confuse args.batch_size, which is actually the num_return_sequences
      __A          =				DataLoader(lowerCamelCase					,				batch_size=1 )

      # Run a quick test to see if code evaluation is enabled
      try:
            __A          =				code_eval_metric.compute(references=['''''']					,				predictions=[['''''']] )
      except ValueError as exception:
            print(
                '''Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL="1"`'''
                ''' flag to enable code evaluation.''' )
            raise exception

      __A   ,						__A          =				accelerator.prepare(lowerCamelCase					,				lowerCamelCase )

      __A          =				complete_code(
          lowerCamelCase					,				lowerCamelCase					,				lowerCamelCase					,				lowerCamelCase					,				n_tasks=lowerCamelCase					,				batch_size=args.batch_size					,				**lowerCamelCase					,				)

      if accelerator.is_main_process:
            __A          =				[]

            for task in tqdm(range(lowerCamelCase ) ):
                  __A          =				human_eval['''test'''][task]['''test''']
                  __A          =				F"""check({human_eval['test'][task]['entry_point']})"""
                  references.append('''\n''' + test_func + '''\n''' + entry_point )

            # Evaluate completions with "code_eval" metric
            __A   ,						__A          =				code_eval_metric.compute(
                references=lowerCamelCase					,				predictions=lowerCamelCase					,				num_workers=args.num_workers )
            print(F"""Results: {pass_at_k}""" )

            # Save results to json file
            with open(args.output_file					,				'''w''' ) as fp:
                  json.dump(lowerCamelCase					,				lowerCamelCase )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
       main()












import numpy as np
from PIL import Image



def _a				(	lowerCamelCase:   np.ndarray					,				lowerCamelCase:   int					,				lowerCamelCase:   int )	->						np.ndarray:







      '''simple docstring'''






      __A          =				np.array(lowerCamelCase )
      if arr.shape[0] != arr.shape[1]:
            raise ValueError('''The input array is not a square matrix''' )
      __A          =				0
      __A          =				0
      __A          =				0
      __A          =				0

      # compute the shape of the output matrix
      __A          =				(arr.shape[0] - size) // stride + 1
      # initialize the output matrix with zeros of shape maxpool_shape
      __A          =				np.zeros((maxpool_shape, maxpool_shape) )

      while i < arr.shape[0]:
            if i + size > arr.shape[0]:
                  # if the end of the matrix is reached, break
                  break
            while j < arr.shape[1]:
                  # if the end of the matrix is reached, break
                  if j + size > arr.shape[1]:
                        break
                  # compute the maximum of the pooling matrix
                  __A          =				np.max(arr[i : i + size, j : j + size] )
                  # shift the pooling matrix by stride of column pixels
                  j += stride
                  mat_j += 1

            # shift the pooling matrix by stride of row pixels
            i += stride
            mat_i += 1

            # reset the column index to 0
            __A          =				0
            __A          =				0

      return updated_arr



def _a				(	lowerCamelCase:   np.ndarray					,				lowerCamelCase:   int					,				lowerCamelCase:   int )	->						np.ndarray:







      '''simple docstring'''






      __A          =				np.array(lowerCamelCase )
      if arr.shape[0] != arr.shape[1]:
            raise ValueError('''The input array is not a square matrix''' )
      __A          =				0
      __A          =				0
      __A          =				0
      __A          =				0

      # compute the shape of the output matrix
      __A          =				(arr.shape[0] - size) // stride + 1
      # initialize the output matrix with zeros of shape avgpool_shape
      __A          =				np.zeros((avgpool_shape, avgpool_shape) )

      while i < arr.shape[0]:
            # if the end of the matrix is reached, break
            if i + size > arr.shape[0]:
                  break
            while j < arr.shape[1]:
                  # if the end of the matrix is reached, break
                  if j + size > arr.shape[1]:
                        break
                  # compute the average of the pooling matrix
                  __A          =				int(np.average(arr[i : i + size, j : j + size] ) )
                  # shift the pooling matrix by stride of column pixels
                  j += stride
                  mat_j += 1

            # shift the pooling matrix by stride of row pixels
            i += stride
            mat_i += 1
            # reset the column index to 0
            __A          =				0
            __A          =				0

      return updated_arr


# Main Function
if __name__ == "__main__":
       from doctest import testmod

       testmod(name='avgpooling', verbose=True)

       # Loading the image
       snake_case__					:							Optional[Any]													=     Image.open('path_to_image')

       # Converting the image to numpy array and maxpooling, displaying the result
       # Ensure that the image is a square matrix

       Image.fromarray(maxpooling(np.array(image), size=3, stride=2)).show()

       # Converting the image to numpy array and averagepooling, displaying the result
       # Ensure that the image is a square matrix

       Image.fromarray(avgpooling(np.array(image), size=3, stride=2)).show()





