implemnting a fix for santacoder

ShaderEval.py

@@ -28,7 +28,7 @@ from evaluate.evaluation_suite import SubTask
 from datasets import Dataset
 from typing import Any, Callable, Dict, List, Optional, Union # used in .prepare_pipeline()
 import transformers
-from transformers import Pipeline, pipeline, GenerationConfig #GenerationConfig to specify greedy and avoid error
+from transformers import Pipeline, pipeline, GenerationConfig, AutoTokenizer #GenerationConfig to specify greedy and avoid error
 from datasets import load_dataset #used by Suite.run()
 
 # write a custom evaluator, inherent from: https://github.com/huggingface/evaluate/blob/v0.4.0/src/evaluate/evaluator/text_generation.py#L31
@@ -76,6 +76,9 @@ class ReturnGenerationEvaluator(evaluate.TextGenerationEvaluator):
             or isinstance(model_or_pipeline, transformers.PreTrainedModel)
             or isinstance(model_or_pipeline, transformers.TFPreTrainedModel)
         ):
+            # load tokenizer manually, since the pipeline does fail to do so at times. needed for bigcode/santacoder for example.
+            tokenizer = AutoTokenizer.from_pretrained(model_or_pipeline)
+            
             pipe = pipeline(
                 self.task,
                 model=model_or_pipeline,
@@ -126,7 +129,7 @@ class ReturnGenerationEvaluator(evaluate.TextGenerationEvaluator):
         if isinstance(model_or_pipeline, transformers.GPT2Model): # you are comparing a string here -.-
             return model_or_pipeline.config.n_ctx # how GPT2 models might handle is, seen with 
         if pipe is not None: #should I figure out a way to pass this.
-            pipe.tokenizer.model_max_length # this is set to something small for pipeline default task, but we would want to put it to the max instead.
+            return pipe.tokenizer.model_max_length # this is set to something small for pipeline default task, but we would want to put it to the max instead.
         # tokenizer needs to know the context length for our pipe strategy, but it has to be passed to the tokenizer, not model.
         # the tokenizer should read from the model config, but that can be wrong, or it has a task overwrite (for "text-generation" for example you get 50)
         #model_or_pipeline only exists via the .compute call, so we have to take it in
@@ -143,7 +146,7 @@ class ReturnGenerationEvaluator(evaluate.TextGenerationEvaluator):
             `int`: the estimated max_new_tokens, should be smaller than context_lenght in all cases
         """
         context_lenght = self._resolve_context_lenght(**kwargs)
-        estimate = min(max([len(ref) for ref in labels]) + 5, context_lenght)
+        estimate = min(max([len(ref) for ref in labels]) + 5, context_lenght) #does the min call get done inside the pipeline anyway? is there even a single case where the return statement is this long?
         return estimate
 
     # this one needs to be adjusted