expanded readme portion in the markdown.

ShaderEval.py

@@ -201,6 +201,7 @@ class Suite(evaluate.EvaluationSuite):
             args_for_task["split"] = (task.split + f"[:{snippet}]") #make a downselection of the split via keywordarg in the .run() call?
             results = task_evaluator.compute(**args_for_task)
 
+            results["model_cp"] = model_or_pipeline #added this to the output, should be useful. But be careful when passed something that is not a string. #TODO: currently the same for all tasks, maybe move to the list?
             results["task_name"] = task_name + "/" + task.subset if task.subset else task_name
             results["data_preprocessor"] = str(task.data_preprocessor) if task.data_preprocessor is not None else None
             results_all.append(results)

app.py

@@ -10,13 +10,23 @@ suite = evaluate.EvaluationSuite.load("Vipitis/ShaderEval") #downloads it
 def run_suite(model_cp, snippet):
     # print(model_cp, snippet)
     results = suite.run(model_cp, snippet)
-    return results[0]["exact_match"]
+    return results[0]
 
 with gr.Blocks() as site:
     text = gr.Markdown("""# Welcome to the ShaderEval Suite.
             
             This space hosts the ShaderEval Suite. more to follow soon.
             
+            # Task1: Return Completion
+            ## Explanation
+            Modelled after the [CodeXGLUE code_completion_line](https://huggingface.co/datasets/code_x_glue_cc_code_completion_line) task. 
+            Using the "return_completion" subset of the [Shadertoys-fine dataset](https://huggingface.co/datasets/Vipits/Shadertoys-fine).
+            All preprocessing and post proessing is done by the custom evaluator for this suite. It should be as easy as just giving it a model checkpoint that can do the "text-generation" task.
+            Evaluated is currently with just [exact_match](https://huggingface.co/metrics/exact_match).
+
+            ## Notice
+            should you find any model that throws an error, please let me know in the issues tab. Several parts of this suite are still missing.
+
             ## Instructions
             ### Run the code yourself:.
             ```python
@@ -31,9 +41,16 @@ with gr.Blocks() as site:
             - Select how many **samples** to run (there us up to 100 from the test set)
             - Click **Run** to run the suite
             - The results will be displayed in the **Output** box
-            """)
+            
+            ## Todo (feel free to contribute in a Pull Request)
+            - leaderboard 
+            - supporting batches to speed up inference
+            - CER metric 
+            - removing the pad_token warning
+            """
+            )
     model_cp = gr.Textbox(value="gpt2", label="Model Checkpoint", interactive=True)
-    first_n = gr.Slider(minimum=1, maximum=100, default=10, label="num_samples", step=1.0)
+    first_n = gr.Slider(minimum=1, maximum=100, default=15, label="num_samples", step=1.0)
     output = gr.Textbox(label="Output")
     run_button = gr.Button(label="Run")
     run_button.click(fn=run_suite, inputs=[model_cp, first_n], outputs=output)