app.py

import gradio as gr
import evaluate
from ShaderEval import Suite

# suite = evaluate.EvaluationSuite.load("Vipitis/ShaderEval") #downloads it
suite = Suite("Vipitis/ShaderEval") #local to the space?

def run_suite(model_cp, snippet):
    suite = Suite("Vipitis/ShaderEval", snippet)
    results = suite.run(model_cp)
    return results[0]["exact_match"]

with gr.Blocks() as site:
    text = gr.Markdown("""# Welcome to the ShaderEval Suite.
            
            This space hosts the ShaderEval Suite. more to follow soon.
            
            ## Instructions
            ### Run the code yourself:.
            ```python
            import evaluate
            suite = evaluate.EvaluationSuite.load("Vipitis/ShaderEval")
            model_cp = "gpt2"
            suite.run("model_cp")
            ```
            
            ### try the demo below
            - Select a **model checkpoint** from the "dropdown"
            - Select how many **samples** to run (there us up to 100 from the test set)
            - Click **Run** to run the suite
            - The results will be displayed in the **Output** box
            """)
    model_cp = gr.Textbox(label="Model Checkpoint")
    first_n = gr.Slider(minimum=1, maximum=100, default=10, label="num_samples", step=1.0)
    output = gr.Textbox(label="Output")
    run_button = gr.Button(label="Run")
    run_button.click(fn=run_suite, inputs=[model_cp, first_n], outputs=output)
site.launch()