from typing import List # this try/except is important for publishing to Hugging Face try: from dagster import Config except ImportError: Config = object class LLMBoardConfig(Config): group_columns: List[str] = ["model", "language", "template_name"] single_values_columns: List[str] = ["execution_time", "characters_count", "words_count"] list_columns: List[str] = ["chunk_sizes", "chunk_generation_times", "chunk_generation_times_by_chunk_sizes"] plot_dir: str = "./html/plots/" plot_json_dir: str = "./data/" saving_path: str = "data/" class QueriesConfig(Config): base_query_template: str = """Summarize me this text, the summary should be in {language} ``` {text} ``` """ query_template: dict = { "markdown": """Return output as markdown""", "json": """Return output as json in format: { "summary": " }""", "call": """Return output by calling summary_result()""", } class MeasurementsConfig(Config): mock: bool = False remove_old_measurements: bool = False small_dataset: bool = False class QueriesDatasetConfig(Config): dataset_name: str = "GEM/xlsum" samples_per_measurement: int = 20 languages: List[str] = ["english", "japanese", "ukrainian"] query_config: QueriesConfig = QueriesConfig() class SummaryConfig(Config): saving_path: str = "data/" class ModelCostsConfig(Config): saving_path: str = "data/" class TimeOfDayComparisonConfig(Config): plots_dir: str = "./html/plots/" saving_path: str = "data/" quantiles: List[float] = [0.2, 0.4, 0.6, 0.8] class GeneralPlotConfig(Config): plots_dir: str = "./html/plots/" saving_path: str = "data/" seconds_per_token: float = 184 / 6 input_size: int = 100 expected_output_size: int = 50 class CombinedPlotsConfig(Config): plots_dir: str = "./html/plots/" saving_path: str = "data/" scatter_plots: bool = True class SummaryMetricsConfig(Config): combined_score: bool = False