<jupyter_start><jupyter_text>Allocation quality vs. runtime for various configurations In this notebook, we evaluate the impact of splitting larger optimization problems into smaller sub-problems on policy runtime and quality of solution for max-min fairness policies.  Import statements and plotting code<jupyter_code>from matplotlib import pyplot as plt
import matplotlib
from matplotlib.backends.backend_pdf import PdfPages
import matplotlib.lines as mlines
import matplotlib.patches as mpatches
from matplotlib.ticker import MultipleLocator
import seaborn as sns
from matplotlib import rc
import numpy as np
sns.set_style('ticks')
font = {
    'font.family':'Roboto',
    'font.size': 12,
}
sns.set_style(font)
paper_rc = {
    'lines.linewidth': 3,
    'lines.markersize': 10,
}
sns.set_context("paper", font_scale=2, rc=paper_rc)
current_palette = sns.color_palette()
def plot_runtime_vs_effective_throughput_ratios(runtimes, all_effective_throughputs, labels,
                                                draw_arrow=False):
    plt.figure(figsize=(7.5, 2.5))
    ax = plt.subplot2grid((1, 1), (0, 0), colspan=1)
    job_ids = list(all_effective_throughputs[0].keys())
    for (runtime, effective_throughputs, label) in zip(
        runtimes,
        all_effective_throughputs,
        labels):
        effective_throughput_ratios = np.array([
            effective_throughputs[job_id] / all_effective_throughputs[0][job_id]
            for job_id in job_ids])
        mean = np.mean(effective_throughput_ratios)
        print(label, runtime, mean, np.std(effective_throughput_ratios))
        ax.scatter(runtime, mean, label=label)
        ax.errorbar(runtime, mean, np.std(effective_throughput_ratios))
        if label == "Exact sol.":
            ax.annotate(label, (runtime*0.7, mean-0.18))
        elif label == "POP-4":
            ax.annotate(label, (runtime*1.1, mean-0.04))
        elif label == "POP-8":
            ax.annotate(label, (runtime*0.9, mean-0.2))
        else:
            ax.annotate(label, (runtime*1.05, mean-0.18))

    ax.set_ylabel("Allocation quality")
    ax.set_xlabel("Runtime (seconds)")
    sns.despine()
    
    ax.set_xscale('log')
    ax.set_yticks([0.0, 0.25, 0.5, 0.75, 1.0])
    xmin, xmax = plt.xlim()
    ymin, ymax = plt.ylim()
    plt.xlim(10, xmax*1.2)
    plt.ylim(0, ymax*1.15)
    
    if draw_arrow:
        bbox_props = dict(boxstyle="larrow", ec="k", lw=2, fc="white")
        t = ax.text(20, 0.4, "Better", ha="center", va="center", rotation=-45,
                    bbox=bbox_props)
        bb = t.get_bbox_patch()
        bb.set_boxstyle("larrow", pad=0.3)
    
    plt.show()
def plot_runtime_vs_metrics(runtimes, metrics,
                            metric_label,
                            yticks,
                            label_locations=None,
                            errorbars=False,
                            xscale_in_log=False):
    plt.figure(figsize=(6.5, 2.5))
    ax = plt.subplot2grid((1, 1), (0, 0), colspan=1)
    labels = list(runtimes.keys())
    for label in labels:
        mean_runtimes = []
        mean_metrics = []
        metric_error_bars = []
        runtime_error_bars = []
        for runtime, metric in zip(runtimes[label], metrics[label]):
            mean_runtimes.append(np.mean(runtime))
            mean_metrics.append(np.mean(metric))
            runtime_error_bars.append(np.std(runtime))
            metric_error_bars.append(np.std(metric))
        print(label, mean_runtimes, mean_metrics)
        ax.scatter(mean_runtimes, mean_metrics, label=label)
        if errorbars:
            ax.errorbar(mean_runtimes, mean_metrics, xerr=runtime_error_bars,
                        yerr=metric_error_bars)
        if label_locations is not None:
            ax.annotate(label, label_locations[label])

    if xscale_in_log:
        ax.set_xscale('log')
    xmin, xmax = plt.xlim()
    ymin, ymax = plt.ylim()
    if yticks is not None:
        plt.ylim(0, ymax*1.15)
    xmin, xmax = plt.xlim()
    ymin, ymax = plt.ylim()

    ax.set_ylabel(metric_label)
    ax.set_xlabel("Runtime (seconds)")
    if yticks is not None:
        ax.set_yticks(yticks)
    sns.despine()
            
    plt.show()
def plot_metrics_vs_num_sub_problems(metrics,
                                     metric_label,
                                     xticks,
                                     yticks):
    plt.figure(figsize=(6.5, 2.5))
    ax = plt.subplot2grid((1, 1), (0, 0), colspan=1)
    labels = list(metrics.keys())
    all_num_sub_problems = []
    all_metrics = []
    for label in labels:
        mean_metrics = []
        metric_error_bars = []
        num_sub_problems = 1
        if label.startswith("POP-"):
            num_sub_problems = int(label.split("POP-")[1].strip())
        for metric in metrics[label]:
            mean_metrics.append(np.mean(metric))
            metric_error_bars.append(np.std(metric))
        all_num_sub_problems.append(num_sub_problems)
        all_metrics.append(mean_metrics[-1])
    ax.plot(all_num_sub_problems, all_metrics, marker='o')

    ax.set_ylabel(metric_label)
    ax.set_xlabel("Number of sub-problems")
    if yticks is not None:
        ax.set_yticks(yticks)
    sns.despine()
            
    plt.show()<jupyter_output><empty_output><jupyter_text>Plot runtime vs. allocation quality for non-trace experiment<jupyter_code>import ast
labels = ['Exact sol.', 'POP-2', 'POP-4', 'POP-8', 'Gandiva']
all_num_sub_clusters = [1, 2, 4, 8]
with open('logs/non_trace_num_jobs=2048.out', 'r') as f:
    runtimes = f.readline().strip()
    runtimes = ast.literal_eval(runtimes)
    all_effective_throughputs = f.readline().strip()
    all_effective_throughputs = ast.literal_eval(all_effective_throughputs)

plot_runtime_vs_effective_throughput_ratios(
    runtimes, all_effective_throughputs, labels,
    draw_arrow=True)<jupyter_output>Exact sol. 285.62330198287964 1.0 0.0
POP-2 62.2029025554657 0.9996573560890376 0.0049829437901136074
POP-4 17.547146320343018 0.9992862267433089 0.008041299337888667
POP-8 12.609423637390137 0.998336143936632 0.01280419813201839
Gandiva 47.67694807052612 0.5288859944725934 0.16918328108682484<jupyter_text>Plot runtime vs. allocation quality and allocation quality vs. number of sub-problems<jupyter_code>def parse_values(filename):
    parsed_values = {}
    skip_line = True
    with open(filename, 'r') as f:
        for line in f:
            line = line.strip()
            line_tokens = line.split('\t')
            
            # Skip first line (header).
            if skip_line:
                skip_line = False
                continue

            [v100s, p100s, k80s, policy, k, seed, num_jobs_or_lambda,
             objective_value, runtime] = line_tokens
            system_name = "POP-%s" % k
            num_jobs_or_lambda = float(num_jobs_or_lambda)
            seed = int(seed)
            try:
                runtime = float(runtime)
                objective_value = float(objective_value)
            except:
                continue
            if num_jobs_or_lambda not in parsed_values:
                parsed_values[num_jobs_or_lambda] = {}
            if system_name not in parsed_values[num_jobs_or_lambda]:
                parsed_values[num_jobs_or_lambda][system_name] = []
            parsed_values[num_jobs_or_lambda][system_name].append(
                (seed, runtime, objective_value))
    runtimes = {}
    objective_values = {}
    for x in parsed_values:
        for y in parsed_values[x]:
            if y not in runtimes:
                runtimes[y] = []
                objective_values[y] = []
            runtimes[y].append([z[1] for z in parsed_values[x][y]])
            objective_values[y].append([z[2] for z in parsed_values[x][y]])
    return runtimes, objective_values
policies = ["max_min_fairness_packed", "min_total_duration_perf"]
ylabels = ["Average JCT\n(hours)", "Makespan (hours)"]
all_yticks = [[0, 10, 20, 30, 40], [0, 75, 150, 225, 300]]
xscale_in_logs = [True, False]
for (policy, ylabel, yticks, xscale_in_log) in zip(
    policies, ylabels, all_yticks, xscale_in_logs):
    print("Policy: %s" % policy)
    runtimes, objective_values = parse_values("logs/%s.tsv" % policy)
    # Use label_locations argument to annotate this scatterplot.
    plot_runtime_vs_metrics(
        runtimes, objective_values,
        ylabel,
        yticks,
        xscale_in_log=xscale_in_log)
    plot_metrics_vs_num_sub_problems(
        objective_values,
        ylabel,
        [1, 2, 4, 8],
        yticks)<jupyter_output>Policy: max_min_fairness_packed
POP-1 [1.269] [28.871]
POP-2 [0.443] [28.765]
POP-4 [0.185] [27.86]
POP-8 [0.117] [28.809]
{'fraction_non_alphanumeric': 0.1115850144092219, 'fraction_numerical': 0.04749279538904899, 'mean_word_length': 2.5821635012386457, 'pattern_counts': {'":': 3, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '644882', 'license': 'MIT', 'n_tokens_mistral': 3208, 'n_tokens_neox': 2931, 'n_words': 576, 'path': 'cluster_scheduling/figures.ipynb', 'repo_name': 'stanford-futuredata/POP', 'size': '8675'}