<commit_before>function compare(data_generator::Function, opts::Vector{SupervisedModelOptions}, opts_names::Vector, metric::Function; iterations::Int=10)
    res = DataFrame(Options=[], Iteration=[], Score=[])
    for i=1:iterations
        split = data_generator(i)
        for j=1:length(opts)
            res = vcat(res, DataFrame(Name=opts_names[j], Option=opts[j], Iteration=i, Score=evaluate(split, opts[j], metric)))
        end
    end
    res
end

function compare(data_generator::Function, opts_generator::Function, opts_sweep::Vector, metric::Function; iterations::Int=10)
    opts = SupervisedModelOptions[opts_generator(s) for s=opts_sweep]
    compare(data_generator, opts, opts_sweep, metric, iterations=iterations)
end<commit_msg>added debug output to experiments<commit_after>function compare(data_generator::Function, opts::Vector{SupervisedModelOptions}, opts_names::Vector, metric::Function; iterations::Int=10)
    res = DataFrame(Options=[], Iteration=[], Score=[])
    for i=1:iterations
        split = data_generator(i)
        for j=1:length(opts)
            println("Running ", opts_names[j], " on split ", i)
            res = vcat(res, DataFrame(Name=opts_names[j], Option=opts[j], Iteration=i, Score=evaluate(split, opts[j], metric)))
        end
    end
    res
end

function compare(data_generator::Function, opts_generator::Function, opts_sweep::Vector, metric::Function; iterations::Int=10)
    opts = SupervisedModelOptions[opts_generator(s) for s=opts_sweep]
    compare(data_generator, opts, opts_sweep, metric, iterations=iterations)
end
{'fraction_non_alphanumeric': 0.1313451776649746, 'fraction_numerical': 0.007614213197969543, 'mean_word_length': 5.54356846473029, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5677588', 'n_tokens_mistral': 490, 'n_tokens_neox': 470, 'n_words': 96}