<commit_before>function used_memory()
    int(Sys.total_memory()) - int(Sys.free_memory())
end 

const MAX_GC_ATTEMPTS = 10 :: Int64

function force_gc()
    force_gc(MAX_GC_ATTEMPTS)
end

function force_gc(max_attempts::Integer)
    @printf("forcing GC....")
    used = used_memory()
    attempts = 1
    while true
        Base.gc()
	new_used = used_memory()
        if used > new_used && attempts < max_attempts
            used = new_used
            attempts += 1
	else
	    break
        end
    end
    @printf("attempts %d\n", attempts)
end 

function benchmark(count, warmup, target_time, func, gc_before)
    force_gc()
    first_execution = @elapsed func()
end  

function run_benchmark(env::Environment,
                       bench::Benchmark, 
                       force_gc::Bool)
    # run for at least a 1/10 of a second 
    run_for_atleast(0.1, 10000, time_clock)
    min_time = min(env.clock_resolution * 1000, 0.1)<commit_msg>remove unused force_gc<commit_after>function used_memory()
    int(Sys.total_memory()) - int(Sys.free_memory())
end 


function benchmark(count, warmup, target_time, func, gc_before)
    force_gc()
    first_execution = @elapsed func()
end  


function run_benchmark(env::Environment,
                       bench::Benchmark, 
                       force_gc::Bool)
    # run for at least a 1/10 of a second 
    run_for_atleast(0.1, 10000, time_clock)
    min_time = min(env.clock_resolution * 1000, 0.1)
{'fraction_non_alphanumeric': 0.10254645560908465, 'fraction_numerical': 0.02615278733654508, 'mean_word_length': 2.6903553299492384, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6130902', 'n_tokens_mistral': 521, 'n_tokens_neox': 466, 'n_words': 111}