<commit_before>using DataFrames
using ClusterManagers
using ArgParse
using YAML

function parse_commandline()
    settings = ArgParseSettings()
    settings.prog = "LogoPipe"
    settings.description = "LogoPipe is a custom application desinged to run NetLogo and Bsearch analyses with Slurm"
    @add_arg_table settings begin
        "model_directory"
            help = "The directory containing the model data"
            required = true
    end
    return parse_args(settings)
end

function parse_csv_to_commands(csv_file, outpath="./out/")
    data = readtable(csv_file)
    commands = []
    col_names = names(data)
    for row in eachrow(data)
        command = row[1]
        for (i, name) in enumerate(col_names[2:end])
            println(name)
            command = join([command, row[i+1]], *(" ", replace(string(name), "_", "-"), " "))
            command = replace(command, "OUT", outpath)
        end
        push!(commands, command)
    end
    return commands
end

function unpack_directory(model_directory, dir_name)
    output_dir = *(dir_name, "_out/") 
    output_path = joinpath(model_directory, "out", output_dir)
    mkpath(output_path)
    
    dir_path = joinpath(model_directory, "in", dir_name)
    dir_listing = readdir(dir_path)

    commands = []
    for item in dir_listing
        item_path = joinpath(dir_path, item)
        if !startswith(item, ".") && endswith(item, ".csv") && isfile(item_path)
            commands = vcat(parse_csv_to_commands(item_path, output_path), commands)
        end
    end
    return commands
end
        
function run_job(command)
    run(`echo test`)
    #run(`$command`)
end


function schedule_jobs(commands, output_dir)
    job_num = length(commands)
    node_num = 1

    #slurm_out = join([output_dir, "slurm_output"], "/")
    #mkdir(slurm_out)

    addprocs(SlurmManager(job_num))#, nodes=node_num)

    #=
    pmap(run_job, commands)
    =#
    @parallel for i in workers()
        #command = commands[i]
        run(`echo $i`)
        #run(`$command`)
    end

    for i in workers()
        rmprocs(i)
    end
end

function main()    
    args = parse_commandline()

    in_dir = joinpath(args["model_directory"], "in")
    out_dir = joinpath(args["model_directory"], "out")

    commands = []
    in_listing = readdir(in_dir)
    for dir in in_listing
        if !startswith(dir, ".") && isdir(joinpath(in_dir, dir))
            commands = vcat(unpack_directory(args["model_directory"], dir), commands)
        end
    end
    for command in commands
        println(command)
    end
    schedule_jobs(commands, out_dir)
end

main()
<commit_msg>Switched to implenting @persist instead of ClusterManagers<commit_after>using DataFrames
#using ClusterManagers
using ArgParse
using YAML
using Persist

function parse_commandline()
    settings = ArgParseSettings()
    settings.prog = "LogoPipe"
    settings.description = "LogoPipe is a custom application desinged to run NetLogo and Bsearch analyses with Slurm"
    @add_arg_table settings begin
        "model_directory"
            help = "The directory containing the model data"
            required = true
    end
    return parse_args(settings)
end

function parse_csv_to_commands(csv_file, model_directory, outpath="./out/")
    data = readtable(csv_file)
    commands = []
    col_names = names(data)
    for row in eachrow(data)
        command = row[1]
        for (i, name) in enumerate(col_names[2:end])
            command = join([command, row[i+1]], *(" ", replace(string(name), "_", "-"), " "))
            command = replace(command, "OUT", outpath)
            command = replace(command, "MOD", model_directory)
        end
        push!(commands, command)
    end
    return commands
end

function unpack_directory(model_directory, dir_name)
    output_dir = *(dir_name, "_out/") 
    output_path = joinpath(model_directory, "out", output_dir)
    mkpath(output_path)
    
    dir_path = joinpath(model_directory, "in", dir_name)
    dir_listing = readdir(dir_path)

    commands = []
    for item in dir_listing
        item_path = joinpath(dir_path, item)
        if !startswith(item, ".") && endswith(item, ".csv") && isfile(item_path)
            commands = vcat(parse_csv_to_commands(item_path, model_directory, output_path), commands)
        end
    end
    return commands
end
        
@everywhere function run_job(command, model_directory)
    cd(model_directory)
    return readall(`$command`)
    #run(`echo test`)
    #run(`$command`)
end


@everywhere function test_run_job()
    return readall(`echo test`)
end

function schedule_jobs(commands, output_dir, model_directory)
#function schedule_jobs()
    #job_num = length(commands)
    #node_num = 1
    job_num = 10

    #=
    addprocs(SlurmManager(job_num))
    output = []
    pids = []
    for i in workers()
        #host, pid = fetch(@spawnat i (run_job("ls", model_directory), getpid()))
        out, pid = fetch(@spawnat i (test_run_job(), getpid()))
        push!(output, out)
        push!(pids, pid)
    end
    println(output)

    for i in workers()
        rmprocs(i)
    end
    =#
    
    jobs = []
    for (count, command) in enumerate(commands)
        jobname = *("logopipe-", string(count)) 
        @persist jobname SlurmManager (cd(model_directory); run(`$command`))
        push!(jobs, jobname)
    end
        



end

function main()    
    args = parse_commandline()

    in_dir = joinpath(args["model_directory"], "in")
    out_dir = joinpath(args["model_directory"], "out")

    commands = []
    in_listing = readdir(in_dir)
    for dir in in_listing
        if !startswith(dir, ".") && isdir(joinpath(in_dir, dir))
            commands = vcat(unpack_directory(args["model_directory"], dir), commands)
        end
    end
    for command in commands
        println(command)
    end
    schedule_jobs(commands, out_dir, args["model_directory"])
end

main()
#schedule_jobs()

{'fraction_non_alphanumeric': 0.11381701336039235, 'fraction_numerical': 0.0016911889058007779, 'mean_word_length': 2.995945945945946, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5027424', 'n_tokens_mistral': 1910, 'n_tokens_neox': 1850, 'n_words': 448}