<commit_before>global ROI_FIRST = (1,2,0,0);
global ROI_LAST = (8,173,0,0);

function get_name(index)
    if is_overview(index)
        if cur_dataset == "zebrafish"
            return string("MontageOverviewImage_W00", index[1], "_sec", index[2])
        else
            return string("MontageOverviewImage_S2-W00", index[1], "_sec", index[2])
        end
    elseif is_montaged(index)
        return string(index[1], ",", index[2], "_montaged")
    elseif is_prealigned(index)
        return string(index[1], ",", index[2], "_prealigned")
    elseif is_aligned(index)
        return string(index[1], ",", index[2], "_aligned")
    elseif is_finished(index)
        return string(index[1], ",", index[2], "_finished")
    else
    return string("Tile_r", index[3], "-c", index[4], "_S2-W00", index[1], "_sec", index[2])
    end
end

# function get_path()
# methods: 
#     
# extensions:
# Mesh.jl: get_path(mesh::Mesh)
function get_path(index, ext = ".h5")
    name = get_name(index)
    if is_overview(index)
        if cur_dataset == "zebrafish"
            section_folder = string("W00", index[1], "_Sec", index[2], "_Montage")
        else
            section_folder = string("S2-W00", index[1], "_Sec", index[2], "_Montage")
        end
        path = joinpath(BUCKET, WAFER_DIR_DICT[index[1]], section_folder, string(name, ext))
    elseif is_montaged(index)
        path = joinpath(MONTAGED_DIR, string(name, ext))
    elseif is_prealigned(index)
        path = joinpath(PREALIGNED_DIR, string(name, ext))
    elseif is_aligned(index)
        path = joinpath(ALIGNED_DIR, string(name, ext))
    elseif is_finished(index)
        path = joinpath(FINISHED_DIR, string(name, ext))
    else
        if cur_dataset == "zebrafish"
            section_folder = string("W00", index[1], "_Sec", index[2], "_Montage")
        else
            section_folder = string("S2-W00", index[1], "_Sec", index[2], "_Montage")
        end
        path = joinpath(BUCKET, WAFER_DIR_DICT[index[1]], section_folder, string(name, ext))
    end
    # println(path)
    return path
end

function get_path(name::String)
    return get_path(parse_name(name))
end

function get_image(index::Index)
    return get_image(get_path(index))
end

function get_thumbnail_path(index::Index)
  fn = string(join(index[1:2], ","), "_thumb.jpg")
  println(fn)
  return joinpath(MONTAGED_DIR, "review", fn) 
end

function get_thumbnail_path(indexA::Index, indexB::Index)
  fn = string(join(indexA[1:2],","), "-", join(indexB[1:2],","), "_thumb.png")
  println(fn)
  if is_prealigned(indexA)
    return joinpath(PREALIGNED_DIR, "review", fn) 
  else
    return joinpath(ALIGNED_DIR, "review", fn) 
  end
end


function waferpaths_to_dict(waferpath_filename)
    wdict = Dict()
    if isfile(waferpath_filename)
        warray = readdlm(waferpath_filename)
        for (idx, path) in zip(warray[:,1], warray[:,2])
            wdict[idx] = path
        end
    end
    return wdict
end

function parse_index(s::String)
    m = Base.match(r"(\d*),(\d*),(\-\d+|\d+),(\-\d+|\d+)", s)
    return (parse(Int, m[1]), parse(Int, m[2]), parse(Int, m[3]), parse(Int, m[4]))
end

function parse_name(name::String)

    ret = (0, 0, 0, 0)
    # singleton tile
    m = Base.match(r"Tile_r(\d*)-c(\d*).*W(\d*)_sec(\d*)", name)
    if typeof(m) != Void
    ret = parse(Int, m[3]), parse(Int, m[4]), parse(Int, m[1]), parse(Int, m[2])
    end

    # overview image
    m = Base.match(r"MontageOverviewImage_S2-W00(\d*)_sec(\d*)", name)
    if typeof(m) != Void
    ret = parse(Int, m[1]), parse(Int, m[2]), OVERVIEW_INDEX, OVERVIEW_INDEX   
    end

    # montaged section
    m = Base.match(r"(\d*),(\d*)_montaged", name)
    if typeof(m) != Void
    ret = montaged(parse(Int, m[1]), parse(Int, m[2]))
    end

    # prealigned section
    m = Base.match(r"(\d*),(\d*)_prealigned", name)
    if typeof(m) != Void
    ret = prealigned(parse(Int, m[1]), parse(Int, m[2]))
    end

    # aligned-section
    m = Base.match(r"(\d*),(\d*)_aligned", name)
    if typeof(m) != Void
    ret = aligned(parse(Int, m[1]), parse(Int, m[2]))
    end

    return ret
    
end

function parse_registry(path::String)
    registry = cell(0, 0)
    if isfile(path)
        file = readdlm(path)
        registry = cell(size(file, 1), size(file, 2) + 1) # name, index, dx, dy
        for i in 1:size(registry, 1)
            index = parse_name(file[i, 1])
            registry[i, 1] = get_name(index)
            registry[i, 2] = index
            for j in 3:size(registry, 2)
                registry[i, j] = file[i, j-1]
            end
        end
      end
    return registry
end

if contains(gethostname(), "seunglab") || contains(gethostname(), "seungom") || contains(gethostname(), "spock") 
 bucket_dir_path = "/mnt/bucket/labs/seung/"
end

if contains(gethostname(), "seungworkstation")
 bucket_dir_path = joinpath(homedir(), "seungmount/")
end

tracerhostnames = ["seungworkstation04", "mycroft", "seungworkstation10", "hogwarts"]
if gethostname() in tracerhostnames
 bucket_dir_path = joinpath(homedir(), "seungmount/Omni/alignment/datasets/")
end

if isdefined(:omni)
    if omni
        bucket_dir_path = joinpath(homedir(), "seungmount/Omni/alignment/datasets/")
    end
end  

#=if isfile("bucket_dir_path.txt")
    bucket_dir_path = rstrip(readall("bucket_dir_path.txt"), '\n')
elseif isfile("../bucket_dir_path.txt")
    bucket_dir_path = rstrip(readall("../bucket_dir_path.txt"), '\n')
else
    bucket_dir_path = joinpath(homedir(), "seungmount/Omni/alignment/datasets")
    if isdefined(:training)
        if training
            println("TRAINING PATHS LOADED")
            bucket_dir_path = joinpath(homedir(), "seungmount/Omni/alignment/training")
        end
    end
    if isdefined(:seunglabs)
        if seunglabs
            println("SEUNGLABS PATHS LOADED")
            bucket_dir_path = "/mnt/bucket/labs/seung/Omni/alignment/datasets"
        end
    end
end
=#

datasets_dir_path = "research/Julimaps/datasets"
cur_dataset = "piriform"
in_alignment_test = false
test_dataset = "piriform_finer_mesh"
#cur_dataset = "zebrafish"
affine_dir_path = "~"

raw_dir_path = "0_raw"
premontaged_dir_path = "1_premontaged"
montaged_dir_path = "2_montaged"
prealigned_dir_path = "3_prealigned"
aligned_dir_path = "4_aligned"
finished_dir_path = "5_finished"

expunged_dir = "expunged"
stack_dir = "stacks"
<commit_msg>Add calls for additional datasets in dataset_default.jl<commit_after>global ROI_FIRST = (1,2,0,0);
global ROI_LAST = (8,173,0,0);

function get_name(index)
    if is_overview(index)
        if cur_dataset == "zebrafish"
            return string("MontageOverviewImage_W00", index[1], "_sec", index[2])
        else
            return string("MontageOverviewImage_S2-W00", index[1], "_sec", index[2])
        end
    elseif is_montaged(index)
        return string(index[1], ",", index[2], "_montaged")
    elseif is_prealigned(index)
        return string(index[1], ",", index[2], "_prealigned")
    elseif is_aligned(index)
        return string(index[1], ",", index[2], "_aligned")
    elseif is_finished(index)
        return string(index[1], ",", index[2], "_finished")
    else
    return string("Tile_r", index[3], "-c", index[4], "_S2-W00", index[1], "_sec", index[2])
    end
end

# function get_path()
# methods: 
#     
# extensions:
# Mesh.jl: get_path(mesh::Mesh)
function get_path(index, ext = ".h5")
    name = get_name(index)
    if is_overview(index)
        if cur_dataset == "zebrafish"
            section_folder = string("W00", index[1], "_Sec", index[2], "_Montage")
        else
            section_folder = string("S2-W00", index[1], "_Sec", index[2], "_Montage")
        end
        path = joinpath(BUCKET, WAFER_DIR_DICT[index[1]], section_folder, string(name, ext))
    elseif is_montaged(index)
        path = joinpath(MONTAGED_DIR, string(name, ext))
    elseif is_prealigned(index)
        path = joinpath(PREALIGNED_DIR, string(name, ext))
    elseif is_aligned(index)
        path = joinpath(ALIGNED_DIR, string(name, ext))
    elseif is_finished(index)
        path = joinpath(FINISHED_DIR, string(name, ext))
    else
        if cur_dataset == "zebrafish"
            section_folder = string("W00", index[1], "_Sec", index[2], "_Montage")
        else
            section_folder = string("S2-W00", index[1], "_Sec", index[2], "_Montage")
        end
        path = joinpath(BUCKET, WAFER_DIR_DICT[index[1]], section_folder, string(name, ext))
    end
    # println(path)
    return path
end

function get_path(name::String)
    return get_path(parse_name(name))
end

function get_image(index::Index)
    return get_image(get_path(index))
end

function get_thumbnail_path(index::Index)
  fn = string(join(index[1:2], ","), "_thumb.jpg")
  println(fn)
  return joinpath(MONTAGED_DIR, "review", fn) 
end

function get_thumbnail_path(indexA::Index, indexB::Index)
  fn = string(join(indexA[1:2],","), "-", join(indexB[1:2],","), "_thumb.png")
  println(fn)
  if is_prealigned(indexA)
    return joinpath(PREALIGNED_DIR, "review", fn) 
  else
    return joinpath(ALIGNED_DIR, "review", fn) 
  end
end


function waferpaths_to_dict(waferpath_filename)
    wdict = Dict()
    if isfile(waferpath_filename)
        warray = readdlm(waferpath_filename)
        for (idx, path) in zip(warray[:,1], warray[:,2])
            wdict[idx] = path
        end
    end
    return wdict
end

function parse_index(s::String)
    m = Base.match(r"(\d*),(\d*),(\-\d+|\d+),(\-\d+|\d+)", s)
    return (parse(Int, m[1]), parse(Int, m[2]), parse(Int, m[3]), parse(Int, m[4]))
end

function parse_name(name::String)

    ret = (0, 0, 0, 0)
    # singleton tile
    m = Base.match(r"Tile_r(\d*)-c(\d*).*W(\d*)_sec(\d*)", name)
    if typeof(m) != Void
    ret = parse(Int, m[3]), parse(Int, m[4]), parse(Int, m[1]), parse(Int, m[2])
    end

    # overview image
    m = Base.match(r"MontageOverviewImage_S2-W00(\d*)_sec(\d*)", name)
    if typeof(m) != Void
    ret = parse(Int, m[1]), parse(Int, m[2]), OVERVIEW_INDEX, OVERVIEW_INDEX   
    end

    # montaged section
    m = Base.match(r"(\d*),(\d*)_montaged", name)
    if typeof(m) != Void
    ret = montaged(parse(Int, m[1]), parse(Int, m[2]))
    end

    # prealigned section
    m = Base.match(r"(\d*),(\d*)_prealigned", name)
    if typeof(m) != Void
    ret = prealigned(parse(Int, m[1]), parse(Int, m[2]))
    end

    # aligned-section
    m = Base.match(r"(\d*),(\d*)_aligned", name)
    if typeof(m) != Void
    ret = aligned(parse(Int, m[1]), parse(Int, m[2]))
    end

    return ret
    
end

function parse_registry(path::String)
    registry = cell(0, 0)
    if isfile(path)
        file = readdlm(path)
        registry = cell(size(file, 1), size(file, 2) + 1) # name, index, dx, dy
        for i in 1:size(registry, 1)
            index = parse_name(file[i, 1])
            registry[i, 1] = get_name(index)
            registry[i, 2] = index
            for j in 3:size(registry, 2)
                registry[i, j] = file[i, j-1]
            end
        end
      end
    return registry
end

if contains(gethostname(), "seunglab") || contains(gethostname(), "seungom") || contains(gethostname(), "spock") 
 bucket_dir_path = "/mnt/bucket/labs/seung/"
end

if contains(gethostname(), "seungworkstation")
 bucket_dir_path = joinpath(homedir(), "seungmount/")
end

tracerhostnames = ["seungworkstation04", "mycroft", "seungworkstation10", "hogwarts"]
if gethostname() in tracerhostnames
 bucket_dir_path = joinpath(homedir(), "seungmount/Omni/alignment/datasets/")
end

if isdefined(:omni)
    if omni
        bucket_dir_path = joinpath(homedir(), "seungmount/Omni/alignment/datasets/")
    end
end  

#=if isfile("bucket_dir_path.txt")
    bucket_dir_path = rstrip(readall("bucket_dir_path.txt"), '\n')
elseif isfile("../bucket_dir_path.txt")
    bucket_dir_path = rstrip(readall("../bucket_dir_path.txt"), '\n')
else
    bucket_dir_path = joinpath(homedir(), "seungmount/Omni/alignment/datasets")
    if isdefined(:training)
        if training
            println("TRAINING PATHS LOADED")
            bucket_dir_path = joinpath(homedir(), "seungmount/Omni/alignment/training")
        end
    end
    if isdefined(:seunglabs)
        if seunglabs
            println("SEUNGLABS PATHS LOADED")
            bucket_dir_path = "/mnt/bucket/labs/seung/Omni/alignment/datasets"
        end
    end
end
=#

datasets_dir_path = "research/Julimaps/datasets"
# cur_dataset = "piriform"
cur_dataset = "AIBS_practice_234251S6R_01_01_aligned_01"
# cur_dataset = "align_net"
in_alignment_test = false
test_dataset = "piriform_finer_mesh"
#cur_dataset = "zebrafish"
affine_dir_path = "~"

raw_dir_path = "0_raw"
premontaged_dir_path = "1_premontaged"
montaged_dir_path = "2_montaged"
prealigned_dir_path = "3_prealigned"
aligned_dir_path = "4_aligned"
finished_dir_path = "5_finished"

expunged_dir = "expunged"
stack_dir = "stacks"

{'fraction_non_alphanumeric': 0.15522736105713175, 'fraction_numerical': 0.017955693742712785, 'mean_word_length': 3.4519031141868513, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3588688', 'n_tokens_mistral': 4765, 'n_tokens_neox': 4645, 'n_words': 942}