<commit_before>#NORMALIZER = Array{Float64, 2}(readdlm("calib"))
using StatsBase

function import_AIBS_dir_tifs(src_folder, suffix="_prealigned", dst_folder=src_folder)
	for tif in sort_dir(src_folder, "tif")
		println(tif)
		img = Array{Float64, 2}(Images.load(joinpath(src_folder, tif)).data')
		#	  img = img ./ NORMALIZER;
		#	  distr = nquantile(img[:], 16)
		#	  minval = distr[2]; maxval = distr[16];
		#	  img = min(1, max(0, (img - minval) / (maxval-minval)))
		img = Array{UInt8, 2}(round(UInt8, img * 255));
		#f = h5open(joinpath(to, import_frame_tif_to_wafer_section_h5(tif, wafer_num, sec_num)), "w")
		f = h5open(joinpath(dst_folder, import_tif_to_wafer_section_h5(tif, suffix)), "w")
		#    	chunksize = div(min(size(img)...), 4);<commit_msg>Update import.jl scripts for AIBS datasets<commit_after>#NORMALIZER = Array{Float64, 2}(readdlm("calib"))
using StatsBase

function import_AIBS_pilot_v1()
	meta_fn = joinpath(RAW_DIR, "2342516R01_ribbonordered.txt")
	meta = readdlm(meta_fn)
	meta_new_fn = joinpath(RAW_DIR, "meta.txt")

	# cleanup meta text file
	path = meta[:,1]
	path_split = map(split, path, repeated("\\"))
	path_grid = [i[1] for i in path_split]
	path_subdir = [i[2] for i in path_split]
	path_old_fn = [i[3] for i in path_split]
	path = [joinpath(a,b,c) for (a,b,c) in zip(path_grid, path_subdir, path_old_fn)]
	path_old_fn_no_ext = [split(i, ".")[1] for i in path_old_fn]
	col = [parse(split(i, "-")[2])+1 for i in path_old_fn_no_ext]
	row = [parse(split(i, "-")[3])+1 for i in path_old_fn_no_ext]
	sec = meta[:,4]+1
	# string("Tile_r", index[3], "-c", index[4], "_S2-W00", index[1], "_sec", index[2])
	# path_new_fn = [string("Tile_r", @sprintf("%02d", i), "-c", @sprintf("%02d", j), "_S2-W001_sec", @sprintf("%04d", k), ".h5") for (i,j,k) in zip(row, col, sec)]
	path_new_fn = [string("Tile_r",i, "-c", j, "_S2-W001_sec", k, ".h5") for (i,j,k) in zip(row, col, sec)]
	meta = hcat(meta, [path path_new_fn row col])
	writedlm(meta_new_fn, meta)

	for i in 1:size(meta, 1)
		old_filepath = joinpath(RAW_DIR, meta[i,5])
		new_filepath = joinpath(RAW_DIR, meta[i,6])
		index = (1, meta[i,4]+1, meta[i,7], meta[i,8])
		offset = [meta[i,3], meta[i,2]]

		println(old_filepath)
		img = Array{Float64, 2}(Images.load(old_filepath).data')
		#	  img = img ./ NORMALIZER;
		#	  distr = nquantile(img[:], 16)
		#	  minval = distr[2]; maxval = distr[16];
		#	  img = min(1, max(0, (img - minval) / (maxval-minval)))
		img = Array{UInt8, 2}(round(UInt8, img * 255));
		#f = h5open(joinpath(to, import_frame_tif_to_wafer_section_h5(tif, wafer_num, sec_num)), "w")
		f = h5open(new_filepath, "w")
		#    	chunksize = div(min(size(img)...), 4);
		chunksize = 1000;
		@time f["img", "chunk", (chunksize,chunksize)] = img
		close(f)
		update_offset(index, offset, [size(img)...])		
	end
end

# function import_AIBS_pilot_v2()
# 	trakem_fn = joinpath(homedir(), "seungmount/research/Julimaps/datasets/AIBS_pilot_v2/234251S6R_01_02_160421_zorder_sectmanifest_columnsappended.txt")
# 	image_dir = joinpath(homedir(), "seungmount/research/Julimaps/datasets/AIBS_pilot_v2/0_raw")

# 	meta = readdlm(trakem_fn)

# 	# cleanup trakem text file
# 	fn_length = map(length, meta[:,1])
# 	fn_start = 3
# 	fn_range = map(range, repeated(fn_start), fn_length-fn_start+1)
# 	meta[:,1] = map(getindex, meta[:,1], fn_range)

# 	# rename files based on ordering
# 	fn_splits = [split(split(i, ".")[1], "_")[end-1:end] for i in meta[:,1]]
# 	row = [parse(i[1]) for i in fn_splits]
# 	col = [parse(i[2]) for i in fn_splits]
# 	meta = hcat(meta, [row col])

# end

function import_trakem_dir_tifs(src_folder, suffix="_prealigned", dst_folder=src_folder)
	for tif in sort_dir(src_folder, "tif")
		println(tif)
		img = Array{Float64, 2}(Images.load(joinpath(src_folder, tif)).data')
		#	  img = img ./ NORMALIZER;
		#	  distr = nquantile(img[:], 16)
		#	  minval = distr[2]; maxval = distr[16];
		#	  img = min(1, max(0, (img - minval) / (maxval-minval)))
		img = Array{UInt8, 2}(round(UInt8, img * 255));
		#f = h5open(joinpath(to, import_frame_tif_to_wafer_section_h5(tif, wafer_num, sec_num)), "w")
		f = h5open(joinpath(dst_folder, import_tif_to_wafer_section_h5(tif, suffix)), "w")
		#    	chunksize = div(min(size(img)...), 4);
{'fraction_non_alphanumeric': 0.18489583333333334, 'fraction_numerical': 0.03740530303030303, 'mean_word_length': 5.132075471698113, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1299670', 'n_tokens_mistral': 1977, 'n_tokens_neox': 1845, 'n_words': 353}