<commit_before>module SDSS

using CelesteTypes

using FITSIO
using WCSLIB
using DataFrames


function load_stamp_blob(stamp_dir, stamp_id)
    function fetch_image(b)
        band_letter = ['u', 'g', 'r', 'i', 'z'][b]
        filename = "$stamp_dir/stamp-$band_letter-$stamp_id.fits"

        fits = FITS(filename)
        original_pixels = read(fits[1])
        hdr = readheader(fits[1])
		close(fits)
        dn = original_pixels / hdr["CALIB"] + hdr["SKY"]
        nelec = float(int(dn * hdr["GAIN"]))

        fits_file = fits_open_file(filename)
        header_str = fits_hdr2str(fits_file)
		close(fits_file)
        ((wcs,),nrejected) = wcspih(header_str)

		alphaBar = [hdr["PSF_P0"], hdr["PSF_P1"], hdr["PSF_P2"]]
		xiBar = [
			[hdr["PSF_P3"]  hdr["PSF_P4"]],
			[hdr["PSF_P5"]  hdr["PSF_P6"]],
			[hdr["PSF_P7"]  hdr["PSF_P8"]]
		]'
		SigmaBar = Array(Float64, 2, 2, 3)
		SigmaBar[:,:,1] = [[hdr["PSF_P9"] hdr["PSF_P11"]], 
                [hdr["PSF_P11"] hdr["PSF_P10"]]]
		SigmaBar[:,:,2] = [[hdr["PSF_P12"] hdr["PSF_P14"]], 
                [hdr["PSF_P14"] hdr["PSF_P13"]]]
		SigmaBar[:,:,3] = [[hdr["PSF_P15"] hdr["PSF_P17"]],
                [hdr["PSF_P17"] hdr["PSF_P16"]]]

		psf = [PsfComponent(alphaBar[k], xiBar[:, k], SigmaBar[:, :, k]) for k in 1:3]

        H, W = size(original_pixels)
		iota = hdr["GAIN"] / hdr["CALIB"]
		epsilon = hdr["SKY"] * hdr["CALIB"]

		run_num = int(hdr["RUN"])
		camcol_num = int(hdr["CAMCOL"])
		field_num = int(hdr["FIELD"])

        Image(H, W, nelec, b, wcs, epsilon, iota, psf, run_num, camcol_num, field_num)
    end

    blob = map(fetch_image, 1:5)
end


function load_stamp_catalog_df(cat_dir, stamp_id, blob; match_blob=false)
    cat_fits = fits_open_table("$cat_dir/cat-$stamp_id.fits")
    num_rows = int(fits_read_keyword(cat_fits, "NAXIS2")[1])
    num_cols = int(fits_read_keyword(cat_fits, "TFIELDS")[1])
	ttypes = [rstrip(fits_read_keyword(cat_fits, "TTYPE$i")[1][2:end-1]) for i in 1:num_cols]
	tforms = [rstrip(fits_read_keyword(cat_fits, "TFORM$i")[1][2:end-1]) for i in 1:num_cols]
	col_types = [l in ("D", "E") ? Float64 : l in ("L",) ? Bool : l in ("B", "I") ? Int64 : None for l in tforms]

    df = DataFrame()
    for i in 1:num_cols
		tmp_data = Array(col_types[i], num_rows)
        fits_read_col(cat_fits, col_types[i], i, 1, 1, tmp_data)
        df[symbol(ttypes[i])] = tmp_data
    end

    fits_close_file(cat_fits)

	if match_blob
		camcol_matches = df[:camcol] .== blob[3].camcol_num
		run_matches = df[:run] .== blob[3].run_num
		field_matches = df[:field] .== blob[3].field_num
		df = df[camcol_matches & run_matches & field_matches, :]
	end

    df
end


function load_stamp_catalog(cat_dir, stamp_id, blob; match_blob=false)
    df = load_stamp_catalog_df(cat_dir, stamp_id, blob, match_blob)

    function row_to_ce(row)
		x_y = wcss2p(blob[1].wcs, [row[1, :ra], row[1, :dec]]'')[:]

		star_fluxes = zeros(5)
		gal_fluxes = zeros(5)
        fracs_dev = [row[1, :frac_dev], 1 - row[1, :frac_dev]]<commit_msg>bug fix to earlier bug fix: not filtering catalog rows that don't match camcol_num, etc.<commit_after>module SDSS

using CelesteTypes

using FITSIO
using WCSLIB
using DataFrames


function load_stamp_blob(stamp_dir, stamp_id)
    function fetch_image(b)
        band_letter = ['u', 'g', 'r', 'i', 'z'][b]
        filename = "$stamp_dir/stamp-$band_letter-$stamp_id.fits"

        fits = FITS(filename)
        original_pixels = read(fits[1])
        hdr = readheader(fits[1])
		close(fits)
        dn = original_pixels / hdr["CALIB"] + hdr["SKY"]
        nelec = float(int(dn * hdr["GAIN"]))

        fits_file = fits_open_file(filename)
        header_str = fits_hdr2str(fits_file)
		close(fits_file)
        ((wcs,),nrejected) = wcspih(header_str)

		alphaBar = [hdr["PSF_P0"], hdr["PSF_P1"], hdr["PSF_P2"]]
		xiBar = [
			[hdr["PSF_P3"]  hdr["PSF_P4"]],
			[hdr["PSF_P5"]  hdr["PSF_P6"]],
			[hdr["PSF_P7"]  hdr["PSF_P8"]]
		]'
		SigmaBar = Array(Float64, 2, 2, 3)
		SigmaBar[:,:,1] = [[hdr["PSF_P9"] hdr["PSF_P11"]], 
                [hdr["PSF_P11"] hdr["PSF_P10"]]]
		SigmaBar[:,:,2] = [[hdr["PSF_P12"] hdr["PSF_P14"]], 
                [hdr["PSF_P14"] hdr["PSF_P13"]]]
		SigmaBar[:,:,3] = [[hdr["PSF_P15"] hdr["PSF_P17"]],
                [hdr["PSF_P17"] hdr["PSF_P16"]]]

		psf = [PsfComponent(alphaBar[k], xiBar[:, k], SigmaBar[:, :, k]) for k in 1:3]

        H, W = size(original_pixels)
		iota = hdr["GAIN"] / hdr["CALIB"]
		epsilon = hdr["SKY"] * hdr["CALIB"]

		run_num = int(hdr["RUN"])
		camcol_num = int(hdr["CAMCOL"])
		field_num = int(hdr["FIELD"])

        Image(H, W, nelec, b, wcs, epsilon, iota, psf, run_num, camcol_num, field_num)
    end

    blob = map(fetch_image, 1:5)
end


function load_stamp_catalog_df(cat_dir, stamp_id, blob; match_blob=false)
    cat_fits = fits_open_table("$cat_dir/cat-$stamp_id.fits")
    num_rows = int(fits_read_keyword(cat_fits, "NAXIS2")[1])
    num_cols = int(fits_read_keyword(cat_fits, "TFIELDS")[1])
	ttypes = [rstrip(fits_read_keyword(cat_fits, "TTYPE$i")[1][2:end-1]) for i in 1:num_cols]
	tforms = [rstrip(fits_read_keyword(cat_fits, "TFORM$i")[1][2:end-1]) for i in 1:num_cols]
	col_types = [l in ("D", "E") ? Float64 : l in ("L",) ? Bool : l in ("B", "I") ? Int64 : None for l in tforms]

    df = DataFrame()
    for i in 1:num_cols
		tmp_data = Array(col_types[i], num_rows)
        fits_read_col(cat_fits, col_types[i], i, 1, 1, tmp_data)
        df[symbol(ttypes[i])] = tmp_data
    end

    fits_close_file(cat_fits)

	if match_blob
		camcol_matches = df[:camcol] .== blob[3].camcol_num
		run_matches = df[:run] .== blob[3].run_num
		field_matches = df[:field] .== blob[3].field_num
		df = df[camcol_matches & run_matches & field_matches, :]
	end

    df
end


function load_stamp_catalog(cat_dir, stamp_id, blob; match_blob=false)
    df = load_stamp_catalog_df(cat_dir, stamp_id, blob, match_blob=match_blob)

    function row_to_ce(row)
		x_y = wcss2p(blob[1].wcs, [row[1, :ra], row[1, :dec]]'')[:]

		star_fluxes = zeros(5)
		gal_fluxes = zeros(5)
        fracs_dev = [row[1, :frac_dev], 1 - row[1, :frac_dev]]
{'fraction_non_alphanumeric': 0.17893526125534012, 'fraction_numerical': 0.025303976339139007, 'mean_word_length': 3.8774038461538463, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3039918', 'n_tokens_mistral': 2795, 'n_tokens_neox': 2592, 'n_words': 469}