<jupyter_start><jupyter_text>Night Averaging the Photometry---Some of the poor fits we get seem to be driven (or at least contributed to) by multiple samples on some epochs. To combat this, implement night-averaging. This should _probably_ be done by default anyway. Will test out on `iPTF13bvn` to check<jupyter_code>%matplotlib notebook
from imp import reload  # Python 3.0 - 3.3
       
from matplotlib import pyplot as plt
from matplotlib import rc
# rc('text', usetex=True)

import os
import numpy as np
from astropy.table import Table

import pycoco as pcc
snname = "iPTF13bvn"

sn = pcc.classes.SNClass(snname)
sn.load_phot( path = os.path.join(pcc.defaults._default_data_dir_path, "lc/"+ snname + ".dat"))
sn.load_list(os.path.join(pcc.defaults._default_list_dir_path, snname + ".list"))

sn.load_spec()
sn.get_lcfit(os.path.join(pcc.defaults._default_recon_dir_path, snname + ".dat"))
sn.check_overlaps()

sn.plot_lc(xextent=60, showsnname=True)<jupyter_output><empty_output><jupyter_text>As can be seen in the above figure, the fit is poor, and there is a large amount of scatter around t=56470<jupyter_code>dt = sn.phot.data["SDSS_i"]
dt["Night"] = list(map(lambda x: np.round(x), dt["MJD"]))
dt.add_index("Night")
dt_grouped = dt.group_by("Night")
for key, group in zip(dt_grouped.groups.keys, dt_grouped.groups):
    print(key)
    print(group)
    print('')
dt_grouped["weights"] = 1.0/((dt_grouped["flux_err"])*(dt_grouped["flux_err"]))
dt_grouped    
from astropy.table import Table, Column, Row

na_table = Table(names = ("MJD","flux","flux_err","filter"),
                dtype = (dt["MJD"].dtype,dt["flux"].dtype,dt["flux_err"].dtype,dt["filter"].dtype))
for i, col in enumerate(na_table.columns):
#     print(i, col)
    na_table[col].unit = dt[col].unit
    
for group in dt_grouped.groups:
    wmean = np.average(group["flux"], weights=group["weights"])
    wmean_err = np.sqrt(1./np.sum(1./(group["flux_err"]*group["flux_err"])))
    print(np.mean(group["MJD"]), wmean, wmean_err)
    na_table.add_row((np.mean(group["MJD"]), wmean, wmean_err, "SDSS_i"))
for i, col in enumerate(na_table.columns):
#     print(i, col)
    na_table[col].unit = dt[col].unit
na_table
sn.load_phot(phot_table=na_table)
sn.plot_lc()<jupyter_output><empty_output><jupyter_text>Using Class Method___<jupyter_code>snname = "iPTF13bvn"

sn = pcc.classes.SNClass(snname)
sn.load_phot( path = os.path.join(pcc.defaults._default_data_dir_path, "lc/"+ snname + ".dat"))
sn.load_list(os.path.join(pcc.defaults._default_list_dir_path, snname + ".list"))

sn.load_spec()
sn.get_lcfit(os.path.join(pcc.defaults._default_recon_dir_path, snname + ".dat"))
sn.check_overlaps()

sn.plot_lc(xextent=60, showsnname=True)
len(sn.phot.phot)
sn.phot.nightaverage(verbose=True)
sn.plot_lc(fit=False)
len(sn.phot.phot)
sn.phot.save(filename=snname+".dat", path = os.path.join(pcc.defaults._default_data_dir_path, "lc_na/"),
                  format="ascii.fast_commented_header", squash=True, verbose = True, sort = False)
# filter_name = "SDSS_i"
filter_names = ["SDSS_i", "SDSS_z"]
for filter_name in filter_names:
    if not os.path.exists(os.path.join(pcc.defaults._default_data_dir_path, "lc_na/"+filter_name)):
        os.mkdir(os.path.join(pcc.defaults._default_data_dir_path, "lc_na/"+filter_name))
    sn.phot.save(filename=snname+".dat", path = os.path.join(pcc.defaults._default_data_dir_path, "lc_na/"+filter_name),
                      filters = filter_name, format="ascii.fast_commented_header", squash=True)
sn.phot._phot_format_for_save()<jupyter_output><empty_output><jupyter_text>Inspect New Fit____<jupyter_code># pcc.coco.run_LCfit(path = os.path.join(pcc.defaults._default_data_dir_path, "lc_na/SDSS_i/"+snname+".dat"), model="Bazin09")
# pcc.coco.run_LCfit(path = os.path.join(pcc.defaults._default_data_dir_path, "lc_na/"+snname+".dat"), model="Bazin09")
# pcc.coco.run_LCfit(path = os.path.join(pcc.defaults._default_data_dir_path, "lc_na/"+snname+".dat"), model="Kessler10")
pcc.coco.run_LCfit(path = os.path.join(pcc.defaults._default_data_dir_path, "lc_na/"+snname+".dat"), model="Karpenka12")
snname = "iPTF13bvn"

sn = pcc.classes.SNClass(snname)
sn.load_phot( path = os.path.join(pcc.defaults._default_data_dir_path, "lc_na/"+ snname + ".dat"))
# sn.load_list(os.path.join(pcc.defaults._default_list_dir_path, snname + ".list"))

# sn.load_spec()
sn.get_lcfit(os.path.join(pcc.defaults._default_recon_dir_path, snname + ".dat"))
sn.check_overlaps()

sn.plot_lc(xextent=60, showsnname=True)
sn.phot.phot
refit = False
if refit:
    pcc.coco.run_LCfit(path = os.path.join(pcc.defaults._default_data_dir_path, "lc_na/SDSS_z/"+snname+".dat"), model="Karpenka12")
    pcc.coco.run_LCfit(path = os.path.join(pcc.defaults._default_data_dir_path, "lc_na/SDSS_i/"+snname+".dat"), model="Kessler10")
snname = "iPTF13bvn"

sn = pcc.classes.SNClass(snname)
sn.load_phot( path = os.path.join(pcc.defaults._default_data_dir_path, "lc_na/"+ snname + ".dat"))
# sn.load_list(os.path.join(pcc.defaults._default_list_dir_path, snname + ".list"))

# sn.load_spec()
sn.get_lcfit(os.path.join(pcc.defaults._default_recon_dir_path, snname + ".dat"))
sn.check_overlaps()

sn.plot_lc(xextent=60, showsnname=True)
## EACH FIT WRITES TO SNNAME.RECON EACH TIME. NEED TO MAKE TEMP AND MERGE - DO IT IN SNCLASS LCFIT THEN DUMP TO RECON?
sn.phot.phot<jupyter_output><empty_output><jupyter_text>Night Average All SNe----<jupyter_code>phot_dir = pcc.defaults._default_data_dir_path+"lc"
ignore_list = ["README"]

for phot_filename in os.listdir(phot_dir):
    if phot_filename not in ignore_list and phot_filename[0] != ".":
        snname = phot_filename.replace(".dat", "")
        sn = pcc.classes.SNClass(snname)
        sn.load_phot( path = os.path.join(phot_dir, phot_filename))
        print(snname, phot_filename, len(sn.phot.phot), end=" ")
        sn.phot.nightaverage(verbose = False)
        print(len(sn.phot.phot))
        sn.phot.save(filename=snname+".dat", path = os.path.join(pcc.defaults._default_data_dir_path, "lc_na/"),
                  format="ascii.fast_commented_header", squash=True)
    sn.phot.save(filename=snname+".dat", path = os.path.join(pcc.defaults._default_data_dir_path, "lc_na/"+filter_name),
                      filters = filter_name, format="ascii.fast_commented_header", squash=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15627466456195738, 'fraction_numerical': 0.006945540647198106, 'mean_word_length': 5.3107569721115535, 'pattern_counts': {'":': 1, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '375428', 'license': 'MIT', 'n_tokens_mistral': 2544, 'n_tokens_neox': 2442, 'n_words': 388, 'path': 'notebooks/Tutorials/Night-Averaging Photometry.ipynb', 'repo_name': 'RobFirth/pycoco', 'size': '6335'}