<commit_before>
#move to new file
import pandas as pd

from sqlalchemy import create_engine
from tqdm import tqdm
from plandb_methods import *

def compileContrastCurves(stars, cont_path):
    scenarios = pd.read_csv('cache/scenario_angles.csv')['scenario_name']

    cols = ['scenario_name', 'st_id', 'r_lamD', 'r_as', 'r_mas', 'contrast', 'dMag', 'lam', 't_int_hr', 'fpp']
    contrastCurves = pd.DataFrame([], columns = cols)
    sts_missed = []
    cont_path.mkdir(parents=True, exist_ok=True)
    for filename in os.listdir(cont_path):
        st_sen = filename[:-2]
        st_name = ""
        scen = ""
        for sen_name in scenarios:
            if sen_name in st_sen:
                st_name = st_sen[:-(1+len(sen_name))].replace("_", " ")
                scen = st_sen[-(len(sen_name)):]

        indCC = pd.read_pickle(Path(cont_path, filename))

        if len( list(stars.index[(stars['st_name'] == st_name)])) == 0:
            if st_name not in sts_missed:
                print(st_name)
                sts_missed.append(st_name)
        else:
            indCC["scenario_name"] = scen
            indCC["st_id"] = list(stars.index[(stars['st_name'] == st_name)])[0]
            contrastCurves = contrastCurves.append(indCC, ignore_index=True)
    return contrastCurves

def compileCompleteness():
    datestr = Time.now().datetime.strftime("%Y-%m")
    comp_data = pd.read_pickle(f"cache/comps_data_{datestr}.p")
    col_names = comp_data.columns.values.tolist()
    scenario_names = []
    for x in col_names:
        if x[:8] == 'complete':
            scenario_names.append(x[13:])
    #drop contr_curve col
    completeness = pd.DataFrame([], columns = ['pl_id', 'completeness',  'scenario_name', 'compMinWA', 'compMaxWA', 'compMindMag', 'compMaxdMag'])
    for i, row in tqdm(comp_data.iterrows()):
        newRows = []
        for scenario_name in scenario_names:
            newRows = []
            if pd.notna(row[('completeness_' + scenario_name)]):
                newRows.append([row['pl_id'], row['completeness_' + scenario_name], scenario_name,
                    row['compMinWA_' + scenario_name], 
                    row['compMaxWA_' + scenario_name],
                    row['compMindMag_' + scenario_name],
                    row['compMaxdMag_' + scenario_name]])

                singleRow = pd.DataFrame(newRows, columns =  ['pl_id', 'completeness',  'scenario_name', 'compMinWA', 'compMaxWA', 'compMindMag', 'compMaxdMag'])
                completeness = completeness.append(singleRow, ignore_index=True)
    # print(completeness)
    return completeness

if __name__ == "__main__":
    datestr = Time.now().datetime.strftime("%Y-%m")
    plandata_path = Path(f'cache/plandata_{datestr}.p')
    planets = pd.read_pickle(plandata_path)

    stdata_path = Path(f'cache/stdata_{datestr}.p')
    stars = pd.read_pickle(stdata_path)

    orbfits_path = Path(f'cache/orbfits_{datestr}.p')
    orbitfits = pd.read_pickle(orbfits_path)

    orbdata_path = Path(f'cache/orbdata_{datestr}.p')
    orbits = pd.read_pickle(orbdata_path)

    comps_path = Path(f'cache/comps_{datestr}.p')
    pdfs = pd.read_pickle(comps_path)

    compiled_contr_curvs_path = Path(f'cache/compiled_cont_curvs_{datestr}.p')
    contr_curvs_path = Path(f'cache/cont_curvs_{datestr.replace("-", "_")}')

    if compiled_contr_curvs_path.exists():
        contrast_curves = pd.read_pickle(compiled_contr_curvs_path)
    else:
        contrast_curves = compileContrastCurves(stars, contr_curvs_path)
        contrast_curves.to_pickle(compiled_contr_curvs_path)
    def addId(r):
        r['pl_id']= list(planets.index[(planets['pl_name'] == r['Name'])])[0]
        return r
    newpdfs = pdfs.apply(addId, axis = 1)

    scenarios = pd.read_csv("cache/scenario_angles.csv")<commit_msg>Adding the quadrature values<commit_after>
#move to new file
import pandas as pd

from sqlalchemy import create_engine
from tqdm import tqdm
from plandb_methods import *

def compileContrastCurves(stars, cont_path):
    scenarios = pd.read_csv('cache/scenario_angles.csv')['scenario_name']

    cols = ['scenario_name', 'st_id', 'r_lamD', 'r_as', 'r_mas', 'contrast', 'dMag', 'lam', 't_int_hr', 'fpp']
    contrastCurves = pd.DataFrame([], columns = cols)
    sts_missed = []
    cont_path.mkdir(parents=True, exist_ok=True)
    for filename in os.listdir(cont_path):
        st_sen = filename[:-2]
        st_name = ""
        scen = ""
        for sen_name in scenarios:
            if sen_name in st_sen:
                st_name = st_sen[:-(1+len(sen_name))].replace("_", " ")
                scen = st_sen[-(len(sen_name)):]

        indCC = pd.read_pickle(Path(cont_path, filename))

        if len( list(stars.index[(stars['st_name'] == st_name)])) == 0:
            if st_name not in sts_missed:
                print(st_name)
                sts_missed.append(st_name)
        else:
            indCC["scenario_name"] = scen
            indCC["st_id"] = list(stars.index[(stars['st_name'] == st_name)])[0]
            contrastCurves = contrastCurves.append(indCC, ignore_index=True)
    return contrastCurves

def compileCompleteness():
    datestr = Time.now().datetime.strftime("%Y-%m")
    comp_data = pd.read_pickle(f"cache/comps_data_{datestr}.p")
    col_names = comp_data.columns.values.tolist()
    scenario_names = []
    for x in col_names:
        if x[:8] == 'complete':
            scenario_names.append(x[13:])
    #drop contr_curve col
    completeness = pd.DataFrame([], columns = ['pl_id', 'completeness',  'scenario_name', 'compMinWA', 'compMaxWA', 'compMindMag', 'compMaxdMag'])
    for i, row in tqdm(comp_data.iterrows()):
        newRows = []
        for scenario_name in scenario_names:
            newRows = []
            if pd.notna(row[('completeness_' + scenario_name)]):
                newRows.append([row['pl_id'], row['completeness_' + scenario_name], scenario_name,
                    row['compMinWA_' + scenario_name], 
                    row['compMaxWA_' + scenario_name],
                    row['compMindMag_' + scenario_name],
                    row['compMaxdMag_' + scenario_name]])

                singleRow = pd.DataFrame(newRows, columns =  ['pl_id', 'completeness',  'scenario_name', 'compMinWA', 'compMaxWA', 'compMindMag', 'compMaxdMag'])
                completeness = completeness.append(singleRow, ignore_index=True)
    # print(completeness)
    return completeness

if __name__ == "__main__":
    datestr = Time.now().datetime.strftime("%Y-%m")
    plandata_path = Path(f'cache/plandata_{datestr}.p')
    planets = pd.read_pickle(plandata_path)

    stdata_path = Path(f'cache/stdata_{datestr}.p')
    stars = pd.read_pickle(stdata_path)

    orbfits_path = Path(f'cache/quadrature_data_{datestr}.p')
    orbitfits = pd.read_pickle(orbfits_path)

    orbdata_path = Path(f'cache/orbdata_{datestr}.p')
    orbits = pd.read_pickle(orbdata_path)

    comps_path = Path(f'cache/comps_{datestr}.p')
    pdfs = pd.read_pickle(comps_path)

    compiled_contr_curvs_path = Path(f'cache/compiled_cont_curvs_{datestr}.p')
    contr_curvs_path = Path(f'cache/cont_curvs_{datestr.replace("-", "_")}')

    if compiled_contr_curvs_path.exists():
        contrast_curves = pd.read_pickle(compiled_contr_curvs_path)
    else:
        contrast_curves = compileContrastCurves(stars, contr_curvs_path)
        contrast_curves.to_pickle(compiled_contr_curvs_path)
    def addId(r):
        r['pl_id']= list(planets.index[(planets['pl_name'] == r['Name'])])[0]
        return r
    newpdfs = pdfs.apply(addId, axis = 1)

    scenarios = pd.read_csv("cache/scenario_angles.csv")
{'fraction_non_alphanumeric': 0.1381483913328956, 'fraction_numerical': 0.0023637557452396585, 'mean_word_length': 3.1846153846153844, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3708811', 'n_tokens_mistral': 2700, 'n_tokens_neox': 2630, 'n_words': 426}