<commit_before>#!/usr/bin/env python

"""Calculate basic efficiency statistics for a ztf_sim run."""
from __future__ import print_function

import sys
# hack to get the path right
sys.path.append('..')
import numpy as np
from collections import OrderedDict
from sqlalchemy import create_engine
import pandas as pd
from astropy.time import Time
import astropy.coordinates as coord
import astropy.units as u
from ztf_sim.constants import *
from ztf_sim.utils import *


def calc_stats(sim_name):

    df = df_read_from_sqlite(sim_name, tablename='Summary', directory='sims')

    stats = OrderedDict()

    stats['Simulation Name'] = sim_name

    # use max - min because there may be no observations some nights
    # TODO: fails if initial nights are weathered out
    stats['Number of Nights'] = df.night.max() - df.night.min() + 1

    # possible nights
    possible_nights = np.linspace(df.night.min(), df.night.max(),
                                  stats['Number of Nights'])

    weathered_out_nights = 0
    actual_nights = df.night.unique()
    for night in possible_nights:
        if night not in actual_nights:
            weathered_out_nights += 1

    stats['Nights completely weathered out'] = weathered_out_nights

    mjds = df.expMJD.apply(np.floor).unique()
    t_mjds = Time(mjds, format='mjd')
    hours_of_darkness = approx_hours_of_darkness(t_mjds).to(u.hour).value

    stats['Average Hours of Darkness'] = np.mean(hours_of_darkness)

    # TODO: use missed visits so we can catch time lost
    stats['Total Science Time (h)'] = \
        (df.visitExpTime.sum() + df.slewTime.sum()) / 3600.

    stats['Average Science Time per night (h)'] = \
        stats['Total Science Time (h)'] / stats['Number of Nights']

    # time used for science: exposing or slewing/filter changing
    stats['Fraction of time usable'] = stats['Total Science Time (h)'] / \
        (stats['Average Hours of Darkness'] * stats['Number of Nights'])

    stats['Average Number of Exposures per hour'] = \
        len(df) / stats['Total Science Time (h)']

    # leave out NaNs from weather/nightly breaks
    w = np.isfinite(df.slewTime)

    # fraction of usable science time
    stats['Open Shutter Fraction'] = df[w].visitExpTime.sum() / \
        (df[w].visitExpTime.sum() + df[w].slewTime.sum())

    stats['Mean Time Between Exposures (s)'] = df[w].slewTime.mean()
    stats['Mean Slew Distance (deg)'] = np.degrees(df[w].slewDist.mean())

    stats['90% Time Between Exposures (s)'] = np.percentile(df[w].slewTime, 90)
    stats['90% Slew Distance (deg)'] = np.percentile(
        np.degrees(df[w].slewDist), 90)

    stats['Median Airmass'] = df.airmass.median()
    stats['90% Airmass'] = np.percentile(df.airmass, 90)

    # program breakdown
    pgrp = df.groupby('propID')
    stats['Program Fraction'] = (pgrp['fieldID'].agg(len) / len(df)).to_dict()

    # filter breakdown
    fgrp = df.groupby('filter')
    stats['Filter Fraction'] = (fgrp['fieldID'].agg(len) / len(df)).to_dict()

    # add filter ID column
    df['filterID'] = df['filter'].apply(lambda x: FILTER_NAME_TO_ID[x])
    ngrp = df.groupby('night')
    nchanges = ngrp['filterID'].agg(lambda x: np.sum(np.abs(np.diff(x))))<commit_msg>use exposure time to determine program fraction<commit_after>#!/usr/bin/env python

"""Calculate basic efficiency statistics for a ztf_sim run."""
from __future__ import print_function

import sys
# hack to get the path right
sys.path.append('..')
import numpy as np
from collections import OrderedDict
from sqlalchemy import create_engine
import pandas as pd
from astropy.time import Time
import astropy.coordinates as coord
import astropy.units as u
from ztf_sim.constants import *
from ztf_sim.utils import *


def calc_stats(sim_name):

    df = df_read_from_sqlite(sim_name, tablename='Summary', directory='sims')

    stats = OrderedDict()

    stats['Simulation Name'] = sim_name

    # use max - min because there may be no observations some nights
    # TODO: fails if initial nights are weathered out
    stats['Number of Nights'] = df.night.max() - df.night.min() + 1

    # possible nights
    possible_nights = np.linspace(df.night.min(), df.night.max(),
                                  stats['Number of Nights'])

    weathered_out_nights = 0
    actual_nights = df.night.unique()
    for night in possible_nights:
        if night not in actual_nights:
            weathered_out_nights += 1

    stats['Nights completely weathered out'] = weathered_out_nights

    mjds = df.expMJD.apply(np.floor).unique()
    t_mjds = Time(mjds, format='mjd')
    hours_of_darkness = approx_hours_of_darkness(t_mjds).to(u.hour).value

    stats['Average Hours of Darkness'] = np.mean(hours_of_darkness)

    # TODO: use missed visits so we can catch time lost
    stats['Total Science Time (h)'] = \
        (df.visitExpTime.sum() + df.slewTime.sum()) / 3600.

    stats['Average Science Time per night (h)'] = \
        stats['Total Science Time (h)'] / stats['Number of Nights']

    # time used for science: exposing or slewing/filter changing
    stats['Fraction of time usable'] = stats['Total Science Time (h)'] / \
        (stats['Average Hours of Darkness'] * stats['Number of Nights'])

    stats['Average Number of Exposures per hour'] = \
        len(df) / stats['Total Science Time (h)']

    # leave out NaNs from weather/nightly breaks
    w = np.isfinite(df.slewTime)

    # fraction of usable science time
    stats['Open Shutter Fraction'] = df[w].visitExpTime.sum() / \
        (df[w].visitExpTime.sum() + df[w].slewTime.sum())

    stats['Mean Time Between Exposures (s)'] = df[w].slewTime.mean()
    stats['Mean Slew Distance (deg)'] = np.degrees(df[w].slewDist.mean())

    stats['90% Time Between Exposures (s)'] = np.percentile(df[w].slewTime, 90)
    stats['90% Slew Distance (deg)'] = np.percentile(
        np.degrees(df[w].slewDist), 90)

    stats['Median Airmass'] = df.airmass.median()
    stats['90% Airmass'] = np.percentile(df.airmass, 90)

    # program breakdown
    pgrp = df.groupby('propID')
    stats['Program Fraction'] = (pgrp['visitExpTime'].agg(np.sum) /df['visitExpTime'].sum()).to_dict()

    # filter breakdown
    fgrp = df.groupby('filter')
    stats['Filter Fraction'] = (fgrp['fieldID'].agg(len) / len(df)).to_dict()

    # add filter ID column
    df['filterID'] = df['filter'].apply(lambda x: FILTER_NAME_TO_ID[x])
    ngrp = df.groupby('night')
    nchanges = ngrp['filterID'].agg(lambda x: np.sum(np.abs(np.diff(x))))
{'fraction_non_alphanumeric': 0.13471023427866832, 'fraction_numerical': 0.005856966707768188, 'mean_word_length': 3.46900826446281, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '305174', 'n_tokens_mistral': 2224, 'n_tokens_neox': 2095, 'n_words': 561}