<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on 05/04/17 at 2:48 PM

@author: neil

Program description here

Version 0.0.0
"""

import numpy as np
from astropy.io import fits
from astropy.table import Table
from astropy import units as u
from tqdm import tqdm
import os
try:
    import periodogram_functions2 as pf2
except ModuleNotFoundError:
    raise Exception("Program requires 'periodogram_functions.py'")


# =============================================================================
# Define variables
# =============================================================================
# Deal with choosing a target and data paths
WORKSPACE = "/Astro/Projects/RayPaul_Work/SuperWASP/"
# location of folder to plot files to
PLOTPATH = WORKSPACE + '/Plots/ls_analysis_run_2/'
# file to save periods to
PERIODPATH = WORKSPACE + '/Data/ls_analysis_run_2/'
PERIODPATH += 'light_curve_analysis_periods_regions.fits'
# if True and periodpath file exists we will skip entries that exist
SKIP_DONE = True
# Write to file (if false does not save to file)
WRITE_TO_FILE = True
# Reset - resets all files (i.e. deletes fits and graphs)
RESET = False
# -----------------------------------------------------------------------------
# set database settings
FROM_DATABASE = True
HOSTNAME = 'localhost'
USERNAME = 'root'
PASSWORD = '<PASSWORD>'
DATABASE = 'swasp'
TABLE = 'swasp_sep16_tab'
# program to run
COMMAND = 'python light_curve_analysis.py '
# -----------------------------------------------------------------------------
# whether to show the graph
SHOW = False
# size in inches of the plot
FIGSIZE = (20, 16)
# decide whether to plot nan periods (saves time)
PLOT_NAN_PERIOD = True
# whether to log progress to standard output (print)
LOG = True
# -----------------------------------------------------------------------------
# minimum time period to be sensitive to (5 hours)
TMIN = 5/24.0
# maximum time period to be sensitive to (100 days)
TMAX = 100
# number of samples per peak
SPP = 5
# -----------------------------------------------------------------------------
# random seed for bootstrapping
RANDOM_SEED = 9999
# number of bootstraps to perform
N_BS = 100
# Phase offset
OFFSET = (-0.5, 0.5)
# define the FAP percentiles
PERCENTILES = [pf2.sigma2percentile(1)*100,
               pf2.sigma2percentile(2)*100,
               pf2.sigma2percentile(3)*100]
# -----------------------------------------------------------------------------
# number of peaks to find
NPEAKS = 5
# number of pixels around a peak to class as same peak
BOXSIZE = 5
# percentage around noise peak to rule out true peak
THRESHOLD = 5.0
# percentile (FAP) to cut peaks at (i.e. any below are not used)
CUTPERCENTILE = pf2.sigma2percentile(1.0)*100
# -----------------------------------------------------------------------------
# minimum number of data points to define a sub region
MINPOINTS = 50   # points
# maximum gap between data points to define a sub region
MAXGAP = 20  # days


# =============================================================================
# Define functions
# =============================================================================
def reset(do_reset):
    if do_reset and os.path.exists(PERIODPATH):
        print('\n Removing {0}'.format(PERIODPATH))
        os.remove(PERIODPATH)
    if do_reset and os.path.exists(PLOTPATH):
        files = os.listdir(PLOTPATH)
        for filename in files:
            loc = PLOTPATH + '/' + filename
            print('\n Removing {0}'.format(loc))
            os.remove(loc)


def load_pre_existing_data(sids, dpath):
    """
    Load pre-exisiting data

    :param dpath: string, filepath to the pre-existing data
    :return:<commit_msg>updated paths for uncertainty clipping run and added uncertainty variable to params<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on 05/04/17 at 2:48 PM

@author: neil

Program description here

Version 0.0.0
"""

import numpy as np
from astropy.io import fits
from astropy.table import Table
from astropy import units as u
from tqdm import tqdm
import os
try:
    import periodogram_functions2 as pf2
except ModuleNotFoundError:
    raise Exception("Program requires 'periodogram_functions.py'")


# =============================================================================
# Define variables
# =============================================================================
# Deal with choosing a target and data paths
WORKSPACE = "/Astro/Projects/RayPaul_Work/SuperWASP/"
# location of folder to plot files to
PLOTPATH = WORKSPACE + '/Plots/ls_analysis_run_3/'
# file to save periods to
PERIODPATH = WORKSPACE + '/Data/ls_analysis_run_3/'
PERIODPATH += 'light_curve_analysis_periods_regions.fits'
# if True and periodpath file exists we will skip entries that exist
SKIP_DONE = True
# Write to file (if false does not save to file)
WRITE_TO_FILE = True
# Reset - resets all files (i.e. deletes fits and graphs)
RESET = False
# -----------------------------------------------------------------------------
# set database settings
FROM_DATABASE = True
HOSTNAME = 'localhost'
USERNAME = 'root'
PASSWORD = '<PASSWORD>'
DATABASE = 'swasp'
TABLE = 'swasp_sep16_tab'
# program to run
COMMAND = 'python light_curve_analysis.py '
# -----------------------------------------------------------------------------
# whether to show the graph
SHOW = False
# size in inches of the plot
FIGSIZE = (20, 16)
# decide whether to plot nan periods (saves time)
PLOT_NAN_PERIOD = True
# whether to log progress to standard output (print)
LOG = True
# -----------------------------------------------------------------------------
# minimum time period to be sensitive to (5 hours)
TMIN = 5/24.0
# maximum time period to be sensitive to (100 days)
TMAX = 100
# number of samples per peak
SPP = 5
# -----------------------------------------------------------------------------
# random seed for bootstrapping
RANDOM_SEED = 9999
# number of bootstraps to perform
N_BS = 100
# Phase offset
OFFSET = (-0.5, 0.5)
# define the FAP percentiles
PERCENTILES = [pf2.sigma2percentile(1)*100,
               pf2.sigma2percentile(2)*100,
               pf2.sigma2percentile(3)*100]
# -----------------------------------------------------------------------------
# number of peaks to find
NPEAKS = 5
# number of pixels around a peak to class as same peak
BOXSIZE = 5
# percentage around noise peak to rule out true peak
THRESHOLD = 5.0
# percentile (FAP) to cut peaks at (i.e. any below are not used)
CUTPERCENTILE = pf2.sigma2percentile(1.0)*100
# -----------------------------------------------------------------------------
# minimum number of data points to define a sub region
MINPOINTS = 50   # points
# maximum gap between data points to define a sub region
MAXGAP = 20  # days
# -----------------------------------------------------------------------------
# Data cleaning
UNCERTAINTY_CLIP = 0.01


# =============================================================================
# Define functions
# =============================================================================
def reset(do_reset):
    if do_reset:
        uinput = input("Are you sure you wish to delete all current files?"
                       "[Y]es or [N]o?\t")
        if "Y" in uinput.upper():
            do_reset = True
        else:
            do_reset = False
    if do_reset and os.path.exists(PERIODPATH):
        print('\n Removing {0}'.format(PERIODPATH))
        os.remove(PERIODPATH)
    if do_reset and os.path.exists(PLOTPATH):
        files = os.listdir(PLOTPATH)
        for filename in files:
            loc = PLOTPATH + '/' + filename
            print('\n Removing {0}'.format(loc))
            os.remove(loc)


def load_pre_existing_data(sids, dpath):
    """
    Load pre-exisiting data

    :param dpath: string, filepath to the pre-existing data
    :return:
{'fraction_non_alphanumeric': 0.2764176075098313, 'fraction_numerical': 0.021184828111125206, 'mean_word_length': 2.2645962732919256, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5978530', 'n_tokens_mistral': 2212, 'n_tokens_neox': 2070, 'n_words': 735}