<commit_before>#!/usr/bin/env python

"""Script to monitor the COS FUV STIM pulses in TIME-TAG observations.

1: move through /smov/cos/Data/ and populate an sql database of STIM locations.
2: Make plots of overall trends, move to webpage.
3: Check through individual datasets, send email of abberant datasets.

"""

__author__ = '<NAME>'
__status__ = 'Active'

import pyfits
import os
import sys
import sqlite3
import matplotlib as mpl
mpl.use('agg')
from matplotlib.ticker import FormatStrFormatter
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import argparse
import datetime
import shutil
import time
import re
from ..support import send_email, corrtag_image
import glob
import numpy as np
from astropy.time import Time
from scipy.stats import linregress
import multiprocessing as mp

import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart

#------------------------------------------------------
#--------------------Constants-------------------------

DATA_DIR = '/smov/cos/Data/'
MONITOR_DIR = '/grp/hst/cos/Monitors/Stim/'
WEB_DIR = '/grp/webpages/COS/stim/'

brf_file = '/grp/hst/cdbs/lref/s7g1700el_brf.fits'
DB_NAME = os.path.join(MONITOR_DIR, 'Stim_Locations.db')

#-----------------------------------------------------

def gaussian(height, center_x, center_y, width_x, width_y):
    """ Returns a gaussian function with the given parameters
    """
    width_x = float(width_x)
    width_y = float(width_y)
    return lambda x, y: height * exp(
        -(((center_x - x) / width_x) ** 2 + ((center_y - y) / width_y) ** 2) / 2)

#-----------------------------------------------------

def find_center(data):
    """ Returns (height, x, y, width_x, width_y)
    the gaussian parameters of a 2D distribution by calculating its
    moments
    """
    total = data.sum()
    if not total:
        return None, None

    X, Y = np.indices(data.shape)
    x = (X * data).sum() / total
    y = (Y * data).sum() / total
    col = data[:, int(y)]
    width_x = np.sqrt(
        abs((np.arange(col.size) - y) ** 2 * col).sum() / col.sum())
    row = data[int(x), :]
    width_y = np.sqrt(
        abs((np.arange(row.size) - x) ** 2 * row).sum() / row.sum())
    height = data.max()

    return y, x

#-----------------------------------------------------

def get_files(search_string='_corrtag_'):
    """ Walk through directories containing all COS exposures
    and return *_flt* path+filename.
    """
    for root, dirs, files in os.walk(DATA_DIR):
        if not re.search('\d\d-\d\d-\d\d\d\d', root):
            continue
        if root.endswith('gzip'):
            continue
        if root.endswith('podfiles'):
            continue
        if 'experimental' in root:
            continue
        if 'Quality' in root:
            continue
        if 'Fasttrack' in root:
            continue
        if 'targets' in root:
            continue
        if 'experimental' in root:
            continue
        if 'Anomalies' in root:
            continue
        if 'Test' in root:
            continue
        print root
        for file_name in files:
            if (search_string in file_name):
                yield root + '/' + file_name

#-----------------------------------------------------

def brf_positions(brftab, segment, position):
    """ Gets the search ranges for a stimpulse from
    the given baseline reference table
    """
    brf = pyfits.getdata(brftab)

    if segment == 'FUVA':
        row = 0
    elif segment == 'FUVB':
        row = 1

    if position == 'ul':
        x_loc = 'SX1'
        y_loc = 'SY1'
    elif position == 'lr':
        x_loc = 'SX2'
        y_loc = 'SY2'

    xcenter = brf[row][x_loc]
    ycenter = brf[row][y_loc]
    xwidth = brf[row]['XWIDTH']
    ywidth = brf[row]['YWIDTH']

    xmin = max(xcenter - xwidth, 0)
    xmax = min(xcenter + xwidth, 16384)
    ymin = max(ycenter - ywidth, 0)
    ymax = min(ycenter + ywidth, 1024)

    return xmin, xmax, ymin, ymax

#-----------------------------------------------------

def find_stims(image, segment, stim, brf_file):
    x1, x2, y1, y2 = brf_positions(brf_file, segment, stim)
    found_x, found_y = find_center(image[y1:y2, x1:x2])
    if (not found_x) and (not found_y):
        return -999, -999

    return found_x + x1, found_y + y1

#-----------------------------------------------------

def check_present(filename, database_name):
    """Returns True if filename is already in sql database,
    otherwise returns False
    """

    db = sqlite3.connect(database_name)
    c = db.cursor()
    c.execute(
     """SELECT * FROM measurements WHERE obsname='%s'""" %
     (filename))

    entries = [item for item in c]
    if len(entries):
        return True
    else:
        return False

#-----------------------------------------------------

def locate_stims(fits_file, start=0, increment=None):

    DAYS_PER_SECOND = 1. / 60. / 60. / 24.

    file_path, file_name = os.path.split(fits_file)

    hdu = pyfits.open(fits_file)
    exptime = hdu[1].header['exptime']
    expstart = hdu[1].header['expstart']
    segment = hdu[0].header['segment']

    # If increment is not supplied, use the rates supplied by the detector
    if not increment:
        if exptime < 10:
            increment = .03
        elif exptime < 100:
            increment = 1
        else:
            increment = 30

    stop = start + increment

    stim_info = []

    while stop < (exptime):
        im = corrtag_image(fits_file,
                           xtype='RAWX',
                           ytype='RAWY',
                           times=(start, stop))
        
        sci_counts = im.sum()
        ABS_TIME = expstart + start * DAYS_PER_SECOND

        found_ul_x, found_ul_y = find_stims(im, segment, 'ul', brf_file)
        found_lr_x, found_lr_y = find_stims(im, segment, 'lr', brf_file)<commit_msg>doubled the increment for each exptime to prevent floating point mismatches while binning in time.  CalCOS uses 200 seconds anyway, so i'm still critically oversampling<commit_after>#!/usr/bin/env python

"""Script to monitor the COS FUV STIM pulses in TIME-TAG observations.

1: move through /smov/cos/Data/ and populate an sql database of STIM locations.
2: Make plots of overall trends, move to webpage.
3: Check through individual datasets, send email of abberant datasets.

"""

__author__ = '<NAME>'
__status__ = 'Active'

import pyfits
import os
import sys
import sqlite3
import matplotlib as mpl
mpl.use('agg')
from matplotlib.ticker import FormatStrFormatter
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import argparse
import datetime
import shutil
import time
import re
from ..support import send_email, corrtag_image
import glob
import numpy as np
from astropy.time import Time
from scipy.stats import linregress
import multiprocessing as mp

import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart

#------------------------------------------------------
#--------------------Constants-------------------------

DATA_DIR = '/smov/cos/Data/'
MONITOR_DIR = '/grp/hst/cos/Monitors/Stim/'
WEB_DIR = '/grp/webpages/COS/stim/'

brf_file = '/grp/hst/cdbs/lref/s7g1700el_brf.fits'
DB_NAME = os.path.join(MONITOR_DIR, 'Stim_Locations.db')

#-----------------------------------------------------

def gaussian(height, center_x, center_y, width_x, width_y):
    """ Returns a gaussian function with the given parameters
    """
    width_x = float(width_x)
    width_y = float(width_y)
    return lambda x, y: height * exp(
        -(((center_x - x) / width_x) ** 2 + ((center_y - y) / width_y) ** 2) / 2)

#-----------------------------------------------------

def find_center(data):
    """ Returns (height, x, y, width_x, width_y)
    the gaussian parameters of a 2D distribution by calculating its
    moments
    """
    total = data.sum()
    if not total:
        return None, None

    X, Y = np.indices(data.shape)
    x = (X * data).sum() / total
    y = (Y * data).sum() / total
    col = data[:, int(y)]
    width_x = np.sqrt(
        abs((np.arange(col.size) - y) ** 2 * col).sum() / col.sum())
    row = data[int(x), :]
    width_y = np.sqrt(
        abs((np.arange(row.size) - x) ** 2 * row).sum() / row.sum())
    height = data.max()

    return y, x

#-----------------------------------------------------

def get_files(search_string='_corrtag_'):
    """ Walk through directories containing all COS exposures
    and return *_flt* path+filename.
    """
    for root, dirs, files in os.walk(DATA_DIR):
        if not re.search('\d\d-\d\d-\d\d\d\d', root):
            continue
        if root.endswith('gzip'):
            continue
        if root.endswith('podfiles'):
            continue
        if 'experimental' in root:
            continue
        if 'Quality' in root:
            continue
        if 'Fasttrack' in root:
            continue
        if 'targets' in root:
            continue
        if 'experimental' in root:
            continue
        if 'Anomalies' in root:
            continue
        if 'Test' in root:
            continue
        print root
        for file_name in files:
            if (search_string in file_name):
                yield root + '/' + file_name

#-----------------------------------------------------

def brf_positions(brftab, segment, position):
    """ Gets the search ranges for a stimpulse from
    the given baseline reference table
    """
    brf = pyfits.getdata(brftab)

    if segment == 'FUVA':
        row = 0
    elif segment == 'FUVB':
        row = 1

    if position == 'ul':
        x_loc = 'SX1'
        y_loc = 'SY1'
    elif position == 'lr':
        x_loc = 'SX2'
        y_loc = 'SY2'

    xcenter = brf[row][x_loc]
    ycenter = brf[row][y_loc]
    xwidth = brf[row]['XWIDTH']
    ywidth = brf[row]['YWIDTH']

    xmin = max(xcenter - xwidth, 0)
    xmax = min(xcenter + xwidth, 16384)
    ymin = max(ycenter - ywidth, 0)
    ymax = min(ycenter + ywidth, 1024)

    return xmin, xmax, ymin, ymax

#-----------------------------------------------------

def find_stims(image, segment, stim, brf_file):
    x1, x2, y1, y2 = brf_positions(brf_file, segment, stim)
    found_x, found_y = find_center(image[y1:y2, x1:x2])
    if (not found_x) and (not found_y):
        return -999, -999

    return found_x + x1, found_y + y1

#-----------------------------------------------------

def check_present(filename, database_name):
    """Returns True if filename is already in sql database,
    otherwise returns False
    """

    db = sqlite3.connect(database_name)
    c = db.cursor()
    c.execute(
     """SELECT * FROM measurements WHERE obsname='%s'""" %
     (filename))

    entries = [item for item in c]
    if len(entries):
        return True
    else:
        return False

#-----------------------------------------------------

def locate_stims(fits_file, start=0, increment=None):

    DAYS_PER_SECOND = 1. / 60. / 60. / 24.

    file_path, file_name = os.path.split(fits_file)

    hdu = pyfits.open(fits_file)
    exptime = hdu[1].header['exptime']
    expstart = hdu[1].header['expstart']
    segment = hdu[0].header['segment']

    # If increment is not supplied, use the rates supplied by the detector
    if not increment:
        if exptime < 10:
            increment = .03
        elif exptime < 100:
            increment = 1
        else:
            increment = 30

        increment *= 2

    stop = start + increment

    stim_info = []

    while stop < (exptime):
        im = corrtag_image(fits_file,
                           xtype='RAWX',
                           ytype='RAWY',
                           times=(start, stop))
        
        sci_counts = im.sum()
        ABS_TIME = expstart + start * DAYS_PER_SECOND

        found_ul_x, found_ul_y = find_stims(im, segment, 'ul', brf_file)
        found_lr_x, found_lr_y = find_stims(im, segment, 'lr', brf_file)
{'fraction_non_alphanumeric': 0.18310330405687997, 'fraction_numerical': 0.012379757423672104, 'mean_word_length': 1.8555051349414855, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6830063', 'n_tokens_mistral': 3904, 'n_tokens_neox': 3725, 'n_words': 1067}