<commit_before>import os
import re
import shutil
import subprocess

from contextlib import suppress

import h5py
import numpy as np
import pandas as pd

from scipy import linalg
from astropy.io import fits
from astropy.table import Table
from astropy.wcs import WCS
from astropy import units as u
from astropy.coordinates import SkyCoord, match_coordinates_sky
from astropy.time import Time
from astropy.stats import sigma_clipped_stats
from astropy.nddata import Cutout2D, PartialOverlapError, NoOverlapError

from tqdm import tqdm, tqdm_notebook

from dateutil.parser import parse as date_parse

from photutils import DAOStarFinder

from pocs.utils.images import fits as fits_utils
from piaa.utils import helpers
from piaa.utils import plot
from piaa.utils.postgres import get_cursor

import logging
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)


def normalize(cube):
    return (cube.T / cube.sum(1)).T

def lookup_sources_for_observation(fits_files=None, filename=None, force_new=False, cursor=None, use_intersection=False):
    if not cursor:
        cursor = get_cursor(port=5433, db_name='v6', db_user='postgres')


    if force_new:
        logger.info(f'Forcing a new source file')
        with suppress(FileNotFoundError):
            os.remove(filename)

    try:
        logger.info(f'Using existing source file: {filename}')
        observation_sources = pd.read_csv(filename, parse_dates=True)
        observation_sources['obstime'] = pd.to_datetime(observation_sources.obstime)
        observation_sources.rename(columns={'id': 'picid'}, inplace=True)

    except FileNotFoundError:
        logger.info(f'Looking up sources in {len(fits_files)} files')
        observation_sources = None

        # Lookup the point sources for all frames
        for fn in tqdm(fits_files):
            point_sources = lookup_point_sources(
                fn, 
                force_new=force_new,
                cursor=cursor,
            )    
            header = fits_utils.getheader(fn)
            obstime = Time(pd.to_datetime(os.path.basename(fn).split('.')[0]))
            exptime = header['EXPTIME'] * u.second
            
            obstime += (exptime / 2)
            <commit_msg>Don't require the DB cursor if just loading files<commit_after>import os
import re
import shutil
import subprocess

from contextlib import suppress

import h5py
import numpy as np
import pandas as pd

from scipy import linalg
from astropy.io import fits
from astropy.table import Table
from astropy.wcs import WCS
from astropy import units as u
from astropy.coordinates import SkyCoord, match_coordinates_sky
from astropy.time import Time
from astropy.stats import sigma_clipped_stats
from astropy.nddata import Cutout2D, PartialOverlapError, NoOverlapError

from tqdm import tqdm, tqdm_notebook

from dateutil.parser import parse as date_parse

from photutils import DAOStarFinder

from pocs.utils.images import fits as fits_utils
from piaa.utils import helpers
from piaa.utils import plot
from piaa.utils.postgres import get_cursor

import logging
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)


def normalize(cube):
    return (cube.T / cube.sum(1)).T

def lookup_sources_for_observation(fits_files=None, filename=None, force_new=False, cursor=None, use_intersection=False):

    if force_new:
        logger.info(f'Forcing a new source file')
        with suppress(FileNotFoundError):
            os.remove(filename)

    try:
        logger.info(f'Using existing source file: {filename}')
        observation_sources = pd.read_csv(filename, parse_dates=True)
        observation_sources['obstime'] = pd.to_datetime(observation_sources.obstime)
        observation_sources.rename(columns={'id': 'picid'}, inplace=True)

    except FileNotFoundError:
        if not cursor:
            cursor = get_cursor(port=5433, db_name='v6', db_user='postgres')
            
        logger.info(f'Looking up sources in {len(fits_files)} files')
        observation_sources = None

        # Lookup the point sources for all frames
        for fn in tqdm(fits_files):
            point_sources = lookup_point_sources(
                fn, 
                force_new=force_new,
                cursor=cursor,
            )    
            header = fits_utils.getheader(fn)
            obstime = Time(pd.to_datetime(os.path.basename(fn).split('.')[0]))
            exptime = header['EXPTIME'] * u.second
            
            obstime += (exptime / 2)
            
{'fraction_non_alphanumeric': 0.0763239875389408, 'fraction_numerical': 0.004450378282153983, 'mean_word_length': 3.22859830667921, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7520577', 'n_tokens_mistral': 1354, 'n_tokens_neox': 1333, 'n_words': 366}