<commit_before>import os
import fitsio
import numpy as np
from glob import glob
from collections import Counter
from astropy.table import Table
import sys

sys.path.append('desicode')
from desimodel.footprint import radec2pix

_sec_cache = {}
def read_secondary(fn):
    '''Read the given secondary targets file (FITS), with caching.'''
    global _sec_cache
    try:
        return _sec_cache[fn]
    except KeyError:
        pass
    scnd = fitsio.read(fn)
    sidmap = dict([(tid,i) for i,tid in enumerate(scnd['TARGETID']) if tid > 0])
    val = scnd,sidmap
    # Only cache one!!
    _sec_cache = {fn: val}
    #_sec_cache[fn] = val
    return val

def patch(infn = 'desi/target/fiberassign/tiles/trunk/001/fiberassign-001000.fits.gz',
          desiroot = '/global/cfs/cdirs/desi',
          outfn = None,
          ):
    '''Repair the data corruption reported in https://github.com/desihub/fiberassign/pull/375
    for a given single fiberassign file.

    The patching proceeds by reading the FIBERASSIGN hdu, while checking the headers
    for the locations of targeting files used.

    Based on TARGETID, we match objects in the FIBERASSIGN file and
    the targeting files.  For a subset of columns, we check for cases
    where the targeting files contain different values, and patch them
    in to the FIBERASSIGN table.

    First we read the primary target files themselves, then secondary
    targets, then ToO targets.

    If any rows are changed, then an updated file is written out, preserving all other HDUs.

    '''
    log_to_string = True
    logstr = ''
    def log(*a):
        nonlocal logstr
        if log_to_string:
            logstr += ' '.join([str(s) for s in a]) + '\n'
        else:
            print(*a)

    stats = {}
    patched_rows = set()
    patched_neg_tids = False

    log('Reading', infn)
    F = fitsio.FITS(infn)
    primhdr = F[0].read_header()
    #tilera  = primhdr['TILERA']
    #tiledec = primhdr['TILEDEC']
    tileid = primhdr['TILEID']
    fa_surv = primhdr['FA_SURV']
    stats.update(infn=infn, tileid=tileid, fa_surv=fa_surv)

    # Find targeting files / directories in the headers.
    #mtldir = primhdr['MTL']
    targdirs = [primhdr['TARG']]
    for i in range(2, 100):
        key = 'TARG%i' % i
        if not key in primhdr:
            break
        targdirs.append(primhdr[key])
    scndfn = primhdr.get('SCND', '').strip()
    toofn = primhdr.get('TOO', '')

    # Swap in the local file locations. (Header values may start with the string "DESIROOT")
    #mtldir = mtldir.replace('DESIROOT', desiroot)
    targdirs = [targdir.replace('DESIROOT', desiroot)
                for targdir in targdirs]
    scndfn = scndfn.replace('DESIROOT', desiroot)
    toofn = toofn.replace('DESIROOT', desiroot)

    # Tile 80611 has /data as the prefix.
    pre = '/data/'
    targdirs = [targdir.replace(pre, desiroot+'/')
                if targdir.startswith(pre)
                else targdir
                for targdir in targdirs]
    if scndfn.startswith(pre):
        scndfn = scndfn.replace(pre, desiroot+'/')
    if toofn.startswith(pre):
        toofn = toofn.replace(pre, desiroot+'/')

    # Tile 80687 has SCND a directory, not a file.
    if os.path.isdir(scndfn):
        # Tile 80687 has SCND a directory that contains two files, with the same number of rows
        # but one includes DR9 photometry columns.
        fn = os.path.join(scndfn, 'sv1targets-dark-secondary-dr9photometry.fits')
        if os.path.exists(fn):
            log('Special-case updated SCND from', scndfn, 'to', fn)
            scndfn = fn
        # Similar with 80717.
        fn = os.path.join(scndfn, 'sv1targets-bright-secondary-dr9photometry.fits')
        if os.path.exists(fn):
            log('Special-case updated SCND from', scndfn, 'to', fn)
            scndfn = fn

    stats.update(scndfn=scndfn, toofn=toofn)
    for i,t in enumerate(targdirs):
        stats['targdir%i' % i] = t

    # Read the original FIBERASSIGN table.
    hduname = 'FIBERASSIGN'
    ff = F[hduname]
    tab = ff.read()

    # We're going to match based on (positive) TARGETID.
    targetid = tab['TARGETID']
    ra = tab['TARGET_RA']
    dec = tab['TARGET_DEC']
    # Per https://github.com/desihub/fiberassign/issues/385,
    # Swap in unique values for negative TARGETIDs.
    I = np.flatnonzero(targetid < 0)
    stats.update(neg_targetids=len(I), pos_targetids=len(np.flatnonzero(targetid>0)))
    if len(I):
        tab['TARGETID'][I] = -(tileid * 10000 + tab['LOCATION'][I])
        log('Updated', len(I), 'negative TARGETID values')
        patched_neg_tids = True

    # Start with primary target files.  These are split into healpixes, so look up which healpixes
    # contain targets used in this tile.

    # Some objects have NaN TARGET_RA; skip these when looking up the healpix
    Iok = np.flatnonzero(np.isfinite(ra))<commit_msg>gah, more special cases<commit_after>import os
import fitsio
import numpy as np
from glob import glob
from collections import Counter
from astropy.table import Table
import sys

sys.path.append('desicode')
from desimodel.footprint import radec2pix

_sec_cache = {}
def read_secondary(fn):
    '''Read the given secondary targets file (FITS), with caching.'''
    global _sec_cache
    try:
        return _sec_cache[fn]
    except KeyError:
        pass
    scnd = fitsio.read(fn)
    sidmap = dict([(tid,i) for i,tid in enumerate(scnd['TARGETID']) if tid > 0])
    val = scnd,sidmap
    # Only cache one!!
    _sec_cache = {fn: val}
    #_sec_cache[fn] = val
    return val

def patch(infn = 'desi/target/fiberassign/tiles/trunk/001/fiberassign-001000.fits.gz',
          desiroot = '/global/cfs/cdirs/desi',
          outfn = None,
          ):
    '''Repair the data corruption reported in https://github.com/desihub/fiberassign/pull/375
    for a given single fiberassign file.

    The patching proceeds by reading the FIBERASSIGN hdu, while checking the headers
    for the locations of targeting files used.

    Based on TARGETID, we match objects in the FIBERASSIGN file and
    the targeting files.  For a subset of columns, we check for cases
    where the targeting files contain different values, and patch them
    in to the FIBERASSIGN table.

    First we read the primary target files themselves, then secondary
    targets, then ToO targets.

    If any rows are changed, then an updated file is written out, preserving all other HDUs.

    '''
    log_to_string = True
    logstr = ''
    def log(*a):
        nonlocal logstr
        if log_to_string:
            logstr += ' '.join([str(s) for s in a]) + '\n'
        else:
            print(*a)

    stats = {}
    patched_rows = set()
    patched_neg_tids = False

    log('Reading', infn)
    F = fitsio.FITS(infn)
    primhdr = F[0].read_header()
    #tilera  = primhdr['TILERA']
    #tiledec = primhdr['TILEDEC']
    tileid = primhdr['TILEID']
    fa_surv = primhdr['FA_SURV']
    stats.update(infn=infn, tileid=tileid, fa_surv=fa_surv)

    # Find targeting files / directories in the headers.
    #mtldir = primhdr['MTL']
    targdirs = [primhdr['TARG']]
    for i in range(2, 100):
        key = 'TARG%i' % i
        if not key in primhdr:
            break
        targdirs.append(primhdr[key])
    scndfn = primhdr.get('SCND', '').strip()
    toofn = primhdr.get('TOO', '')

    # Swap in the local file locations. (Header values may start with the string "DESIROOT")
    #mtldir = mtldir.replace('DESIROOT', desiroot)
    targdirs = [targdir.replace('DESIROOT', desiroot)
                for targdir in targdirs]
    scndfn = scndfn.replace('DESIROOT', desiroot)
    toofn = toofn.replace('DESIROOT', desiroot)

    # Tile 80611 has /data as the prefix.
    pre = '/data/'
    targdirs = [targdir.replace(pre, desiroot+'/')
                if targdir.startswith(pre)
                else targdir
                for targdir in targdirs]
    if scndfn.startswith(pre):
        scndfn = scndfn.replace(pre, desiroot+'/')
    if toofn.startswith(pre):
        toofn = toofn.replace(pre, desiroot+'/')

    # Tile 80687 has SCND a directory, not a file.
    if os.path.isdir(scndfn):
        # Tile 80687 has SCND a directory that contains two files, with the same number of rows
        # but one includes DR9 photometry columns.
        fn = os.path.join(scndfn, 'sv1targets-dark-secondary-dr9photometry.fits')
        if os.path.exists(fn):
            log('Special-case updated SCND from', scndfn, 'to', fn)
            scndfn = fn
        # Similar with 80717.
        fn = os.path.join(scndfn, 'sv1targets-bright-secondary-dr9photometry.fits')
        if os.path.exists(fn):
            log('Special-case updated SCND from', scndfn, 'to', fn)
            scndfn = fn

    # Special-case a missing SCND file from sv2 (tile 81096+)
    if scndfn == '/global/cfs/cdirs/desi/target/catalogs/dr9/0.53.0/targets/sv2/secondary/dark/sv2targets-dark-secondary.fits':
        log('Ignoring SCND file', scndfn)
        scndfn = ''

    oldfn = '/global/cfs/cdirs/desi/target/catalogs/dr9/0.58.0/targets/main/secondary/dark/maintargets-dark-secondary.fits'
    if scndfn == oldfn:
        newfn = '/global/cfs/cdirs/desi/target/catalogs/dr9/0.58.0/targets/main/secondary/dark/targets-dark-secondary.fits'
        log('Replacing', oldfn, 'with', newfn)
        scndfn = newfn

    stats.update(scndfn=scndfn, toofn=toofn)
    for i,t in enumerate(targdirs):
        stats['targdir%i' % i] = t

    # Read the original FIBERASSIGN table.
    hduname = 'FIBERASSIGN'
    ff = F[hduname]
    tab = ff.read()

    # We're going to match based on (positive) TARGETID.
    targetid = tab['TARGETID']
    ra = tab['TARGET_RA']
    dec = tab['TARGET_DEC']
    # Per https://github.com/desihub/fiberassign/issues/385,
    # Swap in unique values for negative TARGETIDs.
    I = np.flatnonzero(targetid < 0)
    stats.update(neg_targetids=len(I), pos_targetids=len(np.flatnonzero(targetid>0)))
    if len(I):
        tab['TARGETID'][I] = -(tileid * 10000 + tab['LOCATION'][I])
        log('Updated', len(I), 'negative TARGETID values')
        patched_neg_tids = True

    # Start with primary target files.  These are split into healpixes, so look up which healpixes
    # contain targets used in this tile.

    # Some objects have NaN TARGET_RA; skip these when looking up the healpix
    Iok = np.flatnonzero(np.isfinite(ra))
{'fraction_non_alphanumeric': 0.11126108754338604, 'fraction_numerical': 0.012630158118010028, 'mean_word_length': 2.727272727272727, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2044308', 'n_tokens_mistral': 3652, 'n_tokens_neox': 3486, 'n_words': 968}