<commit_before>"""Produce calibrated images.

``SDTimage`` is a script that, given a list of cross scans composing an
on-the-fly map, is able to calculate the map and save it in FITS format after
cleaning the data.
"""
from __future__ import (absolute_import, division,
                        print_function)

import numpy as np
import astropy
from astropy import wcs
from astropy.table import Table, vstack, Column
import astropy.io.fits as fits
import astropy.units as u
import sys
import warnings
import logging
import traceback
import six
import copy
import functools
from scipy.stats import binned_statistic_2d
from .scan import Scan, chan_re, list_scans, get_channel_feed
from .read_config import read_config, sample_config_file
from .utils import calculate_zernike_moments, calculate_beam_fom, HAS_MAHO
from .utils import compare_anything, ds9_like_log_scale

from .fit import linear_fun
from .interactive_filter import select_data
from .calibration import CalibratorTable
from .opacity import calculate_opacity
from astropy.table.np_utils import TableMergeError

from .global_fit import fit_full_image
from .interactive_filter import create_empty_info

try:
    import matplotlib.pyplot as plt
    from matplotlib.gridspec import GridSpec
    HAS_MPL = True
except ImportError:
    HAS_MPL = False


__all__ = ["ScanSet"]


def _load_calibration(calibration, map_unit):
    caltable = CalibratorTable().read(calibration, path='table')
    caltable.update()
    caltable.compute_conversion_function(map_unit)

    if map_unit == "Jy/beam":
        conversion_units = u.Jy / u.ct
    elif map_unit in ["Jy/pixel", "Jy/sr"]:
        conversion_units = u.Jy / u.ct / u.steradian
    else:
        raise ValueError("Unit for calibration not recognized")
    return caltable, conversion_units


def outlier_score(x):
    """Give a score to data series, larger if higher chance of outliers.

    Inspired by https://stackoverflow.com/questions/22354094/pythonic-way-of-detecting-outliers-in-one-dimensional-observation-data
    """
    median = np.median(x)
    diff = np.abs(x - median)
    # I calculate the mad here instead of using the statsmodels function,
    # in order to avoid duplicating the efforts.
    median_abs_dev = np.median(diff)
    return np.max(0.6745 * diff / median_abs_dev)<commit_msg>Better outlier score<commit_after>"""Produce calibrated images.

``SDTimage`` is a script that, given a list of cross scans composing an
on-the-fly map, is able to calculate the map and save it in FITS format after
cleaning the data.
"""
from __future__ import (absolute_import, division,
                        print_function)

import numpy as np
import astropy
from astropy import wcs
from astropy.table import Table, vstack, Column
import astropy.io.fits as fits
import astropy.units as u
import sys
import warnings
import logging
import traceback
import six
import copy
import functools
from scipy.stats import binned_statistic_2d
from .scan import Scan, chan_re, list_scans, get_channel_feed
from .read_config import read_config, sample_config_file
from .utils import calculate_zernike_moments, calculate_beam_fom, HAS_MAHO
from .utils import compare_anything, ds9_like_log_scale, jit

from .fit import linear_fun
from .interactive_filter import select_data
from .calibration import CalibratorTable
from .opacity import calculate_opacity
from astropy.table.np_utils import TableMergeError

from .global_fit import fit_full_image
from .interactive_filter import create_empty_info

try:
    import matplotlib.pyplot as plt
    from matplotlib.gridspec import GridSpec
    HAS_MPL = True
except ImportError:
    HAS_MPL = False


__all__ = ["ScanSet"]


def _load_calibration(calibration, map_unit):
    caltable = CalibratorTable().read(calibration, path='table')
    caltable.update()
    caltable.compute_conversion_function(map_unit)

    if map_unit == "Jy/beam":
        conversion_units = u.Jy / u.ct
    elif map_unit in ["Jy/pixel", "Jy/sr"]:
        conversion_units = u.Jy / u.ct / u.steradian
    else:
        raise ValueError("Unit for calibration not recognized")
    return caltable, conversion_units


@jit(nopython=True)
def outlier_score(x):
    """Give a score to data series, larger if higher chance of outliers.

    Inspired by https://stackoverflow.com/questions/22354094/pythonic-way-of-detecting-outliers-in-one-dimensional-observation-data
    """
    xdiff = np.diff(x)
    good = xdiff != 0
    if not np.any(good):
        return 0
    xdiff = xdiff[good]
    if len(xdiff) < 2:
        return 0
    ref_dev = np.std(xdiff - np.median(xdiff))
    if ref_dev == 0.:
        return 0

    median = np.median(x)
    diff = np.abs(x - median)
    return np.max(0.6745 * diff / ref_dev)
{'fraction_non_alphanumeric': 0.07428571428571429, 'fraction_numerical': 0.007619047619047619, 'mean_word_length': 3.9229166666666666, 'pattern_counts': {'":': 2, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7443089', 'n_tokens_mistral': 1529, 'n_tokens_neox': 1439, 'n_words': 466}