<commit_before>"""
Provides support for modeling a universal quasar continuum.

Test laptop dev using::

    time ./examples/fitspec.py --boss-root ~/data/boss -i test.txt -o fitspec-test -n 100 --verbose --sklearn --unweighted --z-col 3 --sn-col 5 --save-model --random
    ./examples/plotfitspec.py --boss-root ~/data/boss -i fitspec-test.hdf5 -o output/fitspec-test --force-y --save-model --examples 0 1 2 3 4

Test on darkmatter using::

    time ./examples/fitspec.py --boss-root /data/boss -i sn-sorted.txt -o fitspec-test -n 1000 --verbose --sklearn --unweighted --z-col 3 --sn-col 5 --save-model --absscale 1 --random --tiltweight 1 --restnormweight 1 --obsnormweight 1e-1
    ./examples/plotfitspec.py --boss-root /data/boss -i fitspec-test.hdf5 -o output/fitspec-test --force-y --save-model --examples 150 300 450 600 750

--------------

Profile on darkmatter::

    python -m cProfile -o profile-test.out ./examples/fitspec.py --boss-root /data/boss -i sn-sorted.txt -o specfits-test --verbose --sklearn --unweighted --z-col 3 --sn-col 5 --fix-norm --fix-tilt --obsnormweight 1e-2 --restnormweight 1 -n 10000

List top function calls by time::

    import pstats
    p = pstats.Stats('profile-test.out')
    p.sort_stats('time').print_stats(10)

Generate call tree::

    gprof2dot -f pstats profile-test.out | dot -Tpng -o profile-test.png

--------------

"""

import inspect
import numpy as np
import scipy.sparse
import h5py

import qusp

class ContinuumModel(object):
    """
    Represents a linearized quasar continuum model.
    """
    def __init__(self, transmission_min, transmission_max, continuum_min, continuum_max, continuum_nparams, tiltwave,
                 absorption_min, absorption_max, absorption_modelexp, absorption_scale, 
                 fix_transmission=False, continuum=None, verbose=False):
        """
        Initializes a linearized quasar continuum model using the specified
        parameter limits and values.

        Args:
            transmission_min (float): minimum observed frame wavelength bin center of
                transmission model.
            transmission_max (float): maximum observed frame wavelength bin center of
                transmission model.
            continuum_min (float): minimum rest frame wavelength bin center of
                continuum model.
            continuum_max (float): maximum rest frame wavelength bin center of
                continuum model.
            continuum_nparams (int): number of rest frame bins of continuum model.
            tiltwave (float): pivot wavelength of rest frame spectral tilt.
            absorption_min (float): minimum rest frame wavelength bin center of
                absorption model.
            absorption_max (float): maximum rest frame wavelength bin center of
                absorption model.
            absorption_modelexp (float): exponent of (1+z) factor of absorption model.
            absorption_scale (float): internal scaling of absorption model coefficients.
            verbose (bool, optional): whether or not to print verbose output.
        """
        self.verbose = verbose
        # the number of model params (excluding per target params)
        self.model_nparams = 0 
        # initialize transmission model params
        assert transmission_max > transmission_min, ('transmission_max must be greater than transmission_min')
        self.transmission_wave_min = transmission_min
        self.transmission_wave_max = transmission_max
        transmission_fid_wave = qusp.wavelength.get_fiducial_wavelength(np.arange(4800))
        self.transmission_wave_min_index = np.argmax(transmission_fid_wave > transmission_min)
        self.transmission_wave_max_index = np.argmax(transmission_fid_wave > transmission_max)+1
        self.transmission_wave_centers = qusp.wavelength.get_fiducial_wavelength(
            np.arange(self.transmission_wave_min_index, self.transmission_wave_max_index))

        self.fix_transmission = fix_transmission
        if fix_transmission:
            self.transmission_nparams = 0
        else:

            self.transmission_nparams = len(self.transmission_wave_centers)

        if verbose:
            print ('Observed frame bin centers span [%.2f:%.2f] with %d bins.' %
                (self.transmission_wave_centers[0], self.transmission_wave_centers[-1], self.transmission_nparams))

        self.model_nparams += self.transmission_nparams

        # initialize continuum model params
        assert (continuum_nparams > 0) or continuum is not None, ('must specify number of continuum params or provide continuum')
        self.continuum = continuum
        if continuum:
            self.continuum_wave_centers = continuum.wavelength
            self.continuum_wave_delta = continuum.wavelength[1]-continuum.wavelength[0]
            self.continuum_wave_min = continuum.wavelength[0]-0.5*self.continuum_wave_delta
            self.continuum_wave_max = continuum.wavelength[-1]+0.5*self.continuum_wave_delta
            self.continuum_nparams = 0
        else:
            assert continuum_max > continuum_min, ('continuum_min must be greater than continuum_max')
            self.continuum_wave_min = continuum_min
            self.continuum_wave_max = continuum_max
            self.continuum_nparams = continuum_nparams
            self.continuum_wave_delta = float(continuum_max-continuum_min)/continuum_nparams
            self.continuum_wave_centers = 0.5*self.continuum_wave_delta + np.linspace(
                continuum_min, continuum_max, continuum_nparams, endpoint=False)
<commit_msg>remove empty line<commit_after>"""
Provides support for modeling a universal quasar continuum.

Test laptop dev using::

    time ./examples/fitspec.py --boss-root ~/data/boss -i test.txt -o fitspec-test -n 100 --verbose --sklearn --unweighted --z-col 3 --sn-col 5 --save-model --random
    ./examples/plotfitspec.py --boss-root ~/data/boss -i fitspec-test.hdf5 -o output/fitspec-test --force-y --save-model --examples 0 1 2 3 4

Test on darkmatter using::

    time ./examples/fitspec.py --boss-root /data/boss -i sn-sorted.txt -o fitspec-test -n 1000 --verbose --sklearn --unweighted --z-col 3 --sn-col 5 --save-model --absscale 1 --random --tiltweight 1 --restnormweight 1 --obsnormweight 1e-1
    ./examples/plotfitspec.py --boss-root /data/boss -i fitspec-test.hdf5 -o output/fitspec-test --force-y --save-model --examples 150 300 450 600 750

--------------

Profile on darkmatter::

    python -m cProfile -o profile-test.out ./examples/fitspec.py --boss-root /data/boss -i sn-sorted.txt -o specfits-test --verbose --sklearn --unweighted --z-col 3 --sn-col 5 --fix-norm --fix-tilt --obsnormweight 1e-2 --restnormweight 1 -n 10000

List top function calls by time::

    import pstats
    p = pstats.Stats('profile-test.out')
    p.sort_stats('time').print_stats(10)

Generate call tree::

    gprof2dot -f pstats profile-test.out | dot -Tpng -o profile-test.png

--------------

"""

import inspect
import numpy as np
import scipy.sparse
import h5py

import qusp

class ContinuumModel(object):
    """
    Represents a linearized quasar continuum model.
    """
    def __init__(self, transmission_min, transmission_max, continuum_min, continuum_max, continuum_nparams, tiltwave,
                 absorption_min, absorption_max, absorption_modelexp, absorption_scale, 
                 fix_transmission=False, continuum=None, verbose=False):
        """
        Initializes a linearized quasar continuum model using the specified
        parameter limits and values.

        Args:
            transmission_min (float): minimum observed frame wavelength bin center of
                transmission model.
            transmission_max (float): maximum observed frame wavelength bin center of
                transmission model.
            continuum_min (float): minimum rest frame wavelength bin center of
                continuum model.
            continuum_max (float): maximum rest frame wavelength bin center of
                continuum model.
            continuum_nparams (int): number of rest frame bins of continuum model.
            tiltwave (float): pivot wavelength of rest frame spectral tilt.
            absorption_min (float): minimum rest frame wavelength bin center of
                absorption model.
            absorption_max (float): maximum rest frame wavelength bin center of
                absorption model.
            absorption_modelexp (float): exponent of (1+z) factor of absorption model.
            absorption_scale (float): internal scaling of absorption model coefficients.
            verbose (bool, optional): whether or not to print verbose output.
        """
        self.verbose = verbose
        # the number of model params (excluding per target params)
        self.model_nparams = 0 
        # initialize transmission model params
        assert transmission_max > transmission_min, ('transmission_max must be greater than transmission_min')
        self.transmission_wave_min = transmission_min
        self.transmission_wave_max = transmission_max
        transmission_fid_wave = qusp.wavelength.get_fiducial_wavelength(np.arange(4800))
        self.transmission_wave_min_index = np.argmax(transmission_fid_wave > transmission_min)
        self.transmission_wave_max_index = np.argmax(transmission_fid_wave > transmission_max)+1
        self.transmission_wave_centers = qusp.wavelength.get_fiducial_wavelength(
            np.arange(self.transmission_wave_min_index, self.transmission_wave_max_index))

        self.fix_transmission = fix_transmission
        if fix_transmission:
            self.transmission_nparams = 0
        else:
            self.transmission_nparams = len(self.transmission_wave_centers)

        if verbose:
            print ('Observed frame bin centers span [%.2f:%.2f] with %d bins.' %
                (self.transmission_wave_centers[0], self.transmission_wave_centers[-1], self.transmission_nparams))

        self.model_nparams += self.transmission_nparams

        # initialize continuum model params
        assert (continuum_nparams > 0) or continuum is not None, ('must specify number of continuum params or provide continuum')
        self.continuum = continuum
        if continuum:
            self.continuum_wave_centers = continuum.wavelength
            self.continuum_wave_delta = continuum.wavelength[1]-continuum.wavelength[0]
            self.continuum_wave_min = continuum.wavelength[0]-0.5*self.continuum_wave_delta
            self.continuum_wave_max = continuum.wavelength[-1]+0.5*self.continuum_wave_delta
            self.continuum_nparams = 0
        else:
            assert continuum_max > continuum_min, ('continuum_min must be greater than continuum_max')
            self.continuum_wave_min = continuum_min
            self.continuum_wave_max = continuum_max
            self.continuum_nparams = continuum_nparams
            self.continuum_wave_delta = float(continuum_max-continuum_min)/continuum_nparams
            self.continuum_wave_centers = 0.5*self.continuum_wave_delta + np.linspace(
                continuum_min, continuum_max, continuum_nparams, endpoint=False)

{'fraction_non_alphanumeric': 0.08768005726044556, 'fraction_numerical': 0.013599355819987474, 'mean_word_length': 2.706233421750663, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1247526', 'n_tokens_mistral': 3338, 'n_tokens_neox': 3062, 'n_words': 870}