<commit_before>"""
Functions for automatically distinguishing between signal and background
in LA-ICPMS data.

(c) <NAME> : https://github.com/oscarbranson
"""
import warnings
import numpy as np
from scipy.stats import gaussian_kde
from scipy.optimize import curve_fit
from sklearn.preprocessing import scale
from sklearn.cluster import KMeans

from ..helpers.helpers import Bunch, fastgrad, fastsmooth, findmins, bool_2_indices, bool_transitions
from ..helpers.stat_fns import gauss

warnings.filterwarnings("ignore")

def separate_signal(X, transform=None, scaleX=True):
    if transform is not None:
        X = transform(X)
    if scale:
        X = scale(X)
    
    init = np.percentile(X, [5, 95], 0)
    
    return KMeans(2, init=init).fit_predict(X)

def log_nozero(a, **kwargs):
    a[a == 0] = a[a != 0].min()
    return np.log(a)


def autorange(xvar, sig, gwin=7, swin=None, win=30,
              on_mult=(1.5, 1.), off_mult=(1., 1.5),
              nbin=10, transform='log', thresh=None):
    """
    Automatically separates signal and background in an on/off data stream.

    **Step 1: Thresholding.**
    KMeans clustering is used to identify data where the laser is 'on' vs
    where the laser is 'off'.

    **Step 2: Transition Removal.**
    The width of the transition regions between signal and background are
    then determined, and the transitions are excluded from analysis. The
    width of the transitions is determined by fitting a gaussian to the
    smoothed first derivative of the analyte trace, and determining its
    width at a point where the gaussian intensity is at at `conf` time the
    gaussian maximum. These gaussians are fit to subsets of the data
    centered around the transitions regions determined in Step 1, +/- `win`
    data points. The peak is further isolated by finding the minima and
    maxima of a second derivative within this window, and the gaussian is
    fit to the isolated peak.

    Parameters
    ----------
    xvar : array-like
        Independent variable (usually time).
    sig : array-like
        Dependent signal, of shape (nsamples, nfeatures). Should be clear 
        distinction between laser 'on' and 'off' regions.
    gwin : int
        The window used for calculating first derivative.
        Defaults to 7.
    swin : int
        The window used for signal smoothing. If None, ``gwin // 2``.
    win : int
        The width (c +/- win) of the transition data subsets.
        Defaults to 20.
    on_mult and off_mult : tuple, len=2
        Control the width of the excluded transition regions, which is defined
        relative to the peak full-width-half-maximum (FWHM) of the transition
        gradient. The region n * FHWM below the transition, and m * FWHM above
        the tranision will be excluded, where (n, m) are specified in `on_mult`
        and `off_mult`.
        `on_mult` and `off_mult` apply to the off-on and on-off transitions,
        respectively.
        Defaults to (1.5, 1) and (1, 1.5).
    transform : str
        How to transform the data. Default is 'log'.

    Returns
    -------
    fbkg, fsig, ftrn, failed : tuple
        where fbkg, fsig and ftrn are boolean arrays the same length as sig,
        that are True where sig is background, signal and transition, respecively.
        failed contains a list of transition positions where gaussian fitting
        has failed.
    """
    failed = []
    sig = np.asanyarray(sig)

    # smooth signal
    if swin is not None:
        sigs = fastsmooth(sig, swin)
    else:
        sigs = sig<commit_msg>updatred docstring<commit_after>"""
Functions for automatically distinguishing between signal and background
in LA-ICPMS data.

(c) <NAME> : https://github.com/oscarbranson
"""
import warnings
import numpy as np
from scipy.stats import gaussian_kde
from scipy.optimize import curve_fit
from sklearn.preprocessing import scale
from sklearn.cluster import KMeans

from ..helpers.helpers import Bunch, fastgrad, fastsmooth, findmins, bool_2_indices, bool_transitions
from ..helpers.stat_fns import gauss

warnings.filterwarnings("ignore")

def separate_signal(X, transform=None, scaleX=True):
    if transform is not None:
        X = transform(X)
    if scale:
        X = scale(X)
    
    init = np.percentile(X, [5, 95], 0)
    
    return KMeans(2, init=init).fit_predict(X)

def log_nozero(a, **kwargs):
    a[a == 0] = a[a != 0].min()
    return np.log(a)


def autorange(xvar, sig, gwin=7, swin=None, win=30,
              on_mult=(1.5, 1.), off_mult=(1., 1.5),
              nbin=10, transform='log', thresh=None):
    """
    Automatically separates signal and background in an on/off data stream.

    **Step 1: Thresholding.**
    KMeans clustering is used to identify data where the laser is 'on' vs
    where the laser is 'off'.

    **Step 2: Transition Removal.**
    The width of the transition regions between signal and background are
    then determined, and the transitions are excluded from analysis. The
    width of the transitions is determined by fitting a gaussian to the
    smoothed first derivative of the analyte trace, and determining its
    width at a point where the gaussian intensity is at at `conf` time the
    gaussian maximum. These gaussians are fit to subsets of the data
    centered around the transitions regions determined in Step 1, +/- `win`
    data points. The peak is further isolated by finding the minima and
    maxima of a second derivative within this window, and the gaussian is
    fit to the isolated peak.

    Parameters
    ----------
    xvar : array-like
        Independent variable (usually time).
    sig : array-like
        Dependent signal, of shape (nsamples, nfeatures). Should be clear 
        distinction between laser 'on' and 'off' regions.
    gwin : int
        The window used for calculating first derivative.
        Defaults to 7.
    swin : int
        The window used for signal smoothing. If None, signal is not smoothed.
    win : int
        The width (c +/- win) of the transition data subsets.
        Defaults to 20.
    on_mult and off_mult : tuple, len=2
        Control the width of the excluded transition regions, which is defined
        relative to the peak full-width-half-maximum (FWHM) of the transition
        gradient. The region n * FHWM below the transition, and m * FWHM above
        the tranision will be excluded, where (n, m) are specified in `on_mult`
        and `off_mult`.
        `on_mult` and `off_mult` apply to the off-on and on-off transitions,
        respectively.
        Defaults to (1.5, 1) and (1, 1.5).
    transform : str
        How to transform the data. Default is 'log'.

    Returns
    -------
    fbkg, fsig, ftrn, failed : tuple
        where fbkg, fsig and ftrn are boolean arrays the same length as sig,
        that are True where sig is background, signal and transition, respecively.
        failed contains a list of transition positions where gaussian fitting
        has failed.
    """
    failed = []
    sig = np.asanyarray(sig)

    # smooth signal
    if swin is not None:
        sigs = fastsmooth(sig, swin)
    else:
        sigs = sig
{'fraction_non_alphanumeric': 0.07993292342090554, 'fraction_numerical': 0.009083286752375628, 'mean_word_length': 2.7140633108458743, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5500683', 'n_tokens_mistral': 2102, 'n_tokens_neox': 2006, 'n_words': 878}