<commit_before>from __future__ import division, print_function
import numpy
from scipy.stats import nanmean, nanmedian
import sys
import logging

ITER = 0


def is_number(s):
    try:
        float(s)
        return True
    except ValueError:
        return False


def cat(dim, *args):
    return numpy.concatenate(args, axis=dim)


def sortrows(a, i):
    i = numpy.argsort(a[:, i])
    b = a[i, :]
    return b


def nan(r, c):
    a = numpy.empty((r, c))
    a[:] = numpy.nan
    return a



def icoshift(xt,  xp,  inter='whole',  n='f',  options=[1,  1,  0,  0,  0],  s_cal=None, average2_multiplier=3):
    '''
    interval Correlation Optimized shifting
    [xcs, ints, ind, target] = icoshift(xt, xp, inter[, n[, options[, s_cal]]])
    Splits a spectral database into "inter" intervals and coshift each vector
    left-right to get the maximum correlation toward a reference or toward an
    average spectrum in that interval. Missing parts on the edges after
    shifting are filled with "closest" value or with "NaNs".
    INPUT
    xt (1 * mt)    : target vector.
                     Use 'average' if you want to use the average spectrum as a reference
                     Use 'median' if you want to use the median spectrum as a reference
                     Use 'max' if you want to use for each segment the corresponding actual spectrum having max features as a reference
                     Use 'average2' for using the average of the average multiplied for a requested number (default=3) as a reference

    xp (np * mp)   : Matrix of sample vectors to be aligned as a sample-set
                     towards common target xt
    inter          : definition of alignment mode
                     'whole'         : it works on the whole spectra (no intervals).
                     nint            : (numeric) number of many intervals.
                     'ndata'         : (string) length of regular intervals
                                       (remainders attached to the last).
                     [I1s I1e, I2s...]: interval definition. ('i(n)s' interval
                                       n start,  'i(n)e' interval n end).
                     (refs:refe)     : shift the whole spectra according to a
                                       reference signal(s) in the region
                                       refs:refe (in sampling points)
                     'refs-refe'     : `shift the whole spectra according to a
                                       reference signal(s) in the region
                                       refs-refe (in s_cal units)
    n (1 * 1)      : (optional)
                     n = integer n.: maximum shift correction in data
                                     points/s_cal units (cf. options[4])
                                     in x/rows. It must be >0
                     n = 'b' (best): the algorithm search for the best n
                                     for each interval (it can be time consuming!)
                     n = 'f' (fast): fast search for the best n for each interval (default)
                     a logging.warn is displayed for each interval if "n" appears too small
    options (1 * 5): (optional)
                     (0) triggers plots & warnings:
                         0 : no on-screen output
                         1 : only warnings (default)
                         2 : warnings and plots
                     (1) selects filling mode
                         0 : using not a number
                         1 : using previous point (default)
                     (2) turns on Co-shift preprocessing
                         0 : no Co-shift preprocessing (default)
                         1 : Executes a Co-shift step before carrying out iCOshift
                     (3) max allowed shift for the Co-shift preprocessing (default = equal to n if not specified)
                         it has to be given in s_cal units if option(5)=1
                     (4) 0 : intervals are given in No. of datapoints  (deafult)
                         1 : intervals are given in ppm --> use s_cal for inter and n
    s_cal           : vector of scalars used as axis for plot (optional)

    average2_multiplier: int (optional)
                    If 'average2' is used as the target, this determines the multiplier used
    OUTPUT
    xcs  (np * mt): shift corrected vector or matrix
    ints (ni * 4) : defined intervals (Int. No.,  starting point,  ending point,  size)
    ind  (np * ni): matrix of indexes reporting how many points each spectrum
                    has been shifted for each interval (+ left,  - right)
    target (1 x mp): actual target used for the final alignment
    Authors:
    <NAME> - Department of Food Science
                         Quality & Technology - Spectroscopy and Chemometrics group
                         Faculty of Sciences
                         University of Copenhagen - Denmark
    email: <EMAIL> - www.models.life.ku.dk
    <NAME> -     Department of Basic Science and Environment
                         Soil and Environmental Chemistry group
                         Faculty of Life Sciences
                         University of Copenhagen - Denmark
    email: <EMAIL> - www.igm.life.ku.dk
    Python conversion by:
    <NAME> -  Rheumatology Research Group
                          Centre for Translational Inflammation Research
                          School of Immunity and Infection
                          University of Birmingham - United Kingdom


    170508 (FrSa) first working code
    211008 (FrSa) improvements and bugs correction
    111108 (Frsa) Splitting into regular intervals (number of intervals or wideness in datapoints) implemented
    141108 (GT)   FFT alignment function implemented
    171108 (FrSa) options implemented
    241108 (FrSa) Automatic search for the best or the fastest n for each interval implemented
    261108 (FrSa) Plots improved
    021208 (FrSa) 'whole' case added & robustness improved
    050309 (GT)   Implentation of interpolation modes (nan); Cosmetics; Graphics
    240309 (GT)   Fixed bug in handling missing values
    060709 (FrSa) 'max' target and output 'target' added. Some speed,  plot and robustness improvements
    241109 (GT)   Interval and band definition in units (added options[4])
    021209 (GT)   Minor debugging for the handling of options[4]
    151209 (FrSa) Cosmetics and minor debugging for the handling of options[4]
    151110 (FrSa) Option 'Max' works now also for alignment towards a reference signal
    310311 (FrSa) Bugfix for the 'whole' case when mp < 101
    030712 (FrSa) Introducing the 'average2' xt (target) for a better automatic target definition. Plots updated to include also this case
    281023 (MF)   Initial implementation of Python version of icoshift algorithm. PLOTS NOT INCLUDED
    '''

    # RETURNS [xcs, ints, ind, target]
    if s_cal is None:
        s_cal = numpy.array(range(0, xp.shape[1]))  # 1:size(xp, 2)]

    # ERRORS CHECK
    # Constant
    # To avoid out of memory errors when 'whole',  the job is divided in
    # blocks of 32MB
    blocksize = 2 ** 25

    if len(s_cal) != max(s_cal.shape):
        logging.error('s_cal must be a vector')
    if max(s_cal.shape) != xp.shape[1]:
        logging.error('x and s_cal are not of compatible length %d vs. %d' %
              (max(s_cal.shape), xp.shape[1]))

    dec_scale = numpy.diff(s_cal)

    inc_scale = s_cal[0] - s_cal[1]
    if inc_scale == 0 or not all(numpy.sign(dec_scale) == - numpy.sign(inc_scale)):
        logging.error('s_cal must be strictly monotonic')

    flag_scale_dir = inc_scale < 0
    flag_di_scale = any(abs(dec_scale) > 2 * numpy.min(abs(dec_scale)))
    max_flag = 0
    avg2_flag = 0
    if xt == 'average':
        xt = numpy.array([nanmean(xp, axis=0), ])
        note = 'average'
    else:
        if xt == 'median':
            xt = numpy.array([nanmedian(xp, axis=0), ])
            note = 'median'
        else:
            if xt == 'average2':
                xt = numpy.array([nanmean(xp, axis=0), ])
                avg2_flag = 1
                note = 'average2'
            else:
                if xt == 'max':
                    xt = numpy.zeros((1, xp.shape[1],))
                    max_flag = 1
                    note = 'max'

    nt, mt = xt.shape
    np, mp = xp.shape
    if mt != mp:
        logging.error('Target "xt" and sample "xp" must have the same number of columns')
    if is_number(inter):
        if inter > mp:
            logging.error('ERROR: number of intervals "inter" must be smaller than number of variables in xp')

<<<<<<< HEAD
=======
    if options[0] < 0 or options[0] > 2:
        error('options(1) must be 0, 1 or 2')
>>>>>>> b24d624ac40a90fd77415d0ebf861220d35084a3

    if options[4]:
        prec = abs(numpy.min(unique(dec_scale)))
        if flag_di_scale:
            logging.warn('icoshift: discontinuous_Scal',
                    's_cal vector is not continuous, the defined intervals might not reflect actual ranges')

    flag_coshift = not inter == 'whole' and options[2]

    if flag_coshift:
        if options[3] == 0:
            n_co = n
        else:
            n_co = options[3]
            if nargin >= 6 and options[4]:
                n_co = dscal2dpts(n_co, s_cal, prec)
        if max_flag:
            xt = nanmean(xp, axis=0)

        xp, nil, wint = icoshift( xt, xp, 'whole', n_co, numpy.array([0, 1, 0]).reshape(1, -1) )<commit_msg>More code cleanup: remove redundant MATLAB ui code and fix function calling<commit_after>from __future__ import division, print_function
import numpy
from scipy.stats import nanmean, nanmedian
import sys
import logging

ITER = 0


def is_number(s):
    try:
        float(s)
        return True
    except ValueError:
        return False


def cat(dim, *args):
    return numpy.concatenate(args, axis=dim)


def sortrows(a, i):
    i = numpy.argsort(a[:, i])
    b = a[i, :]
    return b


def nan(r, c):
    a = numpy.empty((r, c))
    a[:] = numpy.nan
    return a



def icoshift(xt,  xp,  inter='whole',  n='f',  options=[1,  1,  0,  0,  0],  s_cal=None, average2_multiplier=3):
    '''
    interval Correlation Optimized shifting
    [xcs, ints, ind, target] = icoshift(xt, xp, inter[, n[, options[, s_cal]]])
    Splits a spectral database into "inter" intervals and coshift each vector
    left-right to get the maximum correlation toward a reference or toward an
    average spectrum in that interval. Missing parts on the edges after
    shifting are filled with "closest" value or with "NaNs".
    INPUT
    xt (1 * mt)    : target vector.
                     Use 'average' if you want to use the average spectrum as a reference
                     Use 'median' if you want to use the median spectrum as a reference
                     Use 'max' if you want to use for each segment the corresponding actual spectrum having max features as a reference
                     Use 'average2' for using the average of the average multiplied for a requested number (default=3) as a reference

    xp (np * mp)   : Matrix of sample vectors to be aligned as a sample-set
                     towards common target xt
    inter          : definition of alignment mode
                     'whole'         : it works on the whole spectra (no intervals).
                     nint            : (numeric) number of many intervals.
                     'ndata'         : (string) length of regular intervals
                                       (remainders attached to the last).
                     [I1s I1e, I2s...]: interval definition. ('i(n)s' interval
                                       n start,  'i(n)e' interval n end).
                     (refs:refe)     : shift the whole spectra according to a
                                       reference signal(s) in the region
                                       refs:refe (in sampling points)
                     'refs-refe'     : `shift the whole spectra according to a
                                       reference signal(s) in the region
                                       refs-refe (in s_cal units)
    n (1 * 1)      : (optional)
                     n = integer n.: maximum shift correction in data
                                     points/s_cal units (cf. options[4])
                                     in x/rows. It must be >0
                     n = 'b' (best): the algorithm search for the best n
                                     for each interval (it can be time consuming!)
                     n = 'f' (fast): fast search for the best n for each interval (default)
                     a logging.warn is displayed for each interval if "n" appears too small
    options (1 * 5): (optional)
                     (0) triggers plots & warnings:
                         0 : no on-screen output
                         1 : only warnings (default)
                         2 : warnings and plots
                     (1) selects filling mode
                         0 : using not a number
                         1 : using previous point (default)
                     (2) turns on Co-shift preprocessing
                         0 : no Co-shift preprocessing (default)
                         1 : Executes a Co-shift step before carrying out iCOshift
                     (3) max allowed shift for the Co-shift preprocessing (default = equal to n if not specified)
                         it has to be given in s_cal units if option(5)=1
                     (4) 0 : intervals are given in No. of datapoints  (deafult)
                         1 : intervals are given in ppm --> use s_cal for inter and n
    s_cal           : vector of scalars used as axis for plot (optional)

    average2_multiplier: int (optional)
                    If 'average2' is used as the target, this determines the multiplier used
    OUTPUT
    xcs  (np * mt): shift corrected vector or matrix
    ints (ni * 4) : defined intervals (Int. No.,  starting point,  ending point,  size)
    ind  (np * ni): matrix of indexes reporting how many points each spectrum
                    has been shifted for each interval (+ left,  - right)
    target (1 x mp): actual target used for the final alignment
    Authors:
    <NAME> - Department of Food Science
                         Quality & Technology - Spectroscopy and Chemometrics group
                         Faculty of Sciences
                         University of Copenhagen - Denmark
    email: <EMAIL> - www.models.life.ku.dk
    <NAME> -     Department of Basic Science and Environment
                         Soil and Environmental Chemistry group
                         Faculty of Life Sciences
                         University of Copenhagen - Denmark
    email: <EMAIL> - www.igm.life.ku.dk
    Python conversion by:
    <NAME> -  Rheumatology Research Group
                          Centre for Translational Inflammation Research
                          School of Immunity and Infection
                          University of Birmingham - United Kingdom


    170508 (FrSa) first working code
    211008 (FrSa) improvements and bugs correction
    111108 (Frsa) Splitting into regular intervals (number of intervals or wideness in datapoints) implemented
    141108 (GT)   FFT alignment function implemented
    171108 (FrSa) options implemented
    241108 (FrSa) Automatic search for the best or the fastest n for each interval implemented
    261108 (FrSa) Plots improved
    021208 (FrSa) 'whole' case added & robustness improved
    050309 (GT)   Implentation of interpolation modes (nan); Cosmetics; Graphics
    240309 (GT)   Fixed bug in handling missing values
    060709 (FrSa) 'max' target and output 'target' added. Some speed,  plot and robustness improvements
    241109 (GT)   Interval and band definition in units (added options[4])
    021209 (GT)   Minor debugging for the handling of options[4]
    151209 (FrSa) Cosmetics and minor debugging for the handling of options[4]
    151110 (FrSa) Option 'Max' works now also for alignment towards a reference signal
    310311 (FrSa) Bugfix for the 'whole' case when mp < 101
    030712 (FrSa) Introducing the 'average2' xt (target) for a better automatic target definition. Plots updated to include also this case
    281023 (MF)   Initial implementation of Python version of icoshift algorithm. PLOTS NOT INCLUDED
    '''

    # RETURNS [xcs, ints, ind, target]
    if s_cal is None:
        s_cal = numpy.array(range(0, xp.shape[1]))  # 1:size(xp, 2)]

    # ERRORS CHECK
    # Constant
    # To avoid out of memory errors when 'whole',  the job is divided in
    # blocks of 32MB
    blocksize = 2 ** 25

    if len(s_cal) != max(s_cal.shape):
        logging.error('s_cal must be a vector')
    if max(s_cal.shape) != xp.shape[1]:
        logging.error('x and s_cal are not of compatible length %d vs. %d' %
              (max(s_cal.shape), xp.shape[1]))

    dec_scale = numpy.diff(s_cal)

    inc_scale = s_cal[0] - s_cal[1]
    if inc_scale == 0 or not all(numpy.sign(dec_scale) == - numpy.sign(inc_scale)):
        logging.error('s_cal must be strictly monotonic')

    flag_scale_dir = inc_scale < 0
    flag_di_scale = any(abs(dec_scale) > 2 * numpy.min(abs(dec_scale)))
    max_flag = 0
    avg2_flag = 0
    if xt == 'average':
        xt = numpy.array([nanmean(xp, axis=0), ])
        note = 'average'
    else:
        if xt == 'median':
            xt = numpy.array([nanmedian(xp, axis=0), ])
            note = 'median'
        else:
            if xt == 'average2':
                xt = numpy.array([nanmean(xp, axis=0), ])
                avg2_flag = 1
                note = 'average2'
            else:
                if xt == 'max':
                    xt = numpy.zeros((1, xp.shape[1],))
                    max_flag = 1
                    note = 'max'

    nt, mt = xt.shape
    np, mp = xp.shape
    if mt != mp:
        logging.error('Target "xt" and sample "xp" must have the same number of columns')
    if is_number(inter):
        if inter > mp:
            logging.error('ERROR: number of intervals "inter" must be smaller than number of variables in xp')

    if options[4]:
        prec = abs(numpy.min(unique(dec_scale)))
        if flag_di_scale:
            logging.warn('icoshift: discontinuous_Scal',
                    's_cal vector is not continuous, the defined intervals might not reflect actual ranges')

    flag_coshift = not inter == 'whole' and options[2]

    if flag_coshift:
        if options[3] == 0:
            n_co = n
        else:
            n_co = options[3]
            if nargin >= 6 and options[4]:
                n_co = dscal2dpts(n_co, s_cal, prec)
        if max_flag:
            xt = nanmean(xp, axis=0)

        xp, nil, wint = icoshift( xt, xp, 'whole', n_co, numpy.array([0, 1, 0]).reshape(1, -1) )
{'fraction_non_alphanumeric': 0.07920687095748065, 'fraction_numerical': 0.02285017495493585, 'mean_word_length': 1.7907974552448587, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5175910', 'n_tokens_mistral': 5504, 'n_tokens_neox': 5007, 'n_words': 2036}