<commit_before>'''
Scatterplot smoother with a fixed span. Takes x,y scattered data and returns a set of
(x,s) points that form a smoother curve fiting the data with moving least squares estimates.
Similar to a moving average, but with better characteristics. The fundamental issue
with this smoother is that the choice of span (window size) is not known in advance.
The SuperSmoother uses these smoothers to figure out which span is optimal.

This is a Python port of <NAME>'s 1984 fixed-span Smoother

[1] <NAME>, "A Variable Span Smoother", 1984
        http://www.slac.stanford.edu/cgi-wrap/getdoc/slac-pub-3477.pdf

Example of use::

    s = Smoother()
    s.specify_data_set(x, y)
    s.set_span(0.05)
    s.compute()
    smoothed_y = s.smooth_result


'''

import numpy
import pylab

TWEETER_SPAN = 0.05
MID_SPAN = 0.2
BASS_SPAN = 0.5

DEFAULT_SPANS = (TWEETER_SPAN, MID_SPAN, BASS_SPAN)

class Smoother(object):
    '''
    Smoothers accept data and produce smoother curves that fit the data.
    '''

    def __init__(self):
        self.x = []
        self.y = []

        self._mean_x_in_window = 0.0
        self._mean_y_in_window = 0.0
        self._covariance_in_window = 0.0
        self._variance_in_window = 0.0

        self._span = None
        self.smooth_result = []
        self.cross_validated_residual = None
        self.window_size = None
        self._original_index_of_xvalue = []  # for dealing w/ unsorted data
        self._window_bound_lower = 0
        self._x_in_window = []
        self._y_in_window = []
        self._neighbors_on_each_side = None

    def add_data_point_xy(self, x, y):
        """
        add a new data point to the data set to be smoothed
        """
        self.x.append(x)
        self.y.append(y)

    def specify_data_set(self, x_input, y_input, sort_data=False):
        """
        Fully define data by lists of x values and y values.

        This will sort them by increasing x but remember how to unsort them for providing results.

        Parameters
        ----------
        xValues : iterable
            list of floats that represent x
        yValues : iterable
            list of floats that represent y(x) for each x
        sort_data : bool, optional
            If true, the data will be sorted by increasing x values.
        """
        if sort_data:
            xy = sorted(zip(x_input, y_input))
            x, y = zip(*xy)  # pylint: disable=star-args
            x_input_list = list(x_input)
            self._original_index_of_xvalue = [x_input_list.index(xi) for xi in x]
            if len(set(self._original_index_of_xvalue)) != len(x):
                raise RuntimeError('There are some non-unique x-values')
        else:
            x, y = x_input, y_input

        self.x = x
        self.y = y

    def set_span(self, span):
        """
        Set the window-size for computing the least squares fit

        Parameters
        ----------
        span : float
            Fraction on data length N to be considered in smoothing
        """
        self._span = span

    def compute(self):
        """
        Perform the smoothing operations
        """
        raise NotImplementedError

    def plot(self, fname=None):
        """
        Plot the input data and resulting smooth

        Parameters
        ----------
        fname : str, optional
            name of file to produce. If none, will show interactively.
        """
        pylab.figure()
        xy = zip(self.x, self.smooth_result)
        xy.sort()
        x, y = zip(*xy)  # pylint: disable=star-args
        pylab.plot(x, y, '-')
        pylab.plot(self.x, self.y, '.')
        if fname:
            pylab.savefig(fname)
        else:
            pylab.show()
        pylab.close()

    def _store_unsorted_results(self, smooth, residual):
        """
        Convert sorted smooth/residual back to as-input order
        """

        if self._original_index_of_xvalue:
            # data was sorted. Unsort it here.
            self.smooth_result = numpy.zeros(len(self.y))
            self.cross_validated_residual = numpy.zeros(len(residual))
            original_x = numpy.zeros(len(self.y))
            for i, (xval, smooth_val, residual_val) in enumerate(zip(self.x, smooth, residual)):
                original_index = self._original_index_of_xvalue[i]
                original_x[original_index] = xval
                self.smooth_result[original_index] = smooth_val
                self.cross_validated_residual[original_index] = residual_val
                self.x = original_x
        else:
            # no sorting was done. just apply results
            self.smooth_result = smooth
            self.cross_validated_residual = residual



class BasicFixedSpanSmoother(Smoother):
    """
    A basic fixed-span smoother

    Simple least-squares linear local smoother.

    Uses fast updates of means, variances.
    """

    def compute(self):
        """
        Perform the smoothing operations
        """
        self._compute_window_size()
        smooth = []
        residual = []

        x, y = self.x, self.y

        # step through x and y data with a window window_size wide.
        self._update_values_in_window()
        self._update_mean_in_window()
        self._update_variance_in_window()
        for i, (xi, yi) in enumerate(zip(x, y)):
            if ((i - self._neighbors_on_each_side) > 0.0 and
                (i + self._neighbors_on_each_side) < len(x)):
                self._advance_window()
            smooth_here = self._compute_smooth_during_construction(xi)
            residual_here = self._compute_cross_validated_residual_here(xi, yi, smooth_here)
            smooth.append(smooth_here)
            residual.append(residual_here)

        self._store_unsorted_results(smooth, residual)

    def _compute_window_size(self):
        """
        Determine characteristics of symmetric neighborhood with J/2 values on each side
        """
        self._neighbors_on_each_side = int(len(self.x) * self._span) / 2
        self.window_size = self._neighbors_on_each_side * 2 + 1

    def _update_values_in_window(self):
        """
        update which values are in the current window
        """
        window_bound_upper = self._window_bound_lower + self.window_size<commit_msg>Fixed minor Python3 compatibility issue Made window_size be 2 at the least to deal with small data sets.<commit_after>'''
Scatterplot smoother with a fixed span. Takes x,y scattered data and returns a set of
(x,s) points that form a smoother curve fitting the data with moving least squares estimates.
Similar to a moving average, but with better characteristics. The fundamental issue
with this smoother is that the choice of span (window size) is not known in advance.
The SuperSmoother uses these smoothers to figure out which span is optimal.

This is a Python port of <NAME>'s 1982 fixed-span Smoother [Friedman82]_

Example::

    s = Smoother()
    s.specify_data_set(x, y, sort_data = True)
    s.set_span(0.05)
    s.compute()
    smoothed_y = s.smooth_result


'''

import numpy
import pylab

TWEETER_SPAN = 0.05
MID_SPAN = 0.2
BASS_SPAN = 0.5

DEFAULT_SPANS = (TWEETER_SPAN, MID_SPAN, BASS_SPAN)

class Smoother(object):
    '''
    Smoothers accept data and produce smoother curves that fit the data.
    '''

    def __init__(self):
        self.x = []
        self.y = []

        self._mean_x_in_window = 0.0
        self._mean_y_in_window = 0.0
        self._covariance_in_window = 0.0
        self._variance_in_window = 0.0

        self._span = None
        self.smooth_result = []
        self.cross_validated_residual = None
        self.window_size = None
        self._original_index_of_xvalue = []  # for dealing w/ unsorted data
        self._window_bound_lower = 0
        self._x_in_window = []
        self._y_in_window = []
        self._neighbors_on_each_side = None

    def add_data_point_xy(self, x, y):
        """
        add a new data point to the data set to be smoothed
        """
        self.x.append(x)
        self.y.append(y)

    def specify_data_set(self, x_input, y_input, sort_data=False):
        """
        Fully define data by lists of x values and y values.

        This will sort them by increasing x but remember how to unsort them for providing results.

        Parameters
        ----------
        x_input : iterable
            list of floats that represent x
        y_input : iterable
            list of floats that represent y(x) for each x
        sort_data : bool, optional
            If true, the data will be sorted by increasing x values.
        """
        if sort_data:
            xy = sorted(zip(x_input, y_input))
            x, y = zip(*xy)  # pylint: disable=star-args
            x_input_list = list(x_input)
            self._original_index_of_xvalue = [x_input_list.index(xi) for xi in x]
            if len(set(self._original_index_of_xvalue)) != len(x):
                raise RuntimeError('There are some non-unique x-values')
        else:
            x, y = x_input, y_input

        self.x = x
        self.y = y

    def set_span(self, span):
        """
        Set the window-size for computing the least squares fit

        Parameters
        ----------
        span : float
            Fraction on data length N to be considered in smoothing
        """
        self._span = span

    def compute(self):
        """
        Perform the smoothing operations
        """
        raise NotImplementedError

    def plot(self, fname=None):
        """
        Plot the input data and resulting smooth

        Parameters
        ----------
        fname : str, optional
            name of file to produce. If none, will show interactively.
        """
        pylab.figure()
        xy = zip(self.x, self.smooth_result)
        xy.sort()
        x, y = zip(*xy)  # pylint: disable=star-args
        pylab.plot(x, y, '-')
        pylab.plot(self.x, self.y, '.')
        if fname:
            pylab.savefig(fname)
        else:
            pylab.show()
        pylab.close()

    def _store_unsorted_results(self, smooth, residual):
        """
        Convert sorted smooth/residual back to as-input order
        """

        if self._original_index_of_xvalue:
            # data was sorted. Unsort it here.
            self.smooth_result = numpy.zeros(len(self.y))
            self.cross_validated_residual = numpy.zeros(len(residual))
            original_x = numpy.zeros(len(self.y))
            for i, (xval, smooth_val, residual_val) in enumerate(zip(self.x, smooth, residual)):
                original_index = self._original_index_of_xvalue[i]
                original_x[original_index] = xval
                self.smooth_result[original_index] = smooth_val
                self.cross_validated_residual[original_index] = residual_val
                self.x = original_x
        else:
            # no sorting was done. just apply results
            self.smooth_result = smooth
            self.cross_validated_residual = residual



class BasicFixedSpanSmoother(Smoother):
    """
    A basic fixed-span smoother

    Simple least-squares linear local smoother.

    Uses fast updates of means, variances.
    """

    def compute(self):
        """
        Perform the smoothing operations
        """
        self._compute_window_size()
        smooth = []
        residual = []

        x, y = self.x, self.y

        # step through x and y data with a window window_size wide.
        self._update_values_in_window()
        self._update_mean_in_window()
        self._update_variance_in_window()
        for i, (xi, yi) in enumerate(zip(x, y)):
            if ((i - self._neighbors_on_each_side) > 0.0 and
                (i + self._neighbors_on_each_side) < len(x)):
                self._advance_window()
            smooth_here = self._compute_smooth_during_construction(xi)
            residual_here = self._compute_cross_validated_residual_here(xi, yi, smooth_here)
            smooth.append(smooth_here)
            residual.append(residual_here)

        self._store_unsorted_results(smooth, residual)

    def _compute_window_size(self):
        """
        Determine characteristics of symmetric neighborhood with J/2 values on each side
        """
        self._neighbors_on_each_side = int(len(self.x) * self._span) // 2
        self.window_size = self._neighbors_on_each_side * 2 + 1
        if self.window_size <= 1:
            # cannot do averaging with 1 point in window. Force >=2
            self.window_size = 2

    def _update_values_in_window(self):
        """
        update which values are in the current window
        """
        window_bound_upper = self._window_bound_lower + self.window_size
{'fraction_non_alphanumeric': 0.08751376002516119, 'fraction_numerical': 0.005897153640509514, 'mean_word_length': 2.0692567567567566, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1596018', 'n_tokens_mistral': 3989, 'n_tokens_neox': 3737, 'n_words': 1059}