<commit_before>import numpy as np
import Orange.statistics.distribution

from Orange.feature.transformation import ColumnTransformation
from Orange.data.sql.table import SqlTable
from Orange.statistics import contingency

from ..feature import _discretization

def _split_eq_width(dist, n):
    min = dist[0][0]
    max = dist[0][-1]
    if min == max:
        return []
    dif = (max-min)/n
    return [ min + (i+1)*dif for i in range(n-1) ]

def _split_eq_width_fixed(min, max, n):
    if min == max:
        return []
    dif = (max-min)/n
    return [ min + (i+1)*dif for i in range(n-1) ]


class Discretizer(ColumnTransformation):
    """Interval discretizer.
    The lower limits are inclusive, the upper exclusive.
    """
    def __init__(self, variable, points):
        super().__init__(variable)
        self.points = points

    def _transform(self, c):
        if c.size:
            return np.where(np.isnan(c), np.NaN, np.digitize(c, self.points))
        else:
            return np.array([], dtype=int)


def _discretized_var(data, var, points):
    name = "D_" + data.domain[var].name
    var = data.domain[var]

    if len(points) >= 1:
        values = ["<%f" % points[0]] \
            + ["[%f, %f)" % (p1, p2) for p1, p2 in zip(points, points[1:])] \
            + [">=%f" % points[-1]]
        def discretized_attribute():
            return 'bin(%s, ARRAY%s)' % (var.to_sql(), str(list(points)))
    else:
        values = ["single_value"]
        def discretized_attribute():
            return "'%s'" % values[0]

    dvar = Orange.data.variable.DiscreteVariable(name=name, values=values)
    dvar.get_value_from = Discretizer(var, points)
    dvar.to_sql = discretized_attribute
    return dvar


class Discretization:
    """Base class for discretization classes."""
    pass


class EqualFreq(Discretization):
    """Discretizes the feature by spliting its domain to a fixed number of
    equal-width intervals. The span of original variable is the difference
    between the smallest and the largest feature value.<commit_msg>Format the discretized labels using 'number_of_decimals'.<commit_after>import itertools

import numpy as np
import Orange.statistics.distribution

from Orange.feature.transformation import ColumnTransformation
from Orange.data.sql.table import SqlTable
from Orange.statistics import contingency

from ..feature import _discretization

def _split_eq_width(dist, n):
    min = dist[0][0]
    max = dist[0][-1]
    if min == max:
        return []
    dif = (max-min)/n
    return [ min + (i+1)*dif for i in range(n-1) ]

def _split_eq_width_fixed(min, max, n):
    if min == max:
        return []
    dif = (max-min)/n
    return [ min + (i+1)*dif for i in range(n-1) ]


class Discretizer(ColumnTransformation):
    """Interval discretizer.
    The lower limits are inclusive, the upper exclusive.
    """
    def __init__(self, variable, points):
        super().__init__(variable)
        self.points = points

    def _transform(self, c):
        if c.size:
            return np.where(np.isnan(c), np.NaN, np.digitize(c, self.points))
        else:
            return np.array([], dtype=int)


def _fmt_interval(low, high, decimals):
    assert low if low is not None else -np.inf < \
           high if high is not None else np.inf
    assert decimals >= 0

    def fmt_value(value, decimals):
        return (("%%.%if" % decimals) % value).rstrip("0").rstrip(".")

    if (low is None or np.isinf(low)) and \
            not (high is None or np.isinf(high)):
        return "<{}".format(fmt_value(high, decimals))
    elif (high is None or np.isinf(high)) and \
            not (low is None or np.isinf(low)):
        return ">={}".format(fmt_value(low, decimals))
    else:
        return "[{}, {})".format(fmt_value(low, decimals),
                                 fmt_value(high, decimals))


def _discretized_var(data, var, points):
    name = "D_" + data.domain[var].name
    var = data.domain[var]

    def pairwise(iterable):
        "Iterator over neighboring pairs of `iterable`"
        first, second = itertools.tee(iterable, 2)
        next(second)
        yield from zip(first, second)

    if len(points) >= 1:
        values = [_fmt_interval(low, high, var.number_of_decimals)
                  for low, high in pairwise([-np.inf] + list(points) +
                                            [np.inf])]

        def discretized_attribute():
            return 'bin(%s, ARRAY%s)' % (var.to_sql(), str(list(points)))
    else:
        values = ["single_value"]
        def discretized_attribute():
            return "'%s'" % values[0]

    dvar = Orange.data.variable.DiscreteVariable(name=name, values=values)
    dvar.get_value_from = Discretizer(var, points)
    dvar.to_sql = discretized_attribute
    return dvar


class Discretization:
    """Base class for discretization classes."""
    pass


class EqualFreq(Discretization):
    """Discretizes the feature by spliting its domain to a fixed number of
    equal-width intervals. The span of original variable is the difference
    between the smallest and the largest feature value.
{'fraction_non_alphanumeric': 0.123094958968347, 'fraction_numerical': 0.005861664712778429, 'mean_word_length': 2.7013738250180768, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7368678', 'n_tokens_mistral': 1615, 'n_tokens_neox': 1572, 'n_words': 456}