<commit_before>import tensorflow as tf


class TensorFlowModel:
    def __init__(self, model, bounds, preprocessing=None):
        assert tf.executing_eagerly()
        self._bounds = bounds
        self._model = model
        self._init_preprocessing(preprocessing)

    def _init_preprocessing(self, preprocessing):
        if preprocessing is None:
            preprocessing = dict()
        assert set(preprocessing.keys()) - {"mean", "std", "axis", "flip_axis"} == set()
        mean = preprocessing.get("mean", None)
        std = preprocessing.get("std", None)
        axis = preprocessing.get("axis", None)
        flip_axis = preprocessing.get("flip_axis", None)

        if mean is not None:
            mean = tf.convert_to_tensor(mean)
        if std is not None:
            std = tf.convert_to_tensor(std)

        if axis is not None:
            assert (
                axis < 0
            ), "axis must be negative integer, with -1 representing the last axis"
            shape = (1,) * (abs(axis) - 1)
            if mean is not None:
                assert (
                    mean.ndim == 1
                ), "If axis is specified, mean should be 1-dimensional"
                mean = tf.reshape(mean, mean.shape + shape)
            if std is not None:
                assert (
                    std.ndim == 1
                ), "If axis is specified, std should be 1-dimensional"
                std = tf.reshape(std, std.shape + shape)

        self._preprocessing_mean = mean
        self._preprocessing_std = std
        self._preprocessing_flip_axis = flip_axis

    def _preprocess(self, inputs):
        x = inputs
        if self._preprocessing_flip_axis is not None:
            x = tf.reverse(x, self._preprocessing_flip_axis)
        if self._preprocessing_mean is not None:
            x = x - self._preprocessing_mean
        if self._preprocessing_std is not None:
            x = x / self._preprocessing_std
        assert x.dtype == inputs.dtype
        return x

    def bounds(self):
        return self._bounds

    def forward(self, inputs):
        x = inputs
        x = self._preprocess(x)<commit_msg>fixed tensorflow preprocessing<commit_after>import tensorflow as tf


class TensorFlowModel:
    def __init__(self, model, bounds, preprocessing=None):
        assert tf.executing_eagerly()
        self._bounds = bounds
        self._model = model
        self._init_preprocessing(preprocessing)

    def _init_preprocessing(self, preprocessing):
        if preprocessing is None:
            preprocessing = dict()
        assert set(preprocessing.keys()) - {"mean", "std", "axis", "flip_axis"} == set()
        mean = preprocessing.get("mean", None)
        std = preprocessing.get("std", None)
        axis = preprocessing.get("axis", None)
        flip_axis = preprocessing.get("flip_axis", None)

        if mean is not None:
            mean = tf.convert_to_tensor(mean)
        if std is not None:
            std = tf.convert_to_tensor(std)

        if axis is not None:
            assert (
                axis < 0
            ), "axis must be negative integer, with -1 representing the last axis"
            shape = (1,) * (abs(axis) - 1)
            if mean is not None:
                assert (
                    mean.ndim == 1
                ), "If axis is specified, mean should be 1-dimensional"
                mean = tf.reshape(mean, mean.shape + shape)
            if std is not None:
                assert (
                    std.ndim == 1
                ), "If axis is specified, std should be 1-dimensional"
                std = tf.reshape(std, std.shape + shape)

        self._preprocessing_mean = mean
        self._preprocessing_std = std
        self._preprocessing_flip_axis = flip_axis

    def _preprocess(self, inputs):
        x = inputs
        if self._preprocessing_flip_axis is not None:
            x = tf.reverse(x, (self._preprocessing_flip_axis,))
        if self._preprocessing_mean is not None:
            x = x - self._preprocessing_mean
        if self._preprocessing_std is not None:
            x = x / self._preprocessing_std
        assert x.dtype == inputs.dtype
        return x

    def bounds(self):
        return self._bounds

    def forward(self, inputs):
        x = inputs
        x = self._preprocess(x)
{'fraction_non_alphanumeric': 0.08903792784458835, 'fraction_numerical': 0.0037002775208140612, 'mean_word_length': 1.828646173969915, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2874822', 'n_tokens_mistral': 1215, 'n_tokens_neox': 1159, 'n_words': 349}