<commit_before>import tensorflow as tf
import numpy as np
from scipy import ndimage

from .median import median_filter as median_filter_tf
from .median import median_random_filter as median_random_filter_tf
from .median import median_random_pos_size_filter as median_random_pos_size_filter_tf
from .median import median_random_size_filter as median_random_size_filter_tf


def reduce_precision_py(x, npp):
    """
    Reduce the precision of image, the numpy version.
    :param x: a float tensor, which has been scaled to [0, 1].
    :param npp: number of possible values per pixel. E.g. it's 256 for 8-bit gray-scale image, and 2 for binarized image.
    :return: a tensor representing image(s) with lower precision.
    """
    # Note: 0 is a possible value too.
    npp_int = npp - 1
    x_int = np.rint(x * npp_int)
    x_float = x_int / npp_int
    return x_float

def reduce_precision_tf(x, npp):
    """
    Reduce the precision of image, the tensorflow version.
    """
    npp_int = npp - 1
    x_int = tf.rint(tf.multiply(x, npp_int))
    x_float = tf.div(x_int, npp_int)
    return x_float


def bit_depth_py(x, bits):
    precisions = 2**bits
    return reduce_precision_py(x, precisions)

def bit_depth_tf(x, bits):
    precisions = 2**bits
    return reduce_precision_tf(x, precisions)

def binary_filter_tf(x, threshold):
    x_bin = tf.nn.relu(tf.sign(x-threshold))
    return x_bin

def binary_filter_py(x, threshold):
    x_bin = np.maximum(np.sign(x-threshold), 0)<commit_msg>Added random bit dept reduction.<commit_after>import tensorflow as tf
import numpy as np
from scipy import ndimage

from .median import median_filter as median_filter_tf
from .median import median_random_filter as median_random_filter_tf
from .median import median_random_pos_size_filter as median_random_pos_size_filter_tf
from .median import median_random_size_filter as median_random_size_filter_tf


def reduce_precision_py(x, npp):
    """
    Reduce the precision of image, the numpy version.
    :param x: a float tensor, which has been scaled to [0, 1].
    :param npp: number of possible values per pixel. E.g. it's 256 for 8-bit gray-scale image, and 2 for binarized image.
    :return: a tensor representing image(s) with lower precision.
    """
    # Note: 0 is a possible value too.
    npp_int = npp - 1
    x_int = np.rint(x * npp_int)
    x_float = x_int / npp_int
    return x_float

def reduce_precision_tf(x, npp):
    """
    Reduce the precision of image, the tensorflow version.
    """
    npp_int = npp - 1
    x_int = tf.rint(tf.multiply(x, npp_int))
    x_float = tf.div(x_int, npp_int)
    return x_float


def bit_depth_py(x, bits):
    precisions = 2**bits
    return reduce_precision_py(x, precisions)

def bit_depth_tf(x, bits):
    precisions = 2**bits
    return reduce_precision_tf(x, precisions)

def bit_depth_random_py(x, bits, stddev):
    if stddev == 0.:
        rand_array = np.zeros(x.shape)
    else:
        rand_array = np.random.normal(loc=0., scale=stddev, size=x.shape)
    x_random = np.add(x, rand_array)
    return bit_depth_py(x_random, bits)

def bit_depth_random_tf(x, bits, stddev):
    rand_ts = tf.random_normal(x.get_shape(), mean=0, stddev=stddev)
    x_random = tf.add(x, rand_tf)
    return bit_depth_tf(x_random, bits)

def binary_filter_tf(x, threshold):
    x_bin = tf.nn.relu(tf.sign(x-threshold))
    return x_bin

def binary_filter_py(x, threshold):
    x_bin = np.maximum(np.sign(x-threshold), 0)
{'fraction_non_alphanumeric': 0.0901473562554175, 'fraction_numerical': 0.008379081190407396, 'mean_word_length': 3.762035763411279, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '183373', 'n_tokens_mistral': 1259, 'n_tokens_neox': 1208, 'n_words': 349}