<commit_before>import json
import numpy as np
from keras.preprocessing import image
from imagenet_utils import preprocess_input

'''
    From:
    https://blog.keras.io/how-convolutional-neural-networks-see-the-world.html
'''

# util function to convert a tensor into a valid image
def deprocess_image(x):
    # normalize tensor: center on 0., ensure std is 0.1
    x -= x.mean()
    x /= (x.std() + 1e-5)
    x *= 0.1

    # clip to [0, 1]
    x += 0.5
    x = np.clip(x, 0, 1)

    return x

def load_img(input_path, target_shape):
    img = image.load_img(input_path, target_size=target_shape)

    return preprocess_input(
        np.expand_dims(
            image.img_to_array(img),
            axis=0
        )
    )

def get_json(obj):
    return json.dumps(obj, default=get_json_type)
<commit_msg>forgotten method to scale the image<commit_after>import json
import numpy as np
from keras.preprocessing import image
from imagenet_utils import preprocess_input

'''
    From:
    https://blog.keras.io/how-convolutional-neural-networks-see-the-world.html
'''

# util function to convert a tensor into a valid image
def deprocess_image(x):
    # normalize tensor: center on 0., ensure std is 0.1
    x -= x.mean()
    x /= (x.std() + 1e-5)
    x *= 0.1

    # clip to [0, 1]
    x += 0.5
    x = np.clip(x, 0, 1)

    return x

def load_img_scaled(input_path, target_shape):
    return np.expand_dims(
        image.img_to_array(image.load_img(input_path, target_size=target_shape)) / 255.0,
        axis=0
    )

def load_img(input_path, target_shape):
    img = image.load_img(input_path, target_size=target_shape)

    return preprocess_input(
        np.expand_dims(
            image.img_to_array(img),
            axis=0
        )
    )

def get_json(obj):
    return json.dumps(obj, default=get_json_type)

{'fraction_non_alphanumeric': 0.11056105610561057, 'fraction_numerical': 0.018151815181518153, 'mean_word_length': 2.7974947807933193, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5769256', 'n_tokens_mistral': 669, 'n_tokens_neox': 644, 'n_words': 158}