<commit_before>from pathlib import Path

from keras.applications.vgg16 import preprocess_input
from keras.preprocessing import image
from keras.utils.np_utils import to_categorical
import numpy as np


def load_data(path='data/pictures_all', validation_split=0.1, test_split=0.1,
              seed=233, images_per_category=None):
    # Gather and shuffle image paths.
    repo_path = Path(__file__).resolve().parents[2]
    image_paths = list((repo_path / path).glob('*.jpg'))
    np.random.seed(seed)
    np.random.shuffle(image_paths)

    # Gather categories and sort them in alphabet order.
    categories = list(set([get_category(path) for path in image_paths]))
    categories.sort()

    # Split paths into train, val and test sets.
    paths_train = []
    paths_val = []
    paths_test = []
    for cat in categories:
        paths_cat = [path for path in image_paths if get_category(path) == cat]
        vt_edge = int(len(paths_cat) * validation_split)
        tt_edge = int(len(paths_cat) * (validation_split + test_split))
        paths_val += paths_cat[:vt_edge]
        paths_test += paths_cat[vt_edge:tt_edge]
        if images_per_category:
            paths_train += paths_cat[tt_edge:tt_edge + images_per_category]
        else:
            paths_train += paths_cat[tt_edge:]

    # Load images and labels from paths.
    x_train, y_train = load_images(paths_train, categories)
    x_val, y_val = load_images(paths_val, categories)
    x_test, y_test = load_images(paths_test, categories)

    return (x_train, y_train), (x_val, y_val), (x_test, y_test)


def get_category(image_path):
    return image_path.name.split('_')[0]


def load_images(paths, categories):
    xs = []
    ys = []
    for path in paths:
        x = image.load_img(path, target_size=(224, 224))
        x = image.img_to_array(x)
        xs += [x]
        ys += [categories.index(get_category(path))]
    xs = preprocess_input(np.stack(xs))
    ys = to_categorical(ys, nb_classes=len(categories))
    return xs, ys<commit_msg>Split functions so they could be easily used elsewhere<commit_after>from pathlib import Path

from keras.applications.vgg16 import preprocess_input
from keras.preprocessing.image import img_to_array
from keras.preprocessing.image import load_img
from keras.utils.np_utils import to_categorical
import numpy as np


def load_data(path='data/pictures_all', validation_split=0.1, test_split=0.1,
              seed=233, images_per_category=None):
    # Gather and shuffle image paths.
    repo_path = Path(__file__).resolve().parents[2]
    image_paths = list((repo_path / path).glob('*.jpg'))
    np.random.seed(seed)
    np.random.shuffle(image_paths)

    # Gather categories and sort them in alphabet order.
    categories = list(set([get_category(path) for path in image_paths]))
    categories.sort()

    # Split paths into train, val and test sets.
    paths_train = []
    paths_val = []
    paths_test = []
    for cat in categories:
        paths_cat = [path for path in image_paths if get_category(path) == cat]
        vt_edge = int(len(paths_cat) * validation_split)
        tt_edge = int(len(paths_cat) * (validation_split + test_split))
        paths_val += paths_cat[:vt_edge]
        paths_test += paths_cat[vt_edge:tt_edge]
        if images_per_category:
            paths_train += paths_cat[tt_edge:tt_edge + images_per_category]
        else:
            paths_train += paths_cat[tt_edge:]

    # Load images and labels from paths.
    x_train, y_train = load_images_and_labels(paths_train, categories)
    x_val, y_val = load_images_and_labels(paths_val, categories)
    x_test, y_test = load_images_and_labels(paths_test, categories)

    return (x_train, y_train), (x_val, y_val), (x_test, y_test)


def get_category(image_path):
    return image_path.name.split('_')[0]


def load_images_and_labels(paths, categories):
    return load_images(paths), load_labels(paths, categories)


def load_images(paths):
    images = [img_to_array(load_img(path, target_size=(224, 224)))
              for path in paths]
    return preprocess_input(np.stack(images))


def load_labels(paths, categories):
    labels = [categories.index(get_category(path)) for path in paths]
    return to_categorical(labels, nb_classes=len(categories))
{'fraction_non_alphanumeric': 0.10272045028142589, 'fraction_numerical': 0.00797373358348968, 'mean_word_length': 3.7283813747228383, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4098362', 'n_tokens_mistral': 1423, 'n_tokens_neox': 1371, 'n_words': 319}