<commit_before>"""convert_to_records

Convert images to TFRecords
"""

import argparse

import numpy as np
import tensorflow as tf
import tqdm

from scipy.misc import imresize
from tensorflow.examples.tutorials.mnist import input_data

from deep_networks import data_util


def parse_args():
    """Parse command line arguments"""
    parser = argparse.ArgumentParser()
    parser.add_argument('directory', help='Directory of images')
    parser.add_argument('outfile', help='Output tfrecords')
    parser.add_argument(
        '--use-mnist', action='store_true', help='Use MNIST dataset')
    parser.add_argument(
        '--target-height', type=int, default=64, help='Target height')
    parser.add_argument(
        '--target-width', type=int, default=64, help='Target width')
    return parser.parse_args()


def read_mnist(path):
    """Read MNIST images and normalize the data

    :param path: directory of MNIST data
    """
    mnist = input_data.read_data_sets(path)
    images = (mnist.train.images.reshape((-1, 28,
                                          28)) * 255.0).astype(np.uint8)
    labels = mnist.train.labels
    return images, labels


def main():
    args = parse_args()
    if args.use_mnist:
        images, labels = read_mnist(args.directory)

        if args.target_width != 28 or args.target_width != 28:
            images = np.vstack([
                imresize(image, (args.target_height, args.target_width))
                for image in images
            ])
        images = images.reshape((-1, args.target_height, args.target_width, 1))<commit_msg>allow different splits of mnist<commit_after>"""convert_to_records

Convert images to TFRecords
"""

import argparse

import numpy as np
import tensorflow as tf
import tqdm

from scipy.misc import imresize
from tensorflow.examples.tutorials.mnist import input_data

from deep_networks import data_util


def parse_args():
    """Parse command line arguments"""
    parser = argparse.ArgumentParser()
    parser.add_argument('directory', help='Directory of images')
    parser.add_argument('outfile', help='Output tfrecords')
    parser.add_argument(
        '--use-mnist',
        choices=('train', 'test', 'validation'),
        help='Use MNIST dataset')
    parser.add_argument(
        '--target-height', type=int, default=64, help='Target height')
    parser.add_argument(
        '--target-width', type=int, default=64, help='Target width')
    return parser.parse_args()


def read_mnist(path, split):
    """Read MNIST images and normalize the data

    :param path: directory of MNIST data
    """
    mnist = input_data.read_data_sets(path)
    if split == 'train':
        data = mnist.train
    elif split == 'validation':
        data = mnist.validation
    elif split == 'test':
        data = mnist.test

    images = (data.images.reshape((-1, 28, 28)) * 255.0).astype(np.uint8)
    labels = data.labels
    return images, labels


def main():
    args = parse_args()
    if args.use_mnist:
        images, labels = read_mnist(args.directory, args.use_mnist)

        if args.target_width != 28 or args.target_width != 28:
            images = np.vstack([
                imresize(image, (args.target_height, args.target_width))
                for image in images
            ])
        images = images.reshape((-1, args.target_height, args.target_width, 1))
{'fraction_non_alphanumeric': 0.11561382598331346, 'fraction_numerical': 0.011918951132300357, 'mean_word_length': 2.9727810650887574, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4540403', 'n_tokens_mistral': 1058, 'n_tokens_neox': 998, 'n_words': 262}