<commit_before>#!/usr/bin/env /bin/bash
#
# resize.sh
#
# Resizes the all the JPG images in the given directory to the required size
# for the recolorization CNN, 256x256.

# Exit the script on error or an undefined variable
set -e
set -u
set -o pipefail

# Program usage, number of parallel processes to spawn, and output image size
USAGE="./resize.sh <image_dir> <output_dir>"
NUM_JOBS=1024
IMAGE_SIZE=256x256

# Check that the number of command line arguments is valid
num_args=$#
if [ ${num_args} -ne 2 ]; then
    printf "Error: Improper number of command line arguments.\n"
    printf "${USAGE}\n"
    exit 1
fi

# Parse the command line arguments
image_dir=$1
output_dir=$2

# Make the output directory if it doesn't exist
mkdir -p ${output_dir}

# Convert images in parallel, using as many processes as possible
find -L ${image_dir} -type f -name '*.jpg' | parallel -j${NUM_JOBS} --progress \
        convert -resize ${IMAGE_SIZE}\! {} ${output_dir}/{/}<commit_msg>Reduced the number of jobs for the resize script parallelization.<commit_after>#!/usr/bin/env /bin/bash
#
# resize.sh
#
# Resizes the all the JPG images in the given directory to the required size
# for the recolorization CNN, 256x256.

# Exit the script on error or an undefined variable
set -e
set -u
set -o pipefail

# Program usage, number of parallel processes to spawn, and output image size
USAGE="./resize.sh <image_dir> <output_dir>"
IMAGE_SIZE=256x256

# Check that the number of command line arguments is valid
num_args=$#
if [ ${num_args} -ne 2 ]; then
    printf "Error: Improper number of command line arguments.\n"
    printf "${USAGE}\n"
    exit 1
fi

# Parse the command line arguments
image_dir=$1
output_dir=$2

# Make the output directory if it doesn't exist
mkdir -p ${output_dir}

# Convert images in parallel, using as many processes as possible
num_cores=$(getconf _NPROCESSORS_ONLN)
num_jobs=$((2 * ${num_cores}))
find -L ${image_dir} -type f -name '*.jpg' | parallel -j${num_jobs} --progress \
        convert -resize ${IMAGE_SIZE}\! {} ${output_dir}/{/}
{'fraction_non_alphanumeric': 0.10905550146056475, 'fraction_numerical': 0.01801363193768257, 'mean_word_length': 4.0244498777506115, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1001176', 'n_tokens_mistral': 688, 'n_tokens_neox': 637, 'n_words': 263}