<commit_before>#include "task_charset.h"
#include "cortex/class.h"
#include "math/gauss.hpp"
#include "math/clamp.hpp"
#include "math/random.hpp"
#include "tensor/numeric.hpp"
#include "text/to_string.hpp"
#include "text/to_params.hpp"
#include "text/from_params.hpp"
#include "tensor/algorithm.hpp"
#include "vision/warp.h"
#include "vision/image_io.h"
#include "vision/convolve.hpp"
#include "vision/bilinear.hpp"

#include "synth_bitstream_vera_sans_mono_bold.h"
#include "synth_bitstream_vera_sans_mono.h"
#include "synth_dejavu_sans_mono_bold.h"
#include "synth_dejavu_sans_mono.h"
#include "synth_droid_sans_mono.h"
#include "synth_liberation_mono_bold.h"
#include "synth_liberation_mono.h"
#include "synth_nimbus_mono_bold.h"
#include "synth_nimbus_mono.h"
#include "synth_oxygen_mono.h"

namespace nano
{
        static rgba_t make_random_rgba()
        {
                random_t<luma_t> rng(0, 255);
                return rgba_t{rng(), rng(), rng(), rng()};
        }

        static rgba_t make_random_opposite_rgba(const rgba_t& rgba)
        {
                random_t<int> rng(-55, +55);
                return  rgba_t
                {
                        static_cast<luma_t>(clamp(255 - static_cast<int>(rgba(0)) + rng(), 0, 255)),
                        static_cast<luma_t>(clamp(255 - static_cast<int>(rgba(1)) + rng(), 0, 255)),
                        static_cast<luma_t>(clamp(255 - static_cast<int>(rgba(2)) + rng(), 0, 255)),
                        static_cast<luma_t>(clamp(255 - static_cast<int>(rgba(3)) + rng(), 0, 255))
                };
        }

        template <>
        inline std::map<nano::charset, std::string> enum_string<nano::charset>()
        {
                return
                {
                        { nano::charset::digit,         "digit" },
                        { nano::charset::lalpha,        "lalpha" },
                        { nano::charset::ualpha,        "ualpha" },
                        { nano::charset::alpha,         "alpha" },
                        { nano::charset::alphanum,      "alphanum" }
                };
        }

        static tensor_size_t obegin(const charset cs)
        {
                switch (cs)
                {
                case charset::digit:            return 0;
                case charset::lalpha:           return 0 + 10;
                case charset::ualpha:           return 0 + 10 + 26;
                case charset::alpha:            return 10;
                case charset::alphanum:         return 0;
                default:                        assert(false); return 0;
                }
        }

        static tensor_size_t oend(const charset cs)
        {
                switch (cs)
                {
                case charset::digit:            return 10;
                case charset::lalpha:           return 10 + 26;
                case charset::ualpha:           return 10 + 26 + 26;
                case charset::alpha:            return 10 + 26 + 26;
                case charset::alphanum:         return 10 + 26 + 26;
                default:                        assert(false); return 0;
                }
        }

        static tensor_size_t osize(const charset cs)
        {
                return oend(cs) - obegin(cs);
        }

        template
        <
                typename tindex,
                typename tsize
        >
        tensor3d_t get_object_patch(const image_tensor_t& image,
                const tindex object_index, const tsize objects, const scalar_t max_offset)
        {
                nano::random_t<scalar_t> rng(-max_offset, max_offset);

                const auto icols = static_cast<int>(image.cols());
                const auto irows = static_cast<int>(image.rows());

                const auto dx = static_cast<scalar_t>(icols) / static_cast<scalar_t>(objects);

                const auto x = dx * static_cast<scalar_t>(object_index) + rng();

                const auto ppx = nano::clamp(nano::cast<int>(x), 0, icols - 1);
                const auto ppw = nano::clamp(nano::cast<int>(dx + rng()), 0, icols - ppx);

                const auto ppy = nano::clamp(nano::cast<int>(rng()), 0, irows - 1);
                const auto pph = nano::clamp(nano::cast<int>(static_cast<scalar_t>(irows) + rng()), 0, irows - ppy);

                tensor3d_t ret(4, pph, ppw);
                ret.matrix(0) = image.matrix(0).block(ppy, ppx, pph, ppw).cast<scalar_t>() / 255;
                ret.matrix(1) = image.matrix(1).block(ppy, ppx, pph, ppw).cast<scalar_t>() / 255;
                ret.matrix(2) = image.matrix(2).block(ppy, ppx, pph, ppw).cast<scalar_t>() / 255;
                ret.matrix(3) = image.matrix(3).block(ppy, ppx, pph, ppw).cast<scalar_t>() / 255;

                return ret;
        }

        tensor3d_t make_random_rgba_image(const tensor_size_t rows, const tensor_size_t cols,
                const rgba_t back_color,
                const scalar_t max_noise, const scalar_t sigma)
        {
                // noisy background
                tensor3d_t image(4, rows, cols);
                image.matrix(0).setConstant(back_color(0) / 255);
                image.matrix(1).setConstant(back_color(1) / 255);
                image.matrix(2).setConstant(back_color(2) / 255);
                image.matrix(3).setConstant(1);

                tensor::add_random(nano::make_rng<scalar_t>(-max_noise, +max_noise),
                        image.matrix(0), image.matrix(1), image.matrix(2));

                // smooth background
                const nano::gauss_kernel_t<scalar_t> back_gauss(sigma);
                nano::convolve(back_gauss, image.matrix(0));
                nano::convolve(back_gauss, image.matrix(1));<commit_msg>fix warning<commit_after>#include "task_charset.h"
#include "cortex/class.h"
#include "math/gauss.hpp"
#include "math/clamp.hpp"
#include "math/random.hpp"
#include "tensor/numeric.hpp"
#include "text/to_string.hpp"
#include "text/to_params.hpp"
#include "text/from_params.hpp"
#include "tensor/algorithm.hpp"
#include "vision/warp.h"
#include "vision/image_io.h"
#include "vision/convolve.hpp"
#include "vision/bilinear.hpp"

#include "synth_bitstream_vera_sans_mono_bold.h"
#include "synth_bitstream_vera_sans_mono.h"
#include "synth_dejavu_sans_mono_bold.h"
#include "synth_dejavu_sans_mono.h"
#include "synth_droid_sans_mono.h"
#include "synth_liberation_mono_bold.h"
#include "synth_liberation_mono.h"
#include "synth_nimbus_mono_bold.h"
#include "synth_nimbus_mono.h"
#include "synth_oxygen_mono.h"

namespace nano
{
        static rgba_t make_random_rgba()
        {
                random_t<luma_t> rng(0, 255);
                return rgba_t{rng(), rng(), rng(), rng()};
        }

        static rgba_t make_random_opposite_rgba(const rgba_t& rgba)
        {
                random_t<int> rng(-55, +55);
                return  rgba_t
                {
                        static_cast<luma_t>(clamp(255 - static_cast<int>(rgba(0)) + rng(), 0, 255)),
                        static_cast<luma_t>(clamp(255 - static_cast<int>(rgba(1)) + rng(), 0, 255)),
                        static_cast<luma_t>(clamp(255 - static_cast<int>(rgba(2)) + rng(), 0, 255)),
                        static_cast<luma_t>(clamp(255 - static_cast<int>(rgba(3)) + rng(), 0, 255))
                };
        }

        template <>
        inline std::map<nano::charset, std::string> enum_string<nano::charset>()
        {
                return
                {
                        { nano::charset::digit,         "digit" },
                        { nano::charset::lalpha,        "lalpha" },
                        { nano::charset::ualpha,        "ualpha" },
                        { nano::charset::alpha,         "alpha" },
                        { nano::charset::alphanum,      "alphanum" }
                };
        }

        static tensor_size_t obegin(const charset cs)
        {
                switch (cs)
                {
                case charset::digit:            return 0;
                case charset::lalpha:           return 0 + 10;
                case charset::ualpha:           return 0 + 10 + 26;
                case charset::alpha:            return 10;
                case charset::alphanum:         return 0;
                default:                        assert(false); return 0;
                }
        }

        static tensor_size_t oend(const charset cs)
        {
                switch (cs)
                {
                case charset::digit:            return 10;
                case charset::lalpha:           return 10 + 26;
                case charset::ualpha:           return 10 + 26 + 26;
                case charset::alpha:            return 10 + 26 + 26;
                case charset::alphanum:         return 10 + 26 + 26;
                default:                        assert(false); return 0;
                }
        }

        static tensor_size_t osize(const charset cs)
        {
                return oend(cs) - obegin(cs);
        }

        template
        <
                typename tindex,
                typename tsize
        >
        tensor3d_t get_object_patch(const image_tensor_t& image,
                const tindex object_index, const tsize objects, const scalar_t max_offset)
        {
                nano::random_t<scalar_t> rng(-max_offset, max_offset);

                const auto icols = static_cast<int>(image.cols());
                const auto irows = static_cast<int>(image.rows());

                const auto dx = static_cast<scalar_t>(icols) / static_cast<scalar_t>(objects);

                const auto x = dx * static_cast<scalar_t>(object_index) + rng();

                const auto ppx = nano::clamp(nano::cast<int>(x), 0, icols - 1);
                const auto ppw = nano::clamp(nano::cast<int>(dx + rng()), 0, icols - ppx);

                const auto ppy = nano::clamp(nano::cast<int>(rng()), 0, irows - 1);
                const auto pph = nano::clamp(nano::cast<int>(static_cast<scalar_t>(irows) + rng()), 0, irows - ppy);

                tensor3d_t ret(4, pph, ppw);
                ret.matrix(0) = image.matrix(0).block(ppy, ppx, pph, ppw).cast<scalar_t>() / scalar_t(255);
                ret.matrix(1) = image.matrix(1).block(ppy, ppx, pph, ppw).cast<scalar_t>() / scalar_t(255);
                ret.matrix(2) = image.matrix(2).block(ppy, ppx, pph, ppw).cast<scalar_t>() / scalar_t(255);
                ret.matrix(3) = image.matrix(3).block(ppy, ppx, pph, ppw).cast<scalar_t>() / scalar_t(255);

                return ret;
        }

        tensor3d_t make_random_rgba_image(const tensor_size_t rows, const tensor_size_t cols,
                const rgba_t back_color,
                const scalar_t max_noise, const scalar_t sigma)
        {
                // noisy background
                tensor3d_t image(4, rows, cols);
                image.matrix(0).setConstant(back_color(0) / scalar_t(255));
                image.matrix(1).setConstant(back_color(1) / scalar_t(255));
                image.matrix(2).setConstant(back_color(2) / scalar_t(255));
                image.matrix(3).setConstant(1);

                tensor::add_random(nano::make_rng<scalar_t>(-max_noise, +max_noise),
                        image.matrix(0), image.matrix(1), image.matrix(2));

                // smooth background
                const nano::gauss_kernel_t<scalar_t> back_gauss(sigma);
                nano::convolve(back_gauss, image.matrix(0));
                nano::convolve(back_gauss, image.matrix(1));
{'fraction_non_alphanumeric': 0.12365591397849462, 'fraction_numerical': 0.022892819979188347, 'mean_word_length': 1.7355313092979128, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5695923', 'n_tokens_mistral': 4013, 'n_tokens_neox': 3517, 'n_words': 660}