<commit_before>#pragma once

#ifndef SIPL_IMPROC_IMPROC_H
#define SIPL_IMPROC_IMPROC_H

#include <limits>
#include "matrix/Vector.hpp"
#include "matrix/Matrix.hpp"
#include "improc/NearestNeighborInterpolator.hpp"
#include "improc/BilinearInterpolator.hpp"
#include "improc/Utility.hpp"

namespace sipl
{

enum class InterpolateType { NEAREST_NEIGHBOR, BILINEAR };

// Specialized inverse for mat33d
static inline Matrix33d inv(const Matrix33d& m)
{
    double determinant = m(0, 0) * (m(1, 1) * m(2, 2) - m(1, 2) * m(2, 1)) -
                         m(0, 1) * (m(1, 0) * m(2, 2) - m(1, 2) * m(2, 0)) +
                         m(0, 2) * (m(1, 0) * m(2, 1) - m(1, 1) * m(2, 0));
    Matrix33d inverse = {{m(1, 1) * m(2, 2) - m(1, 2) * m(2, 1),
                          m(0, 2) * m(2, 1) - m(0, 1) * m(2, 2),
                          m(0, 1) * m(1, 2) - m(0, 2) * m(1, 1)},
                         {m(1, 2) * m(2, 0) - m(1, 0) * m(2, 2),
                          m(0, 0) * m(2, 2) - m(0, 2) * m(2, 0),
                          m(0, 2) * m(1, 0) - m(0, 0) * m(1, 2)},
                         {m(1, 0) * m(2, 1) - m(1, 1) * m(2, 0),
                          m(0, 1) * m(2, 0) - m(0, 0) * m(2, 1),
                          m(0, 0) * m(1, 1) - m(0, 1) * m(1, 0)}};
    return inverse / determinant;
}

template <typename ElementType, typename InternalType>
MatrixX<ElementType> projective_transform(const MatrixX<ElementType>& image,
                                          const Matrix33d& transform,
                                          const InterpolateType interpolator)
{
    // 1. Figure out sizes of new matrix
    auto c0 = homogenize(transform * Vector3d{0, 0, 1});
    auto c1 = homogenize(transform * Vector3d{0, image.dims[0], 1});
    auto c2 = homogenize(transform * Vector3d{image.dims[1], 0, 1});
    auto c3 = homogenize(transform * Vector3d{image.dims[1], image.dims[0], 1});

    // Raise or lower values as needed
    for (int32_t i = 0; i < 3; ++i) {
        c0[i] = std::round(c0[i]);
    }
    for (int32_t i = 0; i < 3; ++i) {
        c1[i] = std::round(c1[i]);
    }
    for (int32_t i = 0; i < 3; ++i) {
        c2[i] = std::round(c2[i]);
    }
    for (int32_t i = 0; i < 3; ++i) {
        c3[i] = std::round(c3[i]);
    }

    int32_t ymin = int32_t(std::min({c0[1], c1[1], c2[1], c3[1]}));
    int32_t ymax = int32_t(std::max({c0[1], c1[1], c2[1], c3[1]}));
    int32_t xmin = int32_t(std::min({c0[0], c1[0], c2[0], c3[0]}));
    int32_t xmax = int32_t(std::max({c0[0], c1[0], c2[0], c3[0]}));

    // Create new matrix
    Matrix33d inverse = inv(transform);
    MatrixX<ElementType> new_image(ymax - ymin, xmax - xmin);

    // Start filling it
    for (int32_t i = 0; i < new_image.dims[0]; ++i) {
        for (int32_t j = 0; j < new_image.dims[1]; ++j) {
            Vector3d uv{j + xmin, i + ymin, 1};
            Vector3d xy = homogenize(inverse * uv);
            const double x = xy[0];
            const double y = xy[1];

            // Skip any points outside image space
            if (x < 0 || int32_t(x) >= image.dims[1] || y < 0 ||
                int32_t(y) >= image.dims[0]) {
                new_image(i, j) = ElementType(0);
                continue;
            }

            // Need to interpolate
            switch (interpolator) {
            case InterpolateType::NEAREST_NEIGHBOR:
                new_image(i, j) =
                    NearestNeighborInterpolator::interpolate<ElementType,
                                                             InternalType>(
                        image, x, y);
                break;
            case InterpolateType::BILINEAR:
                new_image(i, j) =
                    BilinearInterpolator::interpolate<ElementType,
                                                      InternalType>(
                        image, x, y);
                break;
            }
        }
    }

    return new_image;
}

MatrixX<uint8_t> color_to_grayscale(const MatrixX<RgbPixel>& color)
{
    MatrixX<uint8_t> grayscale(color.dims[0], color.dims[1]);
    for (int32_t i = 0; i < color.dims[0]; ++i) {
        for (int32_t j = 0; j < color.dims[1]; ++j) {
            RgbPixel p = color(i, j);
            grayscale(i, j) = clamp(0.299 * p[0] + 0.587 * p[1] + 0.114 * p[2]);
        }
    }

    return grayscale;
}

// XXX only works for integral-typed matrices for now, need to figure out how to
// get it to work for Vector-type matrices
template <typename Dtype>
MatrixX<Dtype> rotate_image(
    const MatrixX<Dtype>& in_mat,
    const double degrees,
    const InterpolateType type = InterpolateType::BILINEAR)
{
    const double rads = degrees * M_PI / 180;
    Matrix33d rotation_matrix{{std::cos(rads), std::sin(rads), 0},
                              {-std::sin(rads), std::cos(rads), 0},
                              {0, 0, 1}};
    return projective_transform<Dtype, double>(in_mat, rotation_matrix, type);
}

template <typename Dtype>
VectorX<int32_t> histogram(const MatrixX<Dtype>& mat)
{
    const auto max = std::numeric_limits<Dtype>::max();
    VectorX<int32_t> hist(max);
    for (int32_t i = 0; i < mat.size(); ++i) {
        hist[mat[i]]++;
    }
    return hist;
}<commit_msg>Change return type of histogram to be uint32_t since we won't have negative numbers<commit_after>#pragma once

#ifndef SIPL_IMPROC_IMPROC_H
#define SIPL_IMPROC_IMPROC_H

#include <limits>
#include "matrix/Vector.hpp"
#include "matrix/Matrix.hpp"
#include "improc/NearestNeighborInterpolator.hpp"
#include "improc/BilinearInterpolator.hpp"
#include "improc/Utility.hpp"

namespace sipl
{

enum class InterpolateType { NEAREST_NEIGHBOR, BILINEAR };

// Specialized inverse for mat33d
static inline Matrix33d inv(const Matrix33d& m)
{
    double determinant = m(0, 0) * (m(1, 1) * m(2, 2) - m(1, 2) * m(2, 1)) -
                         m(0, 1) * (m(1, 0) * m(2, 2) - m(1, 2) * m(2, 0)) +
                         m(0, 2) * (m(1, 0) * m(2, 1) - m(1, 1) * m(2, 0));
    Matrix33d inverse = {{m(1, 1) * m(2, 2) - m(1, 2) * m(2, 1),
                          m(0, 2) * m(2, 1) - m(0, 1) * m(2, 2),
                          m(0, 1) * m(1, 2) - m(0, 2) * m(1, 1)},
                         {m(1, 2) * m(2, 0) - m(1, 0) * m(2, 2),
                          m(0, 0) * m(2, 2) - m(0, 2) * m(2, 0),
                          m(0, 2) * m(1, 0) - m(0, 0) * m(1, 2)},
                         {m(1, 0) * m(2, 1) - m(1, 1) * m(2, 0),
                          m(0, 1) * m(2, 0) - m(0, 0) * m(2, 1),
                          m(0, 0) * m(1, 1) - m(0, 1) * m(1, 0)}};
    return inverse / determinant;
}

template <typename ElementType, typename InternalType>
MatrixX<ElementType> projective_transform(const MatrixX<ElementType>& image,
                                          const Matrix33d& transform,
                                          const InterpolateType interpolator)
{
    // 1. Figure out sizes of new matrix
    auto c0 = homogenize(transform * Vector3d{0, 0, 1});
    auto c1 = homogenize(transform * Vector3d{0, image.dims[0], 1});
    auto c2 = homogenize(transform * Vector3d{image.dims[1], 0, 1});
    auto c3 = homogenize(transform * Vector3d{image.dims[1], image.dims[0], 1});

    // Raise or lower values as needed
    for (int32_t i = 0; i < 3; ++i) {
        c0[i] = std::round(c0[i]);
    }
    for (int32_t i = 0; i < 3; ++i) {
        c1[i] = std::round(c1[i]);
    }
    for (int32_t i = 0; i < 3; ++i) {
        c2[i] = std::round(c2[i]);
    }
    for (int32_t i = 0; i < 3; ++i) {
        c3[i] = std::round(c3[i]);
    }

    int32_t ymin = int32_t(std::min({c0[1], c1[1], c2[1], c3[1]}));
    int32_t ymax = int32_t(std::max({c0[1], c1[1], c2[1], c3[1]}));
    int32_t xmin = int32_t(std::min({c0[0], c1[0], c2[0], c3[0]}));
    int32_t xmax = int32_t(std::max({c0[0], c1[0], c2[0], c3[0]}));

    // Create new matrix
    Matrix33d inverse = inv(transform);
    MatrixX<ElementType> new_image(ymax - ymin, xmax - xmin);

    // Start filling it
    for (int32_t i = 0; i < new_image.dims[0]; ++i) {
        for (int32_t j = 0; j < new_image.dims[1]; ++j) {
            Vector3d uv{j + xmin, i + ymin, 1};
            Vector3d xy = homogenize(inverse * uv);
            const double x = xy[0];
            const double y = xy[1];

            // Skip any points outside image space
            if (x < 0 || int32_t(x) >= image.dims[1] || y < 0 ||
                int32_t(y) >= image.dims[0]) {
                new_image(i, j) = ElementType(0);
                continue;
            }

            // Need to interpolate
            switch (interpolator) {
            case InterpolateType::NEAREST_NEIGHBOR:
                new_image(i, j) =
                    NearestNeighborInterpolator::interpolate<ElementType,
                                                             InternalType>(
                        image, x, y);
                break;
            case InterpolateType::BILINEAR:
                new_image(i, j) =
                    BilinearInterpolator::interpolate<ElementType,
                                                      InternalType>(
                        image, x, y);
                break;
            }
        }
    }

    return new_image;
}

MatrixX<uint8_t> color_to_grayscale(const MatrixX<RgbPixel>& color)
{
    MatrixX<uint8_t> grayscale(color.dims[0], color.dims[1]);
    for (int32_t i = 0; i < color.dims[0]; ++i) {
        for (int32_t j = 0; j < color.dims[1]; ++j) {
            RgbPixel p = color(i, j);
            grayscale(i, j) = clamp(0.299 * p[0] + 0.587 * p[1] + 0.114 * p[2]);
        }
    }

    return grayscale;
}

// XXX only works for integral-typed matrices for now, need to figure out how to
// get it to work for Vector-type matrices
template <typename Dtype>
MatrixX<Dtype> rotate_image(
    const MatrixX<Dtype>& in_mat,
    const double degrees,
    const InterpolateType type = InterpolateType::BILINEAR)
{
    const double rads = degrees * M_PI / 180;
    Matrix33d rotation_matrix{{std::cos(rads), std::sin(rads), 0},
                              {-std::sin(rads), std::cos(rads), 0},
                              {0, 0, 1}};
    return projective_transform<Dtype, double>(in_mat, rotation_matrix, type);
}

template <typename Dtype>
VectorX<uint32_t> histogram(const MatrixX<Dtype>& mat)
{
    const auto max = std::numeric_limits<Dtype>::max();
    VectorX<uint32_t> hist(max);
    for (int32_t i = 0; i < mat.size(); ++i) {
        hist[mat[i]]++;
    }
    return hist;
}
{'fraction_non_alphanumeric': 0.15865292883037588, 'fraction_numerical': 0.052280099217706544, 'mean_word_length': 1.8626433642818132, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '630783', 'n_tokens_mistral': 4151, 'n_tokens_neox': 3840, 'n_words': 939}