<commit_before>#pragma once

#include <roerei/compact_sparse_matrix.hpp>
#include <roerei/sparse_matrix.hpp>
#include <roerei/common.hpp>

#include <cmath>

namespace roerei
{

namespace detail
{
	template<typename T, typename VEC1, typename VEC2, typename F>
	static inline void yield_nonempty_pairs(VEC1 const& xs, VEC2 const& ys, F const& f)
	{
		auto xs_it = xs.begin();
		auto ys_it = ys.begin();
		auto const xs_it_end = xs.end();
		auto const ys_it_end = ys.end();

		while(xs_it != xs_it_end && ys_it != ys_it_end)
		{
			if(xs_it->first == ys_it->first)
			{
				f(xs_it->first, xs_it->second, ys_it->second);
				xs_it++;
				ys_it++;
			}
			else if(xs_it->first < ys_it->first)
			{
				f(xs_it->first, xs_it->second, 0);
				xs_it++;
			}
			else
			{
				f(ys_it->first, 0, ys_it->second);
				ys_it++;
			}
		}

		while(xs_it != xs_it_end)
		{
			f(xs_it->first, xs_it->second, 0);
			xs_it++;
		}

		while(ys_it != ys_it_end)
		{
			f(ys_it->first, 0, ys_it->second);
			ys_it++;
		}
	}
}

class distance
{
private:
	distance() = delete;

	template<typename T>
	static inline float absdiff(T const x, T const y)
	{
		if(x > y)
			return x - y;
		else
			return y - x;
	}

public:
	template<size_t P, typename T, typename VEC1, typename VEC2>
	static inline float minkowski(VEC1 const& xs, VEC2 const& ys)
	{
		static const float p = P;
		assert(xs.size() == ys.size());

		float sum = 0.0f;
		detail::yield_nonempty_pairs<T, VEC1, VEC2>(xs, ys, [&](size_t const, T const& x, T const& y) {
			sum += std::pow(absdiff(x, y), p);
		});
		return std::pow(sum, 1.0f / P);
	}

	template<typename T, typename VEC1, typename VEC2>
	static inline float euclidean(VEC1 const& xs, VEC2 const& ys)
	{
		assert(xs.size() == ys.size());

		float sum = 0.0f;
		detail::yield_nonempty_pairs<T, VEC1 const&, VEC2 const&>(xs, ys, [&](size_t const, T const& x, T const& y) {
			float v = (float)x - (float)y;
			sum += v * v;
		});
		return std::sqrt(sum);
	}
};

}<commit_msg>Fixed up various const/ref issues in distance<commit_after>#pragma once

#include <roerei/compact_sparse_matrix.hpp>
#include <roerei/sparse_matrix.hpp>
#include <roerei/common.hpp>

#include <cmath>

namespace roerei
{

namespace detail
{
	template<typename T, typename VEC1, typename VEC2, typename F>
	static inline void yield_nonempty_pairs(VEC1 const& xs, VEC2 const& ys, F&& f)
	{
		auto xs_it = xs.begin();
		auto ys_it = ys.begin();
		auto const xs_it_end = xs.end();
		auto const ys_it_end = ys.end();

		while(xs_it != xs_it_end && ys_it != ys_it_end)
		{
			if(xs_it->first == ys_it->first)
			{
				f(xs_it->first, xs_it->second, ys_it->second);
				xs_it++;
				ys_it++;
			}
			else if(xs_it->first < ys_it->first)
			{
				f(xs_it->first, xs_it->second, 0);
				xs_it++;
			}
			else
			{
				f(ys_it->first, 0, ys_it->second);
				ys_it++;
			}
		}

		while(xs_it != xs_it_end)
		{
			f(xs_it->first, xs_it->second, 0);
			xs_it++;
		}

		while(ys_it != ys_it_end)
		{
			f(ys_it->first, 0, ys_it->second);
			ys_it++;
		}
	}
}

class distance
{
private:
	distance() = delete;

	template<typename T>
	static inline float absdiff(T const x, T const y)
	{
		if(x > y)
			return x - y;
		else
			return y - x;
	}

public:
	template<size_t P, typename T, typename VEC1, typename VEC2>
	static inline float minkowski(VEC1 const& xs, VEC2 const& ys)
	{
		static const float p = P;
		assert(xs.size() == ys.size());

		float sum = 0.0f;
		detail::yield_nonempty_pairs<T, VEC1 const&, VEC2 const&>(xs, ys, [&](size_t const, T const x, T const y) {
			sum += std::pow(absdiff(x, y), p);
		});
		return std::pow(sum, 1.0f / P);
	}

	template<typename T, typename VEC1, typename VEC2>
	static inline float euclidean(VEC1 const& xs, VEC2 const& ys)
	{
		assert(xs.size() == ys.size());

		float sum = 0.0f;
		detail::yield_nonempty_pairs<T, VEC1 const&, VEC2 const&>(xs, ys, [&](size_t const, T const x, T const y) {
			float v = (float)x - (float)y;
			sum += v * v;
		});
		return std::sqrt(sum);
	}
};

}
{'fraction_non_alphanumeric': 0.16311166875784192, 'fraction_numerical': 0.013048933500627352, 'mean_word_length': 3.337323177366703, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6504392', 'n_tokens_mistral': 1956, 'n_tokens_neox': 1718, 'n_words': 405}