<commit_before>#define _USE_MATH_DEFINES
#include <cmath>
#include <cassert>
#include <limits>
#include <iomanip>
#include "stream.hpp"
#include "vectnative.hpp"

#ifdef _WIN32
#include <SDKDDKVer.h>
#include <stdio.h>
#else
// verify iostream-free status (see further down)
#if _GLIBCXX_IOSTREAM != 0
#error rogue iostream acquired
#endif
#endif

// use a custom minimalistic implementation of std::istream/ostream to avoid the highly-harmful-for-the-optimisations
// static initialization of std::cin/cout/cerr; our mini-implementation is initialized at the start of main()
namespace stream {
in cin;
out cout;
out cerr;
}

#if _MSC_VER != 0
using namespace simd; // oldest msvc we target does not support using-declarations, so we go full monty there
#else
#if NATIVE_F64X4 != 0
using simd::f64x4;
using simd::as_f64x4;
using simd::get_f64x2;
#endif
#if NATIVE_S64X4 != 0
using simd::s64x4;
using simd::as_s64x4;
using simd::get_s64x2;
#endif
#if NATIVE_U64X4 != 0
using simd::u64x4;
using simd::as_u64x4;
using simd::get_u64x2;
#endif
#if NATIVE_F32X8 != 0
using simd::f32x8;
using simd::as_f32x8;
using simd::get_f32x4;
#endif
#if NATIVE_S32X8 != 0
using simd::s32x8;
using simd::as_s32x8;
using simd::get_s32x4;
#endif
#if NATIVE_U32X8 != 0
using simd::u32x8;
using simd::as_u32x8;
using simd::get_u32x4;
#endif
#if NATIVE_F32X2 != 0
using simd::f32x2;
using simd::as_f32x2;
#endif
#if NATIVE_S32X2 != 0
using simd::s32x2;
using simd::as_s32x2;
#endif
#if NATIVE_U32X2 != 0
using simd::u32x2;
using simd::as_u32x2;
#endif
#if NATIVE_S16X16 != 0
using simd::s16x16;
using simd::as_s16x16;
using simd::get_s16x8;
#endif
#if NATIVE_U16X16 != 0
using simd::u16x16;
using simd::as_u16x16;
using simd::get_u16x8;
#endif
#if NATIVE_S16X4 != 0
using simd::s16x4;
using simd::as_s16x4;
#endif
#if NATIVE_U16X4 != 0
using simd::u16x4;
using simd::as_u16x4;
#endif
using simd::f64x2;
using simd::s64x2;
using simd::u64x2;
using simd::f32x4;
using simd::s32x4;
using simd::u32x4;
using simd::s16x8;
using simd::u16x8;
using simd::as_f64x2;
using simd::as_s64x2;
using simd::as_u64x2;
using simd::as_f32x4;
using simd::as_s32x4;
using simd::as_u32x4;
using simd::as_s16x8;
using simd::as_u16x8;
using simd::all;
using simd::any;
using simd::none;
using simd::min;
using simd::max;
using simd::abs;
using simd::mask;
using simd::select;
using simd::sqrt;
using simd::log;
using simd::exp;
using simd::pow;
using simd::sin;
using simd::cos;
using simd::sincos;
using simd::transpose4x4;
using simd::operator ==;
using simd::operator !=;
using simd::operator <;
using simd::operator >;
using simd::operator <=;
using simd::operator >=;
using simd::flag_zero;
using simd::flag_native;
#endif

template < typename T >
const T& min(const T& a, const T& b) {
	return (a < b) ? a : b;
}

template < typename T >
const T& max(const T& a, const T& b) {
	return (a > b) ? a : b;
}

class formatter_f64 {
	double f;

public:
	formatter_f64(const double a) : f(a) {}

	uint64_t get() const {
		return reinterpret_cast< const uint64_t& >(f);
	}
};<commit_msg>deprecate old msvc in unittest in common<commit_after>#define _USE_MATH_DEFINES
#include <cmath>
#include <cassert>
#include <limits>
#include <iomanip>
#include "stream.hpp"
#include "vectnative.hpp"

#ifdef _WIN32
#include <SDKDDKVer.h>
#include <stdio.h>
#else
// verify iostream-free status (see further down)
#if _GLIBCXX_IOSTREAM != 0
#error rogue iostream acquired
#endif
#endif

// use a custom minimalistic implementation of std::istream/ostream to avoid the highly-harmful-for-the-optimisations
// static initialization of std::cin/cout/cerr; our mini-implementation is initialized at the start of main()
namespace stream {
in cin;
out cout;
out cerr;
}

#if NATIVE_F64X4 != 0
using simd::f64x4;
using simd::as_f64x4;
using simd::get_f64x2;
#endif
#if NATIVE_S64X4 != 0
using simd::s64x4;
using simd::as_s64x4;
using simd::get_s64x2;
#endif
#if NATIVE_U64X4 != 0
using simd::u64x4;
using simd::as_u64x4;
using simd::get_u64x2;
#endif
#if NATIVE_F32X8 != 0
using simd::f32x8;
using simd::as_f32x8;
using simd::get_f32x4;
#endif
#if NATIVE_S32X8 != 0
using simd::s32x8;
using simd::as_s32x8;
using simd::get_s32x4;
#endif
#if NATIVE_U32X8 != 0
using simd::u32x8;
using simd::as_u32x8;
using simd::get_u32x4;
#endif
#if NATIVE_F32X2 != 0
using simd::f32x2;
using simd::as_f32x2;
#endif
#if NATIVE_S32X2 != 0
using simd::s32x2;
using simd::as_s32x2;
#endif
#if NATIVE_U32X2 != 0
using simd::u32x2;
using simd::as_u32x2;
#endif
#if NATIVE_S16X16 != 0
using simd::s16x16;
using simd::as_s16x16;
using simd::get_s16x8;
#endif
#if NATIVE_U16X16 != 0
using simd::u16x16;
using simd::as_u16x16;
using simd::get_u16x8;
#endif
#if NATIVE_S16X4 != 0
using simd::s16x4;
using simd::as_s16x4;
#endif
#if NATIVE_U16X4 != 0
using simd::u16x4;
using simd::as_u16x4;
#endif
using simd::f64x2;
using simd::s64x2;
using simd::u64x2;
using simd::f32x4;
using simd::s32x4;
using simd::u32x4;
using simd::s16x8;
using simd::u16x8;
using simd::as_f64x2;
using simd::as_s64x2;
using simd::as_u64x2;
using simd::as_f32x4;
using simd::as_s32x4;
using simd::as_u32x4;
using simd::as_s16x8;
using simd::as_u16x8;
using simd::all;
using simd::any;
using simd::none;
using simd::min;
using simd::max;
using simd::abs;
using simd::mask;
using simd::select;
using simd::sqrt;
using simd::log;
using simd::exp;
using simd::pow;
using simd::sin;
using simd::cos;
using simd::sincos;
using simd::transpose4x4;
using simd::operator ==;
using simd::operator !=;
using simd::operator <;
using simd::operator >;
using simd::operator <=;
using simd::operator >=;
using simd::flag_zero;
using simd::flag_native;

template < typename T >
const T& min(const T& a, const T& b) {
	return (a < b) ? a : b;
}

template < typename T >
const T& max(const T& a, const T& b) {
	return (a > b) ? a : b;
}

class formatter_f64 {
	double f;

public:
	formatter_f64(const double a) : f(a) {}

	uint64_t get() const {
		return reinterpret_cast< const uint64_t& >(f);
	}
};
{'fraction_non_alphanumeric': 0.13820865481382086, 'fraction_numerical': 0.0743039248574304, 'mean_word_length': 6.219128329297821, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1832047', 'n_tokens_mistral': 2838, 'n_tokens_neox': 2637, 'n_words': 662}