<commit_before>#include <chrono>
#include <cmath>
#include <iostream>
#include <libdariadb/ads/fixed_tree.h>
#include <libdariadb/meas.h>
#include <libdariadb/timeutil.h>
#include <libdariadb/utils/exception.h>
#include <map>
#include <numeric>
#include <thread>
#include <vector>

template <class T> struct ByteKeySplitter {
	static const size_t levels_count = sizeof(T);
	typedef std::array<size_t, levels_count> splited_key;
	size_t level_size(size_t level_num) const {
		auto res = std::pow(2, sizeof(uint8_t) * 8);
		return res;
	}

	splited_key split(const T &k) const {
		splited_key result;
		auto in_bts = reinterpret_cast<const uint8_t *>(&k);
		for (size_t i = 0; i < levels_count; ++i) {
			result[levels_count - i - 1] = in_bts[i];
		}
		return result;
	}
};

struct KeySplitter {
  static const size_t levels_count = size_t(4) + sizeof(int64_t) / 2;
  typedef std::array<size_t, levels_count> splited_key;
  size_t level_size(size_t level_num) const {
    switch (level_num) {
    case 0: // year
      return 2500;
    case 1: // day of year
      return 366;
    case 2: // hour
      return 24;
    case 3: // minute
      return 60;
    case 4: // second
      return 60;
    default: // fraces
      return 65536;
    }
  }

  splited_key split(const dariadb::Time &k) const {
    auto dt = dariadb::timeutil::to_datetime(k);

    splited_key result;
    result[0] = dt.year;
    result[1] = dt.day_of_year;
    result[2] = dt.hour;
    result[3] = dt.minute;
    result[4] = dt.second;
    auto in_bts = reinterpret_cast<const uint16_t *>(&dt.fracsec);
    for (size_t i = 0; i < sizeof(dt.fracsec) / 2; ++i) {
      result[levels_count - i - 1] = in_bts[i];
    }

    return result;
  }
};

template <typename T> struct Statistic {
  void append(const T &t) {}
};<commit_msg>split step.<commit_after>#include <chrono>
#include <cmath>
#include <iostream>
#include <libdariadb/ads/fixed_tree.h>
#include <libdariadb/meas.h>
#include <libdariadb/timeutil.h>
#include <libdariadb/utils/exception.h>
#include <map>
#include <numeric>
#include <thread>
#include <vector>

template <class T> struct ByteKeySplitter {
	static const size_t levels_count = sizeof(T);
	typedef std::array<size_t, levels_count> splited_key;
	size_t level_size(size_t level_num) const {
		auto res = std::pow(2, sizeof(uint8_t) * 8);
		return res;
	}

	splited_key split(const T &k) const {
		splited_key result;
		auto in_bts = reinterpret_cast<const uint8_t *>(&k);
		for (size_t i = 0; i < levels_count; ++i) {
			result[levels_count - i - 1] = in_bts[i];
		}
		return result;
	}
};

struct KeySplitter {
  static const size_t levels_count = size_t(5) + sizeof(int64_t);
  typedef std::array<size_t, levels_count> splited_key;
  size_t level_size(size_t level_num) const {
    switch (level_num) {
    case 0: // year
      return 2500;
    case 1: // day of year
      return 366;
    case 2: // hour
      return 24;
    case 3: // minute
      return 60;
    case 4: // second
      return 60;
    default: // fraces
      return 256;
    }
  }

  splited_key split(const dariadb::Time &k) const {
    auto dt = dariadb::timeutil::to_datetime(k);

    splited_key result;
    result[0] = dt.year;
    result[1] = dt.day_of_year;
    result[2] = dt.hour;
    result[3] = dt.minute;
    result[4] = dt.second;
    auto in_bts = reinterpret_cast<const uint8_t *>(&dt.fracsec);
    for (size_t i = 0; i < sizeof(dt.fracsec); ++i) {
      result[levels_count - i - 1] = in_bts[i];
    }

    return result;
  }
};

template <typename T> struct Statistic {
  void append(const T &t) {}
};
{'fraction_non_alphanumeric': 0.13432002234012846, 'fraction_numerical': 0.022619380061435355, 'mean_word_length': 3.300120048019208, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4805061', 'n_tokens_mistral': 1467, 'n_tokens_neox': 1372, 'n_words': 339}