<commit_before>/*******************************************************************************
 * Common definitions for parsing sample data.
 ******************************************************************************/

#include "sys/time.h"
#include "sample_data.hpp"

//Get the time in milliseconds
uint64_t msecTime() {
  //Set this to the real timestamp, milliseconds since 1970
  timeval tval;
  gettimeofday(&tval, NULL);
  //Cast to uint64_t to prevent rounding problems on 32 bit systems
  return (uint64_t)tval.tv_sec*(uint64_t)1000 + tval.tv_usec/1000;
}

bool operator==(const uint128_t& a, const uint128_t& b) {
  return a.upper == b.upper and a.lower == b.lower;
}

std::ostream& operator<<(std::ostream& os, const uint128_t& val) {
  //The packed struct seems to have problems with the stream operators
  //so pull out the upper and lower values into temporary variables before
  //applying the << operator.
  uint64_t upper = val.upper;
  uint64_t lower = val.lower;
  //return os<<"0x"<<std::hex<<upper<<lower<<std::dec;
  //TODO FIXME This assumes that the number is actually only 64 bits long.
  return os<<lower<<std::dec;
}

//TODO FIXME This assumes that the input number is actually only 64 bits long.
std::istream& operator>>(std::istream& is, uint128_t& val) {
  //The packed nature of the structure seems to cause errors with the
  //stream operator so make a temporary variable to read data into.
  uint64_t newval;
  is >> newval;
  val.lower = newval;
  return is;
  //return is>>val.lower;
}

uint128_t operator&(const uint128_t& a, const uint128_t& b) {
  uint128_t c;
  c.upper = a.upper & b.upper;
  c.lower = a.lower & b.lower;
  return c;
}

bool operator<(const uint128_t& a, const uint128_t& b) {
  return (a.upper < b.upper or (a.upper == b.upper and a.lower < b.lower));<commit_msg>Fixing warning (formatting 128 numeric output as hex).<commit_after>/*******************************************************************************
 * Common definitions for parsing sample data.
 ******************************************************************************/

#include "sys/time.h"
#include "sample_data.hpp"

//Get the time in milliseconds
uint64_t msecTime() {
  //Set this to the real timestamp, milliseconds since 1970
  timeval tval;
  gettimeofday(&tval, NULL);
  //Cast to uint64_t to prevent rounding problems on 32 bit systems
  return (uint64_t)tval.tv_sec*(uint64_t)1000 + tval.tv_usec/1000;
}

bool operator==(const uint128_t& a, const uint128_t& b) {
  return a.upper == b.upper and a.lower == b.lower;
}

std::ostream& operator<<(std::ostream& os, const uint128_t& val) {
  //The packed struct seems to have problems with the stream operators
  //so pull out the upper and lower values into temporary variables before
  //applying the << operator.
  uint64_t upper = val.upper;
  uint64_t lower = val.lower;
  //Save current format flags
  std::ios_base::fmtflags flags = os.flags();
  return os<<std::hex<<std::showbase<<upper<<lower<<std::dec;
  //Reset format flags
  os.flags(flags);
  //TODO FIXME This assumes that the number is actually only 64 bits long.
  //return os<<lower<<std::dec;
}

//TODO FIXME This assumes that the input number is actually only 64 bits long.
std::istream& operator>>(std::istream& is, uint128_t& val) {
  //The packed nature of the structure seems to cause errors with the
  //stream operator so make a temporary variable to read data into.
  uint64_t newval;
  is >> newval;
  val.lower = newval;
  return is;
  //return is>>val.lower;
}

uint128_t operator&(const uint128_t& a, const uint128_t& b) {
  uint128_t c;
  c.upper = a.upper & b.upper;
  c.lower = a.lower & b.lower;
  return c;
}

bool operator<(const uint128_t& a, const uint128_t& b) {
  return (a.upper < b.upper or (a.upper == b.upper and a.lower < b.lower));
{'fraction_non_alphanumeric': 0.19172341540073337, 'fraction_numerical': 0.03352540597171294, 'mean_word_length': 4.296809986130374, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '309961', 'n_tokens_mistral': 1295, 'n_tokens_neox': 1186, 'n_words': 413}