<commit_before>// This file contains the tools used to compute the various coefficients needed
// for sampling estimates, namely the y_s, c_s, and c_st coefficients. For more
// information, see http://www.vldb.org/pvldb/vol6/p1798-nirkhiwale.pdf.

// Resources:
// armadillo: matrix and vector containers.


// The bitwise-SWAR algorithm for computing the Hamming Weight of an integer.
// The return type is a signed integer but the actual result is an unsigned
// integer. This causes an implicit static cast, which is safe as the result is
// at most 32 and necessary because the expressions used to compute the various
// coefficients would otherwise be erroneously promoted to unsigned integers
// when they need to support negative values.
inline int CountBits(uint32_t i) {
   i = i - ((i >> 1) & 0x55555555);
   i = (i & 0x33333333) + ((i >> 2) & 0x33333333);
   return (((i + (i >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24;
}

// This function computes the value of c_s given s as a bit mask. It should be
// noted that when t is a subset of s, |t| + |s| = |s \ t| % 2 because whichever
// elements that appear in t will appear in s as well. This means that such an
// element will be counted twice on the left hand side and therefore not affect
// the parity of the result. Furthermore, the VLDb paper has a typo in it and t
// should in fact only range across the subsets of s and not the power set of n.
inline double ComputeCCoefficient(uint32_t s, const arma::vec& b) {
  double coef = 0;
  for (uint32_t t = 0; t <= s; t++)
    if (t & ~s)  // t is not a subset of s.
      continue;
    else
      coef += (1 - 2 * (CountBits(s - t) % 2)) * b[t];
  return coef;
}

// This function computes the value of c_s,t given s and t as bit masks. Note
// that the computation differs slightly from that described in the VLDb paper,
// which has a typo in the exponent. The power of -1 should be |t \ u| and not
// |u| + |s|, the former of which can be simplified to |t - u| because u is
// taken to be a subset of t.
inline double ComputeCCoefficient(uint32_t s, uint32_t t, const arma::vec& b) {
  double coef = 0;
  for (uint32_t u = 0; u <= t; u++)  // A bit mask representing the set u.
    if (u & ~t)  // u is not a subset of t.
      continue;
    else<commit_msg>Fixed VLDB typo, added manual include of armadillo for completeness<commit_after>// This file contains the tools used to compute the various coefficients needed
// for sampling estimates, namely the y_s, c_s, and c_st coefficients. For more
// information, see http://www.vldb.org/pvldb/vol6/p1798-nirkhiwale.pdf.

// Resources:
// armadillo: matrix and vector containers.
using <armadillo>

// The bitwise-SWAR algorithm for computing the Hamming Weight of an integer.
// The return type is a signed integer but the actual result is an unsigned
// integer. This causes an implicit static cast, which is safe as the result is
// at most 32 and necessary because the expressions used to compute the various
// coefficients would otherwise be erroneously promoted to unsigned integers
// when they need to support negative values.
inline int CountBits(uint32_t i) {
   i = i - ((i >> 1) & 0x55555555);
   i = (i & 0x33333333) + ((i >> 2) & 0x33333333);
   return (((i + (i >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24;
}

// This function computes the value of c_s given s as a bit mask. It should be
// noted that when t is a subset of s, |t| + |s| = |s \ t| % 2 because whichever
// elements that appear in t will appear in s as well. This means that such an
// element will be counted twice on the left hand side and therefore not affect
// the parity of the result. Furthermore, the VLDb paper has a typo in it and t
// should in fact only range across the subsets of s and not the power set of n.
inline double ComputeCCoefficient(uint32_t s, const arma::vec& b) {
  double coef = 0;
  for (uint32_t t = 0; t <= s; t++)
    if (t & ~s)  // t is not a subset of s.
      continue;
    else
      coef += (1 - 2 * (CountBits(s - t) % 2)) * b[t];
  return coef;
}

// This function computes the value of c_s,t given s and t as bit masks. Note
// that the computation differs slightly from that described in the VLDB paper,
// which has a typo in the exponent. The power of -1 should be |t \ u| and not
// |u| + |s|, the former of which can be simplified to |t - u| because u is
// taken to be a subset of t.
inline double ComputeCCoefficient(uint32_t s, uint32_t t, const arma::vec& b) {
  double coef = 0;
  for (uint32_t u = 0; u <= t; u++)  // A bit mask representing the set u.
    if (u & ~t)  // u is not a subset of t.
      continue;
    else
{'fraction_non_alphanumeric': 0.09673230902402077, 'fraction_numerical': 0.03202769963211426, 'mean_word_length': 3.029642545771578, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 6, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7625832', 'n_tokens_mistral': 1563, 'n_tokens_neox': 1426, 'n_words': 732}