<commit_before>#include "common/harness.h"

#include <vector>

#include "common/logger.h"
#include "optimizer/stats/column_stats.h"
#include "type/type.h"
#include "type/value.h"
#include "type/value_factory.h"

#define private public

namespace peloton {
namespace test {

using namespace optimizer;

class ColumnStatsTests : public PelotonTest {};

TEST_F(ColumnStatsTests, BasicTests) {
  ColumnStats colstats{0, 0, 0, type::Type::TypeId::INTEGER};
  for (int i = 0; i < 100000; i++) {
    type::Value v = type::ValueFactory::GetIntegerValue(i);
    colstats.AddValue(v);
  }
  // Minimum accuracy requirement
  uint64_t cardinality = colstats.GetCardinality();
  EXPECT_GE(cardinality, 50000);
  EXPECT_LE(cardinality, 150000);
  std::vector<double> bounds = colstats.GetHistogramBound();

  EXPECT_EQ(colstats.GetFracNull(), 0);
}

TEST_F(ColumnStatsTests, SkewedTests) {
  ColumnStats colstats{0, 0, 0, type::Type::TypeId::DATETIME};


}

} /* namespace test */
} /* namespace peloton */<commit_msg>add skewed test<commit_after>#include "common/harness.h"

#include <vector>

#include "common/logger.h"
#include "optimizer/stats/column_stats.h"
#include "type/type.h"
#include "type/value.h"
#include "type/value_factory.h"

#define private public

namespace peloton {
namespace test {

using namespace optimizer;

class ColumnStatsTests : public PelotonTest {};

TEST_F(ColumnStatsTests, BasicTests) {
  ColumnStats colstats{0, 0, 0, type::Type::TypeId::INTEGER};
  for (int i = 0; i < 100000; i++) {
    type::Value v = type::ValueFactory::GetIntegerValue(i);
    colstats.AddValue(v);
  }
  // Minimum accuracy requirement
  uint64_t cardinality = colstats.GetCardinality();
  EXPECT_GE(cardinality, 50000);
  EXPECT_LE(cardinality, 150000);
  // Histogram bound
  std::vector<double> bounds = colstats.GetHistogramBound();
  // Null value fraction
  EXPECT_EQ(colstats.GetFracNull(), 0);
}

TEST_F(ColumnStatsTests, SkewedTests) {
  ColumnStats colstats{0, 0, 0, type::Type::TypeId::DECIMAL};
  for (int i = 0; i < 1000; i++) {
    type::Value v = type::ValueFactory::GetDecimalValue(4.1525);
    colstats.AddValue(v);
  }
  type::Value v1 = type::ValueFactory::GetDecimalValue(7.12);
  type::Value v2 = type::ValueFactory::GetDecimalValue(10.25);
  colstats.AddValue(v1);
  colstats.AddValue(v2);
  uint64_t cardinality = colstats.GetCardinality();
  EXPECT_GE(cardinality, 1);
  EXPECT_LE(cardinality, 5);
}

} /* namespace test */
} /* namespace peloton */
{'fraction_non_alphanumeric': 0.14580801944106925, 'fraction_numerical': 0.0319967598217902, 'mean_word_length': 5.269035532994923, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3936313', 'n_tokens_mistral': 922, 'n_tokens_neox': 859, 'n_words': 177}