<commit_before>//===----------------------------------------------------------------------===//
//
//                         Peloton
//
// stats_test.cpp
//
// Identification: test/statistics/stats_test.cpp
//
// Copyright (c) 2015-16, Carnegie Mellon University Database Group
//
//===----------------------------------------------------------------------===//

#include <iostream>

#include "common/harness.h"
#include "common/value.h"
#include "common/config.h"
#include "common/value_factory.h"

#include "catalog/catalog.h"
#include "concurrency/transaction_manager_factory.h"
#include "concurrency/transaction_tests_util.h"
#include "statistics/stats_aggregator.h"
#include "statistics/backend_stats_context.h"
#include "statistics/stats_tests_util.h"
#include "storage/data_table.h"
#include "storage/tuple.h"
#include "storage/tile.h"
#include "executor/executor_tests_util.h"
#include "executor/insert_executor.h"
#include "executor/executor_context.h"
#include "planner/insert_plan.h"

namespace peloton {
namespace test {

class StatsTest : public PelotonTest {};

void TransactionTest(storage::Database *database, storage::DataTable *table,
                     UNUSED_ATTRIBUTE uint64_t thread_itr) {
  uint64_t thread_id = TestingHarness::GetInstance().GetThreadId();
  auto tile_group_id = table->GetTileGroup(0)->GetTileGroupId();
  auto index_metadata = table->GetIndex(0)->GetMetadata();
  auto context = stats::BackendStatsContext::GetInstance();

  for (oid_t txn_itr = 1; txn_itr <= 50; txn_itr++) {

    if (thread_id % 2 == 0) {
      std::chrono::microseconds sleep_time(1);
      std::this_thread::sleep_for(sleep_time);
    }
    // Record database stat
    context->IncrementTxnCommitted(database->GetOid());
    context->IncrementTxnAborted(database->GetOid());
    context->IncrementTxnAborted(database->GetOid());
    // Record table stat
    context->IncrementTableReads(tile_group_id);
    context->IncrementTableReads(tile_group_id);
    context->IncrementTableUpdates(tile_group_id);
    context->IncrementTableDeletes(tile_group_id);
    context->IncrementTableDeletes(tile_group_id);
    context->IncrementTableInserts(tile_group_id);
    // Record index stat
    context->IncrementIndexReads(1, index_metadata);
    context->IncrementIndexDeletes(2, index_metadata);
    context->IncrementIndexInserts(index_metadata);
  }
}

TEST_F(StatsTest, MultiThreadStatsTest) {

  FLAGS_stats_mode = STATS_TYPE_ENABLE;
  auto &aggregator = peloton::stats::StatsAggregator::GetInstance(100);

  /*
   * Register to StatsAggregator
   */
  peloton::stats::StatsAggregator::GetInstance(100);

  // Create dummy database, table and index
  auto catalog = catalog::Catalog::GetInstance();
  auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance();
  auto txn = txn_manager.BeginTransaction();
  auto id_column = catalog::Column(common::Type::INTEGER,
                                   common::Type::GetTypeSize(common::Type::INTEGER), "id", true);
  auto name_column = catalog::Column(common::Type::VARCHAR, 32, "name", true);

  std::unique_ptr<catalog::Schema> table_schema(
      new catalog::Schema({id_column, name_column}));

  catalog->CreateDatabase("EMP_DB", txn);
  catalog::Catalog::GetInstance()->CreateTable("EMP_DB", "emp_table",
                                               std::move(table_schema), txn);
  txn_manager.CommitTransaction(txn);

  // Create multiple stat worker threads
  storage::Database *database = catalog->GetDatabaseWithName("EMP_DB");
  storage::DataTable *table = database->GetTableWithName("emp_table");
  LaunchParallelTest(8, TransactionTest, database, table);

  // Wait for aggregation to finish
  std::chrono::microseconds sleep_time(200 * 1000);
  std::this_thread::sleep_for(sleep_time);

  // Check database metrics
  auto &aggregated_stats =
      peloton::stats::StatsAggregator::GetInstance().GetAggregatedStats();
  auto db_oid = database->GetOid();
  auto db_metric = aggregated_stats.GetDatabaseMetric(db_oid);
  ASSERT_EQ(db_metric->GetTxnCommitted().GetCounter(), 8 * 50);
  ASSERT_EQ(db_metric->GetTxnAborted().GetCounter(), 8 * 50 * 2);

  // Check table metrics
  auto table_oid = table->GetOid();
  auto table_metric = aggregated_stats.GetTableMetric(db_oid, table_oid);
  auto table_access = table_metric->GetTableAccess();
  ASSERT_EQ(table_access.GetReads(), 8 * 50 * 2);
  ASSERT_EQ(table_access.GetUpdates(), 8 * 50 * 1);
  ASSERT_EQ(table_access.GetDeletes(), 8 * 50 * 2);
  ASSERT_EQ(table_access.GetInserts(), 8 * 50 * 1);

  // Check index metrics
  auto index = table->GetIndex(0);
  auto index_oid = index->GetOid();
  auto index_metric =
      aggregated_stats.GetIndexMetric(db_oid, table_oid, index_oid);
  auto index_access = index_metric->GetIndexAccess();
  ASSERT_EQ(index_access.GetReads(), 8 * 50 * 1);
  ASSERT_EQ(index_access.GetDeletes(), 8 * 50 * 2);
  ASSERT_EQ(index_access.GetInserts(), 8 * 50 * 1);

  aggregator.ShutdownAggregator();
  txn = txn_manager.BeginTransaction();
  catalog->DropDatabaseWithName("EMP_DB", txn);
  txn_manager.CommitTransaction(txn);
}

TEST_F(StatsTest, PerThreadStatsTest) {<commit_msg>[stat] replace magic number with const variable<commit_after>//===----------------------------------------------------------------------===//
//
//                         Peloton
//
// stats_test.cpp
//
// Identification: test/statistics/stats_test.cpp
//
// Copyright (c) 2015-16, Carnegie Mellon University Database Group
//
//===----------------------------------------------------------------------===//

#include <iostream>

#include "common/harness.h"
#include "common/value.h"
#include "common/config.h"
#include "common/value_factory.h"

#include "catalog/catalog.h"
#include "concurrency/transaction_manager_factory.h"
#include "concurrency/transaction_tests_util.h"
#include "statistics/stats_aggregator.h"
#include "statistics/backend_stats_context.h"
#include "statistics/stats_tests_util.h"
#include "storage/data_table.h"
#include "storage/tuple.h"
#include "storage/tile.h"
#include "executor/executor_tests_util.h"
#include "executor/insert_executor.h"
#include "executor/executor_context.h"
#include "planner/insert_plan.h"

#define NUM_ITERATION 50

namespace peloton {
namespace test {

class StatsTest : public PelotonTest {};

void TransactionTest(storage::Database *database, storage::DataTable *table,
                     UNUSED_ATTRIBUTE uint64_t thread_itr) {
  uint64_t thread_id = TestingHarness::GetInstance().GetThreadId();
  auto tile_group_id = table->GetTileGroup(0)->GetTileGroupId();
  auto index_metadata = table->GetIndex(0)->GetMetadata();
  auto context = stats::BackendStatsContext::GetInstance();

  for (oid_t txn_itr = 1; txn_itr <= NUM_ITERATION; txn_itr++) {

    if (thread_id % 2 == 0) {
      std::chrono::microseconds sleep_time(1);
      std::this_thread::sleep_for(sleep_time);
    }
    // Record database stat
    context->IncrementTxnCommitted(database->GetOid());
    context->IncrementTxnAborted(database->GetOid());
    context->IncrementTxnAborted(database->GetOid());
    // Record table stat
    context->IncrementTableReads(tile_group_id);
    context->IncrementTableReads(tile_group_id);
    context->IncrementTableUpdates(tile_group_id);
    context->IncrementTableDeletes(tile_group_id);
    context->IncrementTableDeletes(tile_group_id);
    context->IncrementTableInserts(tile_group_id);
    // Record index stat
    context->IncrementIndexReads(1, index_metadata);
    context->IncrementIndexDeletes(2, index_metadata);
    context->IncrementIndexInserts(index_metadata);
  }
}

TEST_F(StatsTest, MultiThreadStatsTest) {

  FLAGS_stats_mode = STATS_TYPE_ENABLE;
  auto &aggregator = peloton::stats::StatsAggregator::GetInstance(100);

  /*
   * Register to StatsAggregator
   */
  peloton::stats::StatsAggregator::GetInstance(100);

  // Create dummy database, table and index
  auto catalog = catalog::Catalog::GetInstance();
  auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance();
  auto txn = txn_manager.BeginTransaction();
  auto id_column = catalog::Column(
      common::Type::INTEGER, common::Type::GetTypeSize(common::Type::INTEGER),
      "id", true);
  auto name_column = catalog::Column(common::Type::VARCHAR, 32, "name", true);

  std::unique_ptr<catalog::Schema> table_schema(
      new catalog::Schema({id_column, name_column}));

  catalog->CreateDatabase("EMP_DB", txn);
  catalog::Catalog::GetInstance()->CreateTable("EMP_DB", "emp_table",
                                               std::move(table_schema), txn);
  txn_manager.CommitTransaction(txn);

  // Create multiple stat worker threads
  int num_threads = 8;
  storage::Database *database = catalog->GetDatabaseWithName("EMP_DB");
  storage::DataTable *table = database->GetTableWithName("emp_table");
  LaunchParallelTest(num_threads, TransactionTest, database, table);

  // Wait for aggregation to finish
  std::chrono::microseconds sleep_time(200 * 1000);
  std::this_thread::sleep_for(sleep_time);

  // Check database metrics
  auto &aggregated_stats =
      peloton::stats::StatsAggregator::GetInstance().GetAggregatedStats();
  auto db_oid = database->GetOid();
  auto db_metric = aggregated_stats.GetDatabaseMetric(db_oid);
  ASSERT_EQ(db_metric->GetTxnCommitted().GetCounter(),
            num_threads * NUM_ITERATION);
  ASSERT_EQ(db_metric->GetTxnAborted().GetCounter(),
            num_threads * NUM_ITERATION * 2);

  // Check table metrics
  auto table_oid = table->GetOid();
  auto table_metric = aggregated_stats.GetTableMetric(db_oid, table_oid);
  auto table_access = table_metric->GetTableAccess();
  ASSERT_EQ(table_access.GetReads(), num_threads * NUM_ITERATION * 2);
  ASSERT_EQ(table_access.GetUpdates(), num_threads * NUM_ITERATION);
  ASSERT_EQ(table_access.GetDeletes(), num_threads * NUM_ITERATION * 2);
  ASSERT_EQ(table_access.GetInserts(), num_threads * NUM_ITERATION);

  // Check index metrics
  auto index = table->GetIndex(0);
  auto index_oid = index->GetOid();
  auto index_metric =
      aggregated_stats.GetIndexMetric(db_oid, table_oid, index_oid);
  auto index_access = index_metric->GetIndexAccess();
  ASSERT_EQ(index_access.GetReads(), num_threads * NUM_ITERATION);
  ASSERT_EQ(index_access.GetDeletes(), num_threads * NUM_ITERATION * 2);
  ASSERT_EQ(index_access.GetInserts(), num_threads * NUM_ITERATION);

  aggregator.ShutdownAggregator();
  txn = txn_manager.BeginTransaction();
  catalog->DropDatabaseWithName("EMP_DB", txn);
  txn_manager.CommitTransaction(txn);
}

TEST_F(StatsTest, PerThreadStatsTest) {
{'fraction_non_alphanumeric': 0.15637120423800965, 'fraction_numerical': 0.010689622552265632, 'mean_word_length': 4.325944584382872, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2622418', 'n_tokens_mistral': 3392, 'n_tokens_neox': 3243, 'n_words': 555}