<commit_before>#include <wdt/Throttler.h>
#include <wdt/test/TestCommon.h>

#include <glog/logging.h>
#include <gtest/gtest.h>
#include <chrono>

namespace facebook {
namespace wdt {

void testThrottling(std::shared_ptr<Throttler> throttler, int expectedRate) {
  int64_t numTransferred = 0;
  auto startTime = Clock::now();
  while (durationSeconds(Clock::now() - startTime) < 2) {
    throttler->limit(1000);
    numTransferred += 1000;
  }
  auto endTime = Clock::now();
  double durationSecs = durationSeconds(endTime - startTime);
  double throughputMBps = numTransferred / durationSecs / kMbToB;
  EXPECT_NEAR(throughputMBps, expectedRate, 3);
}

TEST(ThrottlerTest, RATE_CHANGE) {
  WdtOptions options;
  options.avg_mbytes_per_sec = 500;
  std::shared_ptr<Throttler> throttler = Throttler::makeThrottler(options);
  throttler->startTransfer();

  testThrottling(throttler, 500);

  // test rate decrease
  options.avg_mbytes_per_sec = 300;
  throttler->setThrottlerRates(options);
  testThrottling(throttler, 300);

  // test rate increase
  options.avg_mbytes_per_sec = 700;
  throttler->setThrottlerRates(options);
  testThrottling(throttler, 700);

  throttler->endTransfer();
}
}
}<commit_msg>Reducing throttler rate in test so that it passes with asan<commit_after>#include <wdt/Throttler.h>
#include <wdt/test/TestCommon.h>

#include <glog/logging.h>
#include <gtest/gtest.h>
#include <chrono>

namespace facebook {
namespace wdt {

void testThrottling(std::shared_ptr<Throttler> throttler, int expectedRate) {
  int64_t numTransferred = 0;
  auto startTime = Clock::now();
  while (durationSeconds(Clock::now() - startTime) <= 2) {
    throttler->limit(1000);
    numTransferred += 1000;
  }
  auto endTime = Clock::now();
  double durationSecs = durationSeconds(endTime - startTime);
  double throughputMBps = numTransferred / durationSecs / kMbToB;
  EXPECT_NEAR(throughputMBps, expectedRate, 1);
}

TEST(ThrottlerTest, RATE_CHANGE) {
  WdtOptions options;
  options.avg_mbytes_per_sec = 50;
  std::shared_ptr<Throttler> throttler = Throttler::makeThrottler(options);
  throttler->startTransfer();

  testThrottling(throttler, 50);

  // test rate decrease
  options.avg_mbytes_per_sec = 30;
  throttler->setThrottlerRates(options);
  testThrottling(throttler, 30);

  // test rate increase
  options.avg_mbytes_per_sec = 70;
  throttler->setThrottlerRates(options);
  testThrottling(throttler, 70);

  throttler->endTransfer();
}
}
}
{'fraction_non_alphanumeric': 0.12199102407180742, 'fraction_numerical': 0.02284781721746226, 'mean_word_length': 5.368831168831169, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2095001', 'n_tokens_mistral': 942, 'n_tokens_neox': 890, 'n_words': 162}