<commit_before>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include <vespa/vespalib/util/clock.h>
#include <cassert>
#include <vector>
#include <atomic>

using vespalib::Clock;
using fastos::TimeStamp;

struct UpdateClock {
    virtual ~UpdateClock() {}
    virtual void update() = 0;
};

struct NSValue : public UpdateClock {
    void update() override { _value = std::chrono::steady_clock::now().time_since_epoch().count(); }
    int64_t _value;
};

struct NSVolatile : public UpdateClock {
    void update() override { _value = std::chrono::steady_clock::now().time_since_epoch().count(); }
    volatile int64_t _value;
};
struct NSAtomic : public UpdateClock {
    void update() override { _value.store(std::chrono::steady_clock::now().time_since_epoch().count()); }
    std::atomic<int64_t> _value;
};

class TestClock : public FastOS_Runnable
{
private:
    int                       _timePeriodMS;
    std::mutex                _lock;
    std::condition_variable   _cond;
    UpdateClock              &_clock;
    bool                      _stop;

    void Run(FastOS_ThreadInterface *thisThread, void *arguments) override;

public:
    TestClock(UpdateClock & clock, double timePeriod)
        : _timePeriodMS(static_cast<uint32_t>(timePeriod*1000)),
          _lock(),
          _cond(),
          _clock(clock),
          _stop(false)
    { }
    ~TestClock() {
        std::lock_guard<std::mutex> guard(_lock);
        _stop = true;
        _cond.notify_all();
    }
};

void TestClock::Run(FastOS_ThreadInterface *thread, void *)
{
    std::unique_lock<std::mutex> guard(_lock);
    while ( ! thread->GetBreakFlag() && !_stop) {
        _clock.update();
        _cond.wait_for(guard, std::chrono::milliseconds(_timePeriodMS));
    }
}

struct SamplerBase : public FastOS_Runnable {
    FastOS_ThreadInterface * _thread;
    uint64_t _samples;
};

template<typename Func>
struct Sampler : public SamplerBase {
    Sampler(Func func) :
       _func(func)
   { }
    void Run(FastOS_ThreadInterface *, void *) override {
        fastos::SteadyTimeStamp start = fastos::ClockSteady::now();
        printf("Starting at %s \n", fastos::ClockSystem::now().toString().c_str());
        uint64_t samples;
        uint64_t countFalse(0);
        for (samples = 0; (samples < _samples); samples++) {
            if ( ! _func(samples)) countFalse++;
        }
        printf("Took %ld clock samples in %2.3f with %ld keeping up\n", samples, (fastos::ClockSteady::now() - start).sec(), countFalse);
    }
    Func _func;
};

template<typename Func>
void benchmark(FastOS_ThreadPool & pool, uint64_t samples, int numThreads, Func func) {
    std::vector<std::unique_ptr<SamplerBase>> threads;
    threads.reserve(numThreads);
    for (int i(0); i < numThreads; i++) {
        SamplerBase * sampler = new Sampler(func);
        sampler->_samples = samples;
        sampler->_thread = pool.NewThread(sampler, nullptr);
        threads.emplace_back(sampler);
    }
    for (const auto & sampler : threads) {
        sampler->_thread->Join();
    }
}

int
main(int , char *argv[])
{
    long frequency = atoll(argv[1]);
    int numThreads = atoi(argv[2]);
    uint64_t samples = atoll(argv[3]);
    FastOS_ThreadPool pool(0x10000);
    NSValue nsValue;
    NSVolatile nsVolatile;
    NSAtomic nsAtomic;
    Clock clock(1.0/frequency);
    TestClock nsClock(nsValue, 1.0/frequency);
    TestClock nsVolatileClock(nsVolatile, 1.0/frequency);
    TestClock nsAtomicClock(nsAtomic, 1.0/frequency);
    assert(pool.NewThread(&clock, nullptr) != nullptr);
    assert(pool.NewThread(&nsClock, nullptr) != nullptr);
    assert(pool.NewThread(&nsVolatileClock, nullptr) != nullptr);
    assert(pool.NewThread(&nsAtomicClock, nullptr) != nullptr);
    fastos::SteadyTimeStamp now = clock.getTimeNSAssumeRunning();
    FastOS_Thread::Sleep(100);

    benchmark(pool, samples, numThreads, [&clock, &now](int64_t i){ return (now+i < clock.getTimeNSAssumeRunning());});
    now = clock.getTimeNSAssumeRunning();
    benchmark(pool, samples, numThreads, [&nsValue, &now](int64_t i){ return (now+i < fastos::SteadyTimeStamp(nsValue._value));});
    now = clock.getTimeNSAssumeRunning();
    benchmark(pool, samples, numThreads, [&nsVolatile, &now](int64_t i){ return (now+i < fastos::SteadyTimeStamp(nsVolatile._value));});
    now = clock.getTimeNSAssumeRunning();
    benchmark(pool, samples, numThreads, [&nsAtomic, &now](int64_t i){ return (now+i < fastos::SteadyTimeStamp(nsAtomic._value.load(std::memory_order_relaxed)));});
    now = clock.getTimeNSAssumeRunning();
    benchmark(pool, samples, numThreads, [&nsAtomic, &now](int64_t i){ return (now+i < fastos::SteadyTimeStamp(nsAtomic._value.load(std::memory_order_consume)));});
    now = clock.getTimeNSAssumeRunning();
    benchmark(pool, samples, numThreads, [&nsAtomic, &now](int64_t i){ return (now+i < fastos::SteadyTimeStamp(nsAtomic._value.load(std::memory_order_acquire)));});
    now = clock.getTimeNSAssumeRunning();
    benchmark(pool, samples, numThreads, [&nsAtomic, &now](int64_t i){ return (now+i < fastos::SteadyTimeStamp(nsAtomic._value.load(std::memory_order_seq_cst)));});

    benchmark(pool, samples, numThreads, [&now](uint64_t i){ return (now+i < fastos::ClockSteady::now());});

    pool.Close();
    clock.stop();
    return 0;
}<commit_msg>Add sanity checking of clock<commit_after>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include <vespa/vespalib/util/clock.h>
#include <cassert>
#include <vector>
#include <atomic>
#include <cstring>

using vespalib::Clock;
using fastos::TimeStamp;

struct UpdateClock {
    virtual ~UpdateClock() {}
    virtual void update() = 0;
};

struct NSValue : public UpdateClock {
    void update() override { _value = std::chrono::steady_clock::now().time_since_epoch().count(); }
    int64_t _value;
};

struct NSVolatile : public UpdateClock {
    void update() override { _value = std::chrono::steady_clock::now().time_since_epoch().count(); }
    volatile int64_t _value;
};
struct NSAtomic : public UpdateClock {
    void update() override { _value.store(std::chrono::steady_clock::now().time_since_epoch().count()); }
    std::atomic<int64_t> _value;
};

class TestClock : public FastOS_Runnable
{
private:
    int                       _timePeriodMS;
    std::mutex                _lock;
    std::condition_variable   _cond;
    UpdateClock              &_clock;
    bool                      _stop;

    void Run(FastOS_ThreadInterface *thisThread, void *arguments) override;

public:
    TestClock(UpdateClock & clock, double timePeriod)
        : _timePeriodMS(static_cast<uint32_t>(timePeriod*1000)),
          _lock(),
          _cond(),
          _clock(clock),
          _stop(false)
    { }
    ~TestClock() {
        std::lock_guard<std::mutex> guard(_lock);
        _stop = true;
        _cond.notify_all();
    }
};

void TestClock::Run(FastOS_ThreadInterface *thread, void *)
{
    std::unique_lock<std::mutex> guard(_lock);
    while ( ! thread->GetBreakFlag() && !_stop) {
        _clock.update();
        _cond.wait_for(guard, std::chrono::milliseconds(_timePeriodMS));
    }
}

struct SamplerBase : public FastOS_Runnable {
    FastOS_ThreadInterface * _thread;
    uint64_t _samples;
};

template<typename Func>
struct Sampler : public SamplerBase {
    Sampler(Func func) :
       _func(func)
    { }
    void Run(FastOS_ThreadInterface *, void *) override {
        fastos::SteadyTimeStamp start = fastos::ClockSteady::now();
        printf("Starting at %s \n", fastos::ClockSystem::now().toString().c_str());
        uint64_t samples;
        uint64_t count[3];
        memset(count, 0, sizeof(count));
        for (samples = 0; (samples < _samples); samples++) {
            count[1 + _func()]++;
        }
        printf("Took %ld clock samples in %2.3f with [%ld, %ld, %ld] counts\n", samples, (fastos::ClockSteady::now() - start).sec(), count[0], count[1], count[2]);

    }
    Func _func;
};

template<typename Func>
void benchmark(FastOS_ThreadPool & pool, uint64_t samples, int numThreads, Func func) {
    std::vector<std::unique_ptr<SamplerBase>> threads;
    threads.reserve(numThreads);
    for (int i(0); i < numThreads; i++) {
        SamplerBase * sampler = new Sampler(func);
        sampler->_samples = samples;
        sampler->_thread = pool.NewThread(sampler, nullptr);
        threads.emplace_back(sampler);
    }
    for (const auto & sampler : threads) {
        sampler->_thread->Join();
    }
}

int
main(int , char *argv[])
{
    long frequency = atoll(argv[1]);
    int numThreads = atoi(argv[2]);
    uint64_t samples = atoll(argv[3]);
    FastOS_ThreadPool pool(0x10000);
    NSValue nsValue;
    NSVolatile nsVolatile;
    NSAtomic nsAtomic;
    Clock clock(1.0/frequency);
    TestClock nsClock(nsValue, 1.0/frequency);
    TestClock nsVolatileClock(nsVolatile, 1.0/frequency);
    TestClock nsAtomicClock(nsAtomic, 1.0/frequency);
    assert(pool.NewThread(&clock, nullptr) != nullptr);
    assert(pool.NewThread(&nsClock, nullptr) != nullptr);
    assert(pool.NewThread(&nsVolatileClock, nullptr) != nullptr);
    assert(pool.NewThread(&nsAtomicClock, nullptr) != nullptr);

    fastos::SteadyTimeStamp prev;
    prev = clock.getTimeNSAssumeRunning();
    benchmark(pool, samples, numThreads, [&clock, &prev]() {
        fastos::SteadyTimeStamp now = clock.getTimeNSAssumeRunning();
        auto diff = now - prev;
        if (diff > 0) prev = now;
        return (diff == 0) ? 0 : (diff > 0) ? 1 : -1;
    });
    prev = fastos::SteadyTimeStamp(nsValue._value);
    benchmark(pool, samples, numThreads, [&nsValue, &prev]() {
        fastos::SteadyTimeStamp now = fastos::SteadyTimeStamp(nsValue._value) ;
        auto diff = now - prev;
        if (diff > 0) prev = now;
        return (diff == 0) ? 0 : (diff > 0) ? 1 : -1;
    });
    prev = fastos::SteadyTimeStamp(nsVolatile._value);
    benchmark(pool, samples, numThreads, [&nsVolatile, &prev]() {
        fastos::SteadyTimeStamp now = fastos::SteadyTimeStamp(nsVolatile._value) ;
        auto diff = now - prev;
        if (diff > 0) prev = now;
        return (diff == 0) ? 0 : (diff > 0) ? 1 : -1;
    });
    prev = fastos::SteadyTimeStamp(nsAtomic._value.load());
    benchmark(pool, samples, numThreads, [&nsAtomic, &prev]() {
        fastos::SteadyTimeStamp now = fastos::SteadyTimeStamp(nsAtomic._value.load(std::memory_order_relaxed)) ;
        auto diff = now - prev;
        if (diff > 0) prev = now;
        return (diff == 0) ? 0 : (diff > 0) ? 1 : -1;
    });
    prev = fastos::SteadyTimeStamp(nsAtomic._value.load());
    benchmark(pool, samples, numThreads, [&nsAtomic, &prev]() {
        fastos::SteadyTimeStamp now = fastos::SteadyTimeStamp(nsAtomic._value.load(std::memory_order_consume)) ;
        auto diff = now - prev;
        if (diff > 0) prev = now;
        return (diff == 0) ? 0 : (diff > 0) ? 1 : -1;
    });
    prev = fastos::SteadyTimeStamp(nsAtomic._value.load());
    benchmark(pool, samples, numThreads, [&nsAtomic, &prev]() {
        fastos::SteadyTimeStamp now = fastos::SteadyTimeStamp(nsAtomic._value.load(std::memory_order_acquire)) ;
        auto diff = now - prev;
        if (diff > 0) prev = now;
        return (diff == 0) ? 0 : (diff > 0) ? 1 : -1;
    });
    prev = fastos::SteadyTimeStamp(nsAtomic._value.load());
    benchmark(pool, samples, numThreads, [&nsAtomic, &prev]() {
        fastos::SteadyTimeStamp now = fastos::SteadyTimeStamp(nsAtomic._value.load(std::memory_order_seq_cst)) ;
        auto diff = now - prev;
        if (diff > 0) prev = now;
        return (diff == 0) ? 0 : (diff > 0) ? 1 : -1;
    });

    benchmark(pool, samples, numThreads, [&prev]() {
        fastos::SteadyTimeStamp now = fastos::ClockSteady::now();
        auto diff = now - prev;
        if (diff > 0) prev = now;
        return (diff == 0) ? 0 : (diff > 0) ? 1 : -1;
    });

    pool.Close();
    clock.stop();
    return 0;
}
{'fraction_non_alphanumeric': 0.1457415870378064, 'fraction_numerical': 0.01462401329455754, 'mean_word_length': 3.426627436557558, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6715312', 'n_tokens_mistral': 4109, 'n_tokens_neox': 3972, 'n_words': 841}