<commit_before>#ifndef K3_LIFETIME
#define K3_LIFETIME

#include <array>
#include <chrono>
#include <fstream>
#include <iostream>
#include <random>
#include <string>
#include <sstream>
#include <vector>
#include <thread>

namespace K3 {
  namespace lifetime {
    using lifetime_t = int64_t;

    const uint32_t RESERVOIR_SIZE = 1000;

    class counter {
     public:
      void push() { ++object_counter; }
      void dump() { std::cout << "Object count: " << object_counter << std::endl; }
      uint64_t object_counter;
    };

    // A reservoir sampling implementation using Algorithm L from:
    // http://dl.acm.org/citation.cfm?id=198435
    using LOSample = std::pair<lifetime_t, size_t>;

    template<size_t n = 1 << 10>
    class sampler {
    public:
      sampler(const std::string& path)
        : gen(std::random_device()()), u(), sz(0)
      {
        auto path_stream = std::ostringstream {};
        path_stream << path << "_" << std::this_thread::get_id();
        out = std::ofstream(path_stream.str());
        // Initialize w, S for use in the first instance that the reservoir is full.
        step(true);
      }

      ~sampler() {
        dump();
      }

      void step(bool initial = false) {
        w = (initial? 1 : w) * exp(log(u(gen))/static_cast<double>(n));
        S = std::floor( log(u(gen)) / log(1-w) );
      }

      void push(const lifetime_t& l, const size_t& o) {
        if ( sz < n ) { x[sz] = std::make_pair(l,o); sz++; }
        else if ( S > 0 ) { S--; }
        else {
          x[std::floor(n*u(gen))] = std::make_pair(l, o);
          step();
        }
      }

      void dump() {
        for (const auto& i : x) {
          out << i.first << "," << i.second << std::endl;
        }
      }

     protected:
      std::ofstream out;

      std::mt19937 gen;
      std::uniform_real_distribution<> u;

      uint64_t S;
      uint32_t sz;
      double w;
      std::array<LOSample, n> x;
    };

    class histogram {
     public:
      template<typename T>
      using HSpec = std::pair<T, T>; // Left as max value, right as bucket width.
      using Frequency = uint64_t;

      histogram(const std::string& path, HSpec<lifetime_t> lspec, HSpec<size_t> ospec)
        : out(path),
          lifetime_spec(lspec), objsize_spec(ospec),
          lbuckets(num_buckets(lifetime_spec)), obuckets(num_buckets(objsize_spec)),
          hdata(lbuckets * obuckets)
      {}

      ~histogram() {
        dump();
      }

      void push(const lifetime_t& l, const size_t& o) {
        hdata[bucket_index(l, o)] += 1;
      }<commit_msg>Add memory load computation to lifetime sampler.<commit_after>#ifndef K3_LIFETIME
#define K3_LIFETIME

#include <array>
#include <chrono>
#include <fstream>
#include <iostream>
#include <random>
#include <string>
#include <sstream>
#include <vector>
#include <thread>

namespace K3 {
  namespace lifetime {
    using lifetime_t = int64_t;

    const uint32_t RESERVOIR_SIZE = 1000;

    class counter {
     public:
      void push() { ++object_counter; }
      void dump() { std::cout << "Object count: " << object_counter << std::endl; }
      uint64_t object_counter;
    };

    // A reservoir sampling implementation using Algorithm L from:
    // http://dl.acm.org/citation.cfm?id=198435
    using LOSample = std::pair<lifetime_t, size_t>;

    template<size_t n = 1 << 10>
    class sampler {
    public:
      sampler(const std::string& path)
        : gen(std::random_device()()), u(), sz(0)
      {
        auto path_stream = std::ostringstream {};
        path_stream << path << "_" << std::this_thread::get_id();
        out = std::ofstream(path_stream.str());
        // Initialize w, S for use in the first instance that the reservoir is full.
        step(true);
      }

      ~sampler() {
        dump();
      }

      void step(bool initial = false) {
        w = (initial? 1 : w) * exp(log(u(gen))/static_cast<double>(n));
        S = std::floor( log(u(gen)) / log(1-w) );
      }

      void push(const lifetime_t& l, const size_t& o) {
        memory_load += l * o;
        if ( sz < n ) { x[sz] = std::make_pair(l,o); sz++; }
        else if ( S > 0 ) { S--; }
        else {
          x[std::floor(n*u(gen))] = std::make_pair(l, o);
          step();
        }
      }

      void dump() {
        out << -1 << "," << memory_load << std::endl;
        for (const auto& i : x) {
          out << i.first << "," << i.second << std::endl;
        }
      }

     protected:
      std::ofstream out;

      std::mt19937 gen;
      std::uniform_real_distribution<> u;

      uint64_t S;
      uint32_t sz;
      double w;
      std::array<LOSample, n> x;
      double memory_load = 0.0;
    };

    class histogram {
     public:
      template<typename T>
      using HSpec = std::pair<T, T>; // Left as max value, right as bucket width.
      using Frequency = uint64_t;

      histogram(const std::string& path, HSpec<lifetime_t> lspec, HSpec<size_t> ospec)
        : out(path),
          lifetime_spec(lspec), objsize_spec(ospec),
          lbuckets(num_buckets(lifetime_spec)), obuckets(num_buckets(objsize_spec)),
          hdata(lbuckets * obuckets)
      {}

      ~histogram() {
        dump();
      }

      void push(const lifetime_t& l, const size_t& o) {
        hdata[bucket_index(l, o)] += 1;
      }
{'fraction_non_alphanumeric': 0.14645610077082158, 'fraction_numerical': 0.014852415867644294, 'mean_word_length': 2.3125778331257782, 'pattern_counts': {'":': 0, '<': 93, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7653235', 'n_tokens_mistral': 1877, 'n_tokens_neox': 1778, 'n_words': 432}