<commit_before>#include "upload_riak.h"

#include <fcntl.h>
#include <sys/stat.h>
#include <unistd.h>

#include <algorithm>

#include "duplex_curl.h"
#include "logging.h"

using namespace upload;

RiakPushWorker::Context* RiakPushWorker::GenerateContext(
                            SpoolerBackend<RiakPushWorker> *master,
                            const std::string              &upstream_urls) {
  std::vector<std::string> upstream_url_vector = SplitString(upstream_urls, '@');
  return new Context(master, upstream_url_vector);
}


int RiakPushWorker::GetNumberOfWorkers(const Context *context) {
  return context->upstream_urls.size() * GetNumberOfCpuCores();
}


const std::string& RiakPushWorker::Context::AcquireUpstreamUrl() const {
  assert (upstream_urls.size() > 0);
  // get an unique upstream URL and set the pointer to the next one...
  const std::string& result = upstream_urls[next_upstream_url_];
  next_upstream_url_ = (next_upstream_url_ + 1) % upstream_urls.size();
  return result;
}
<commit_msg>determine the number of used worker threads for riak<commit_after>#include "upload_riak.h"

#include <fcntl.h>
#include <sys/stat.h>
#include <unistd.h>

#include <algorithm>

#include "duplex_curl.h"
#include "logging.h"

using namespace upload;

RiakPushWorker::Context* RiakPushWorker::GenerateContext(
                            SpoolerBackend<RiakPushWorker> *master,
                            const std::string              &upstream_urls) {
  std::vector<std::string> upstream_url_vector = SplitString(upstream_urls, '@');
  return new Context(master, upstream_url_vector);
}


int RiakPushWorker::GetNumberOfWorkers(const Context *context) {
  return std::max((int)context->upstream_urls.size(), GetNumberOfCpuCores());
}


const std::string& RiakPushWorker::Context::AcquireUpstreamUrl() const {
  assert (upstream_urls.size() > 0);
  // get an unique upstream URL and set the pointer to the next one...
  const std::string& result = upstream_urls[next_upstream_url_];
  next_upstream_url_ = (next_upstream_url_ + 1) % upstream_urls.size();
  return result;
}

{'fraction_non_alphanumeric': 0.11956001912960305, 'fraction_numerical': 0.001912960306073649, 'mean_word_length': 4.309644670050761, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5354545', 'n_tokens_mistral': 698, 'n_tokens_neox': 683, 'n_words': 144}