text:
/* <bruce/discard_file_logger.cc>

   ----------------------------------------------------------------------------
   Copyright 2013-2014 if(we)

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
   ----------------------------------------------------------------------------

   Implements <bruce/discard_file_logger.h>.
 */

#include <bruce/discard_file_logger.h>

#include <algorithm>
#include <array>
#include <cctype>
#include <cerrno>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <limits>
#include <sstream>
#include <stdexcept>

#include <boost/lexical_cast.hpp>
#include <fcntl.h>
#include <poll.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <syslog.h>
#include <unistd.h>

#include <base/dir_iter.h>
#include <base/error_utils.h>
#include <base/gettid.h>
#include <base/no_default_case.h>
#include <bruce/util/msg_util.h>
#include <bruce/util/time_util.h>
#include <third_party/base64/base64.h>

using namespace Base;
using namespace Bruce;
using namespace Bruce::Util;

static std::string ComposeLogEntry(TMsg::TTimestamp timestamp,
    const char *event, const char *info, const std::string &topic,
    const void *key, size_t key_size, const void *msg, size_t msg_size) {
  assert(event);
  assert(info);
  assert(key || (key_size == 0));
  assert(msg || (msg_size == 0));
  uint64_t now = GetEpochMilliseconds();
  std::string encoded_key;

  if (key_size) {
    encoded_key = base64_encode(reinterpret_cast<const unsigned char *>(key),
                                key_size);
  }

  std::string encoded_msg;

  if (msg_size) {
    encoded_msg = base64_encode(reinterpret_cast<const unsigned char *>(msg),
                                msg_size);
  }

  std::ostringstream os;
  os << "now: " << now << " ts: " << timestamp << " event: " << event
      << " info: " << info << " topic: " << topic.size() << "[" << topic
      << "] key: " << encoded_key.size() << "[" << encoded_key << "] msg: "
      << encoded_msg.size() << "[" << encoded_msg << "]\n";
  return os.str();
}

static void CreateDir(const char *dir) {
  assert(dir);
  std::string cmd("/bin/mkdir -p ");
  cmd += dir;

  if (std::system(cmd.c_str()) < 0) {
    syslog(LOG_ERR, "Failed to create discard log directory [%s]", dir);
    IfLt0(-1);  // this will throw
    /* TODO: Modify implementation to keep running, with discard file logging
       disabled. */
  }
}

TDiscardFileLogger::TDiscardFileLogger()
    : MaxMsgPrefixLen(std::numeric_limits<size_t>::max()),
      UseOldOutputFormat(false),
      Enabled(false),
      MaxFileSize(0),
      MaxArchiveSize(0) {
}

TDiscardFileLogger::~TDiscardFileLogger() noexcept {
  assert(this);

  try {
    Shutdown();
  } catch (const std::exception &x) {
    syslog(LOG_ERR, "Caught unexpected exception in TDiscardFileLogger "
           "destructor: %s", x.what());
    assert(false);
  } catch (...) {
    syslog(LOG_ERR, "Caught unexpected unknown exception in "
           "TDiscardFileLogger destructor");
    assert(false);
  }
}

void TDiscardFileLogger::Init(const char *log_path, uint64_t max_file_size,
    uint64_t max_archive_size, size_t max_msg_prefix_len,
    bool use_old_output_format) {
  assert(this);

  /* Will contain absolute path of directory containing logfile. */
  std::string log_dir;

  /* Will contain just the log filename (without the absolute path). */
  std::string log_filename;

  ParseLogPath(log_path, log_dir, log_filename);

  /* Absolute path of logfile. */
  std::string path(log_path);

  if (Enabled) {
    throw std::logic_error("Discard file logging already initialized");
  }

  CreateDir(log_dir.c_str());
  MaxMsgPrefixLen = max_msg_prefix_len;
  UseOldOutputFormat = use_old_output_format;

  /* Since we are executing during server initialization before any threads can
     create log entries, we modify our internal state without grabbing 'Mutex'.
   */

  LogFd = OpenLogPath(log_path);

  if (!LogFd.IsOpen()) {
    return;
  }

  ArchiveCleaner.reset(new TArchiveCleaner(max_archive_size, log_dir.c_str(),
      log_filename.c_str()));
  ArchiveCleaner->Start();
  LogPath = std::move(path);
  LogDir = std::move(log_dir);
  LogFilename = std::move(log_filename);
  MaxFileSize = max_file_size;
  MaxArchiveSize = max_archive_size;
  Enabled = true;
  CheckMaxFileSize(0);
  ArchiveCleaner->SendCleanRequest();
}

void TDiscardFileLogger::Shutdown() {
  assert(this);

  std::lock_guard<std::mutex> lock(Mutex);
  DisableLogging();

  if (ArchiveCleaner) {
    ArchiveCleaner->RequestShutdown();
    ArchiveCleaner->Join();
    ArchiveCleaner.reset();
  }
}

static const char *ReasonToBlurb(TDiscardFileLogger::TDiscardReason reason) {
  switch (reason) {
    case TDiscardFileLogger::TDiscardReason::Bug:
      break;
    case TDiscardFileLogger::TDiscardReason::FailedDeliveryAttemptLimit:
      return "DELIVERY_ATTEMPT_LIMIT";
    case TDiscardFileLogger::TDiscardReason::KafkaErrorAck:
      return "KAFKA_ERROR_ACK";
    case TDiscardFileLogger::TDiscardReason::ServerShutdown:
      return "SERVER_SHUTDOWN";
    case TDiscardFileLogger::TDiscardReason::NoAvailablePartitions:
      return "NO_AVAILABLE_PARTITIONS";
    case TDiscardFileLogger::TDiscardReason::RateLimit:
      return "RATE_LIMIT";
    case TDiscardFileLogger::TDiscardReason::FailedTopicAutocreate:
      return "TOPIC_AUTOCREATE_FAIL";
    NO_DEFAULT_CASE;
  }

  return "BUG";
}

void TDiscardFileLogger::LogDiscard(const TMsg &msg, TDiscardReason reason) {
  assert(this);

  if (!Enabled) {
    return;  // fast path for case where logging is disabled
  }

  std::vector<uint8_t> key_buf;
  WriteKey(key_buf, 0, msg);
  EnforceMaxPrefixLen(key_buf);
  std::vector<uint8_t> value_buf;
  WriteValue(value_buf, 0, msg, false, UseOldOutputFormat);
  EnforceMaxPrefixLen(value_buf);
  const uint8_t *key_buf_begin = key_buf.empty() ? nullptr : &key_buf[0];
  const uint8_t *value_buf_begin = value_buf.empty() ? nullptr : &value_buf[0];
  WriteToLog(ComposeLogEntry(msg.GetTimestamp(), "DISC", ReasonToBlurb(reason),
                 msg.GetTopic(), key_buf_begin, key_buf.size(),
                 value_buf_begin, value_buf.size()));
}

void TDiscardFileLogger::LogDuplicate(const TMsg &msg) {
  assert(this);

  if (!Enabled) {
    return;  // fast path for case where logging is disabled
  }

  std::vector<uint8_t> key_buf;
  WriteKey(key_buf, 0, msg);
  EnforceMaxPrefixLen(key_buf);
  std::vector<uint8_t> value_buf;
  WriteValue(value_buf, 0, msg, false, UseOldOutputFormat);
  EnforceMaxPrefixLen(value_buf);
  const uint8_t *key_buf_begin = key_buf.empty() ? nullptr : &key_buf[0];
  const uint8_t *value_buf_begin = value_buf.empty() ? nullptr : &value_buf[0];
  WriteToLog(ComposeLogEntry(msg.GetTimestamp(), "DUP", "NONE", msg.GetTopic(),
                 key_buf_begin, key_buf.size(), value_buf_begin,
                 value_buf.size()));
}

void TDiscardFileLogger::LogNoMemDiscard(TMsg::TTimestamp timestamp,
    const char *topic_begin, const char *topic_end, const void *key_begin,
    const void *key_end, const void *value_begin, const void *value_end) {
  assert(this);
  assert(topic_begin);
  assert(topic_end >= topic_begin);
  assert(key_begin || (key_end == key_begin));
  assert(key_end >= key_begin);
  assert(value_begin || (value_end == value_begin));
  assert(value_end >= value_begin);

  if (!Enabled) {
    return;  // fast path for case where logging is disabled
  }

  const std::string topic(topic_begin, topic_end);
  size_t key_size = reinterpret_cast<const uint8_t *>(key_end) -
      reinterpret_cast<const uint8_t *>(key_begin);
  size_t value_size = reinterpret_cast<const uint8_t *>(value_end) -
      reinterpret_cast<const uint8_t *>(value_begin);
  WriteToLog(ComposeLogEntry(timestamp, "DISC", "NO_MEM", topic, key_begin,
                 std::min(key_size, MaxMsgPrefixLen), value_begin,
                 std::min(value_size, MaxMsgPrefixLen)));
}

void TDiscardFileLogger::LogMalformedMsgDiscard(const void *msg_begin,
    const void *msg_end) {
  assert(this);

  if (!Enabled) {
    return;  // fast path for case where logging is disabled
  }

  size_t msg_size = reinterpret_cast<const uint8_t *>(msg_end) -
      reinterpret_cast<const uint8_t *>(msg_begin);
  WriteToLog(ComposeLogEntry(GetEpochMilliseconds(), "DISC", "MALFORMED",
                 std::string(), nullptr, 0, msg_begin, std::min(msg_size,
                 MaxMsgPrefixLen)));
}

void TDiscardFileLogger::LogUnsupportedApiKeyDiscard(const void *msg_begin,
    const void *msg_end, int api_key) {
  assert(this);

  if (!Enabled) {
    return;  // fast path for case where logging is disabled
  }

  std::ostringstream os;
  os << "API_KEY(" << api_key << ")";
  std::string info(os.str());
  size_t msg_size = reinterpret_cast<const uint8_t *>(msg_end) -
      reinterpret_cast<const uint8_t *>(msg_begin);
  WriteToLog(ComposeLogEntry(GetEpochMilliseconds(), "DISC", info.c_str(),
                 std::string(), nullptr, 0, msg_begin,
                 std::min(msg_size, MaxMsgPrefixLen)));
}

void TDiscardFileLogger::LogUnsupportedMsgVersionDiscard(const void *msg_begin,
    const void *msg_end, int version) {
  assert(this);

  if (!Enabled) {
    return;  // fast path for case where logging is disabled
  }

  std::ostringstream os;
  os << "VERSION(" << version << ")";
  std::string info(os.str());
  size_t msg_size = reinterpret_cast<const uint8_t *>(msg_end) -
      reinterpret_cast<const uint8_t *>(msg_begin);
  WriteToLog(ComposeLogEntry(GetEpochMilliseconds(), "DISC", info.c_str(),
                 std::string(), nullptr, 0, msg_begin,
                 std::min(msg_size, MaxMsgPrefixLen)));
}

void TDiscardFileLogger::LogBadTopicDiscard(TMsg::TTimestamp timestamp,
    const char *topic_begin, const char *topic_end, const void *key_begin,
    const void *key_end, const void *value_begin, const void *value_end) {
  assert(this);
  assert(topic_begin);
  assert(topic_end >= topic_begin);
  assert(key_begin || (key_end == key_begin));
  assert(key_end >= key_begin);
  assert(value_begin || (value_end == value_begin));
  assert(value_end >= value_begin);

  if (!Enabled) {
    return;  // fast path for case where logging is disabled
  }

  const std::string topic(topic_begin, topic_end);
  size_t key_size = reinterpret_cast<const uint8_t *>(key_end) -
      reinterpret_cast<const uint8_t *>(key_begin);
  size_t value_size = reinterpret_cast<const uint8_t *>(value_end) -
      reinterpret_cast<const uint8_t *>(value_begin);
  WriteToLog(ComposeLogEntry(timestamp, "DISC", "BAD_TOPIC", topic, key_begin,
                 std::min(key_size, MaxMsgPrefixLen), value_begin,
                 std::min(value_size, MaxMsgPrefixLen)));
}

void TDiscardFileLogger::LogBadTopicDiscard(const TMsg &msg) {
  assert(this);

  if (!Enabled) {
    return;  // fast path for case where logging is disabled
  }

  std::vector<uint8_t> key_buf;
  WriteKey(key_buf, 0, msg);
  EnforceMaxPrefixLen(key_buf);
  std::vector<uint8_t> value_buf;
  WriteValue(value_buf, 0, msg, false, UseOldOutputFormat);
  EnforceMaxPrefixLen(value_buf);
  const uint8_t *key_buf_begin = key_buf.empty() ? nullptr : &key_buf[0];
  const uint8_t *value_buf_begin = value_buf.empty() ? nullptr : &value_buf[0];
  WriteToLog(ComposeLogEntry(msg.GetTimestamp(), "DISC", "BAD_TOPIC",
                 msg.GetTopic(), key_buf_begin, key_buf.size(),
                 value_buf_begin, value_buf.size()));
}

void TDiscardFileLogger::LogLongMsgDiscard(const TMsg &msg) {
  assert(this);

  if (!Enabled) {
    return;  // fast path for case where logging is disabled
  }

  std::vector<uint8_t> key_buf;
  WriteKey(key_buf, 0, msg);
  EnforceMaxPrefixLen(key_buf);
  std::vector<uint8_t> value_buf;
  WriteValue(value_buf, 0, msg, false, UseOldOutputFormat);
  EnforceMaxPrefixLen(value_buf);
  const uint8_t *key_buf_begin = key_buf.empty() ? nullptr : &key_buf[0];
  const uint8_t *value_buf_begin = value_buf.empty() ? nullptr : &value_buf[0];
  WriteToLog(ComposeLogEntry(msg.GetTimestamp(), "DISC", "LONG_MSG",
                 msg.GetTopic(), key_buf_begin, key_buf.size(),
                 value_buf_begin, value_buf.size()));
}

TDiscardFileLogger::TArchiveCleaner::TArchiveCleaner(uint64_t max_archive_size,
    const char *log_dir, const char *log_filename)
    : MaxArchiveSize(max_archive_size),
      LogDir(log_dir),
      LogFilename(log_filename) {
}

TDiscardFileLogger::TArchiveCleaner::~TArchiveCleaner() noexcept {
  /* This will shut down the thread if something unexpected happens. */
  ShutdownOnDestroy();
}

void TDiscardFileLogger::TArchiveCleaner::Run() {
  assert(this);
  int tid = static_cast<int>(Gettid());
  syslog(LOG_INFO, "Discard log cleaner thread %d started", tid);
  bool caught_fatal_exception = false;

  try {
    DoRun();
  } catch (const std::exception &x) {
    caught_fatal_exception = true;
    syslog(LOG_ERR, "Fatal error in discard log cleaner thread %d: %s", tid,
           x.what());
  } catch (...) {
    caught_fatal_exception = true;
    syslog(LOG_ERR, "Fatal unknown error in discard log cleaner thread %d",
           tid);
  }

  syslog(LOG_INFO, "Discard log cleaner thread %d finished %s", tid,
         caught_fatal_exception ? "on error" : "normally");
}

struct TOldLogFileInfo {
  std::string AbsolutePath;

  /* File size in bytes. */
  uint64_t Size;

  /* Epoch milliseconds value obtained from filename. */
  uint64_t EpochMilliseconds;

  TOldLogFileInfo() = default;

  TOldLogFileInfo(std::string &&absolute_path, uint64_t size,
      uint64_t epoch_ms)
      : AbsolutePath(std::move(absolute_path)),
        Size(size),
        EpochMilliseconds(epoch_ms) {
  }

  TOldLogFileInfo(const TOldLogFileInfo &) = default;

  TOldLogFileInfo(TOldLogFileInfo &&that)
      : AbsolutePath(std::move(that.AbsolutePath)),
        Size(that.Size),
        EpochMilliseconds(that.EpochMilliseconds) {
  }

  TOldLogFileInfo& operator=(const TOldLogFileInfo &) = default;

  TOldLogFileInfo& operator=(TOldLogFileInfo &&that) {
    assert(this);

    if (this != &that) {
      AbsolutePath = std::move(that.AbsolutePath);
      Size = that.Size;
      EpochMilliseconds = that.EpochMilliseconds;
    }

    return *this;
  }
};  // TOldLogFileInfo

static std::vector<TOldLogFileInfo>
GetOldLogFileSizes(const char *log_dir, const char *log_filename) {
  assert(log_dir);
  std::vector<TOldLogFileInfo> result;
  size_t name_len = std::strlen(log_filename);
  std::string file_path;

  for (TDirIter iter(log_dir); iter; ++iter) {
    if (iter.GetKind() != TDirIter::File) {
      continue;
    }

    const char *name = iter.GetName();
    size_t i = 0;

    for (i = 0; (i < name_len) && (name[i] == log_filename[i]); ++i);

    if ((i < name_len) || (name[i] != '.')) {
      /* If 'log_filename' is "foo", then we are looking for files named
         "foo.N", where N is a string of digits interpreted as an integer
         number of milliseconds since the epoch.  In this case, we found a
         nonmatching file because either ('log_filename' is not a prefix of
         'name') or ('log_filename' is a prefix of name but the next character
         of 'name' is not '.'). */
      continue;
    }

    if (name[++i] == '\0') {
      /* We were expecting a string of digits after the '.', but there are no
         more characters. */
      continue;
    }

    size_t digits_start = i;
    bool all_digits = true;

    for (; name[i]; ++i) {
      if (!std::isdigit(name[i])) {
        all_digits = false;
        break;
      }
    }

    if (!all_digits) {
      /* This filename contains a nondigit character after the '.'. */
      continue;
    }

    /* 'name[i]' shouldn't contain any leading zeros, but just in case someone
       messed with the filenames, we will skip leading zeros.  A leading zero
       would be bad because it would cause the digit string to be interpreted
       as an octal value when converted to an integer. */
    for (i = digits_start; name[i] == '0'; ++i);

    /* Initializing to 0 handles the case where the digit string is either a
       single zero or a longer string consisting only of zeroes. */
    uint64_t epoch_ms = 0;

    if (name[i]) {
      /* 'name' contains at least one nonzero digit.  Convert the digit string
         (minus any leading zeroes) to an integer value. */
      try {
        epoch_ms = boost::lexical_cast<uint64_t>(&name[i]);
      } catch (const boost::bad_lexical_cast &) {
        syslog(LOG_WARNING, "Failed to extract timestamp from apparently "
               "valid old discard logfile name %s", name);
        continue;
      }
    }

    file_path = log_dir;
    file_path += "/";
    file_path += name;
    struct stat stat_buf;
    int ret = stat(file_path.c_str(), &stat_buf);

    if (ret < 0) {
      if (errno != ENOENT) {
        char buf[256];
        Strerror(errno, buf, sizeof(buf));
        syslog(LOG_WARNING, "Failed to stat() old discard logfile %s: %s",
               file_path.c_str(), buf);
      }

      continue;
    }

    if (!S_ISREG(stat_buf.st_mode)) {
      /* It's not a regular file, so skip it. */
      continue;
    }

    /* We found a file whose name conforms to our old logfile naming scheme, so
       append it to the result vector. */
    result.push_back(TOldLogFileInfo(std::move(file_path), stat_buf.st_size,
                                     epoch_ms));
  }

  return std::move(result);
}

bool TDiscardFileLogger::TArchiveCleaner::HandleCleanRequest() {
  assert(this);
  std::vector<TOldLogFileInfo> old_log_files;

  try {
    old_log_files = GetOldLogFileSizes(LogDir.c_str(), LogFilename.c_str());
  } catch (const std::system_error &x) {
    syslog(LOG_WARNING, "Discard log cleaner thread got fatal error while "
           "examining log directory: %s", x.what());
    return false;
  }

  uint64_t total_size = 0;

  for (const TOldLogFileInfo &info : old_log_files) {
    total_size += info.Size;
  }

  if (total_size <= MaxArchiveSize) {
    return true;
  }

  /* Sort old logfiles from oldest to newest, according to the timestamp values
     extracted from their names. */
  std::sort(old_log_files.begin(), old_log_files.end(),
      [](const TOldLogFileInfo &x, const TOldLogFileInfo &y) {
        return x.EpochMilliseconds < y.EpochMilliseconds;
      });

  /* Delete old logfiles, from oldest to newest, until total size limit is no
     longer violated. */
  for (size_t i = 0; i < old_log_files.size(); ++i) {
    const TOldLogFileInfo &info = old_log_files[i];

    if (unlink(info.AbsolutePath.c_str()) && (errno != ENOENT)) {
      char buf[256];
      Strerror(errno, buf, sizeof(buf));
      syslog(LOG_WARNING, "Failed to unlink old discard logfile %s: %s",
             info.AbsolutePath.c_str(), buf);
    } else {
      total_size -= info.Size;

      if (total_size <= MaxArchiveSize) {
        return true;
      }
    }
  }

  assert(total_size > MaxArchiveSize);
  syslog(LOG_WARNING, "Discard log cleaner failed to delete old logfiles in "
         "directory %s", LogDir.c_str());
  return false;
}

void TDiscardFileLogger::TArchiveCleaner::DoRun() {
  assert(this);
  std::array<struct pollfd, 2> events;
  struct pollfd &shutdown_request_event = events[0];
  struct pollfd &clean_request_event = events[1];
  shutdown_request_event.fd = GetShutdownRequestFd();
  shutdown_request_event.events = POLLIN;
  clean_request_event.fd = CleanRequestSem.GetFd();
  clean_request_event.events = POLLIN;

  for (; ; ) {
    for (auto &item : events) {
      item.revents = 0;
    }

    int ret = IfLt0(poll(&events[0], events.size(), -1));
    assert(ret > 0);

    if (shutdown_request_event.revents) {
      syslog(LOG_INFO, "Discard log cleaner thread got shutdown request");
      break;
    }

    assert(clean_request_event.revents);
    syslog(LOG_INFO,
           "Discard log cleaner thread start handling clean request");
    CleanRequestSem.Pop();

    if (!HandleCleanRequest()) {
      syslog(LOG_WARNING, "Discard log cleaner thread shutting down due to "
             "fatal error handling clean request");
      break;
    }

    syslog(LOG_INFO,
           "Discard log cleaner thread finish handling clean request");
  }
}

void TDiscardFileLogger::ParseLogPath(const char *log_path,
    std::string &log_dir, std::string &log_filename) {
  size_t len = std::strlen(log_path);

  if (len == 0) {
    THROW_ERROR(TInvalidDiscardLogPath);
  }

  if (log_path[0] != '/') {
    THROW_ERROR(TDiscardLogPathMustBeAbsolute);
  }

  /* By standard UNIX convention, a path with a trailing '/' is interpreted
     specifically as a directory.  Since the log path specifies a regular file,
     we treat this case as an error. */
  if (log_path[len - 1] == '/') {
    THROW_ERROR(TDiscardLogPathIsDir);
  }

  /* Below is a partial reimplementation of the dirname() library function.
     We avoid dirname() because it's not guaranteed to be thread safe. */

  size_t last_slash = len - 1;

  /* On loop termination, 'last_slash' is the index of the last '/' character
     in 'log_path'.  The loop is guaranteed to terminate because we already
     verified that (log_path[0] == '/'). */
  for (; log_path[last_slash] != '/'; --last_slash);

  size_t dir_last_char = last_slash;

  /* On loop termination, 'dir_last_char' will either be the index of the last
     nonslash character preceding log_path[last_slash], or 0 if no such
     nonslash character exists.  In either case, the prefix of 'log_path' whose
     last character is at index 'dir_last_char' specifies an absolute path for
     the directory containing the file given by 'log_path'.  This loop is
     necessary because by convention, adjacent '/' characters in a pathname are
     coalesced.  For instance, '/foo/bar' and '/foo//bar' specify the same
     path. */
  for (; dir_last_char && (log_path[dir_last_char] == '/'); --dir_last_char);

  /* Assign absolute path of directory containing logfile. */
  log_dir.assign(log_path, dir_last_char + 1);

  /* Assign log filename (everything after the last '/' character). */
  size_t filename_prefix_len = last_slash + 1;
  log_filename.assign(&log_path[filename_prefix_len],
                      len - filename_prefix_len);
}

const uint8_t *TDiscardFileLogger::EnforceMaxPrefixLen(const void *msg_begin,
    const void *msg_end) {
  assert(this);
  const uint8_t *p1 = reinterpret_cast<const uint8_t *>(msg_begin);
  const uint8_t *p2 = reinterpret_cast<const uint8_t *>(msg_end);
  assert(p2 >= p1);
  size_t msg_size = p2 - p1;
  return p1 + std::min(msg_size, MaxMsgPrefixLen);
}

void TDiscardFileLogger::EnforceMaxPrefixLen(std::vector<uint8_t> &msg) {
  assert(this);

  if (msg.size() > MaxMsgPrefixLen) {
    msg.resize(MaxMsgPrefixLen);
  }
}

TFd TDiscardFileLogger::OpenLogPath(const char *log_path) {
  assert(log_path);
  TFd fd = open(log_path, O_CREAT | O_APPEND | O_WRONLY,
                S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);

  if (!fd.IsOpen()) {
    char buf[256];
    Strerror(errno, buf, sizeof(buf));
    syslog(LOG_WARNING, "Disabling discard logfile mechanism due to failure "
           "to open discard logfile %s for append: %s", log_path, buf);
    DisableLogging();
  }

  return std::move(fd);
}

void TDiscardFileLogger::DisableLogging() {
  assert(this);
  Enabled = false;
  LogFd.Reset();
}

/* See if writing an entry of size 'next_entry_size' in bytes would cause the
   logfile size to exceed the limit.  If so, rename the logfile, create a new
   one in its place, and return true.  Otherwise return false. */
bool TDiscardFileLogger::CheckMaxFileSize(uint64_t next_entry_size) {
  assert(this);
  assert(Enabled);
  assert(ArchiveCleaner);
  assert(LogFd.IsOpen());
  struct stat stat_buf;
  IfLt0(fstat(LogFd, &stat_buf));

  if (!S_ISREG(stat_buf.st_mode)) {
    syslog(LOG_WARNING, "Disabling discard file logging because logfile is "
           "not a regular file: mode is 0x%lx",
           static_cast<unsigned long>(stat_buf.st_mode));
    DisableLogging();
    return false;
  }

  if ((static_cast<uint64_t>(stat_buf.st_size) <= MaxFileSize) &&
      ((MaxFileSize - stat_buf.st_size) >= next_entry_size)) {
    return false;
  }

  uint64_t epoch_ms = GetEpochMilliseconds();
  std::string rename_path(LogPath);
  rename_path += ".";
  rename_path += boost::lexical_cast<std::string>(epoch_ms);
  LogFd.Reset();
  int ret = rename(LogPath.c_str(), rename_path.c_str());

  if (ret < 0) {
    char buf[256];
    Strerror(errno, buf, sizeof(buf));
    syslog(LOG_WARNING, "Disabling discard file logging on failure to rename "
           "logfile: %s", buf);
    DisableLogging();
    return false;
  }

  LogFd = OpenLogPath(LogPath.c_str());
  return LogFd.IsOpen();
}

void TDiscardFileLogger::WriteToLog(const std::string &log_entry) {
  assert(this);

  if (log_entry.empty()) {
    return;
  }

  std::lock_guard<std::mutex> lock(Mutex);

  /* No other thread can change 'Enabled' while we hold 'Mutex'.  Even if we
     just checked 'Enabled' before grabbing 'Mutex', test it again in case it
     changed. */
  if (!Enabled) {
    return;
  }

  assert(LogFd.IsOpen());
  assert(ArchiveCleaner);

  if (CheckMaxFileSize(log_entry.size())) {
    if (ArchiveCleaner->GetShutdownWaitFd().IsReadable()) {
      syslog(LOG_WARNING, "Disabling discard file logging because discard log "
             "cleaner thread shut down unexpectedly");
      DisableLogging();
      return;
    }

    ArchiveCleaner->SendCleanRequest();
  }

  ssize_t ret = IfLt0(write(LogFd, log_entry.data(), log_entry.size()));

  if (static_cast<size_t>(ret) < log_entry.size()) {
    syslog(LOG_ERR, "write() to discard logfile returned short count: "
           "expected %lu actual %lu",
           static_cast<unsigned long>(log_entry.size()),
           static_cast<unsigned long>(ret));
  }
}

max_stars_repo_name: czchen/debian-bruce

max_issues_repo_name: czchen/debian-bruce

max_forks_repo_name: czchen/debian-bruce


text:
//  explore_limits.cpp  ----------------------------------------------------------//

//  Copyright 2008 Howard Hinnant
//  Copyright 2008 Beman Dawes
//  Copyright 2009 Vicente J. Botet Escriba

//  Distributed under the Boost Software License, Version 1.0.
//  See http://www.boost.org/LICENSE_1_0.txt

/*
This code was extracted by Vicente J. Botet Escriba from Beman Dawes time2_demo.cpp which
was derived by Beman Dawes from Howard Hinnant's time2_demo prototype.
Many thanks to Howard for making his code available under the Boost license.
The original code was modified to conform to Boost conventions and to section
20.9 Time utilities [time] of the C++ committee's working paper N2798.
See http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2008/n2798.pdf.

time2_demo contained this comment:

    Much thanks to Andrei Alexandrescu,
                   Walter Brown,
                   Peter Dimov,
                   Jeff Garland,
                   Terry Golubiewski,
                   Daniel Krugler,
                   Anthony Williams.
*/

#include <boost/chrono/chrono.hpp>
#include <boost/type_traits.hpp>

#include <iostream>

using namespace boost::chrono;

void explore_limits()
{
    typedef duration<long long, boost::ratio_multiply<boost::ratio<24*3652425,10000>,
      hours::period>::type> Years;
#ifdef BOOST_CHRONO_HAS_CLOCK_STEADY
    steady_clock::time_point t1( Years(250));
    steady_clock::time_point t2(-Years(250));
#else
    system_clock::time_point t1( Years(250));
    system_clock::time_point t2(-Years(250));
#endif
    // nanosecond resolution is likely to overflow.  "up cast" to microseconds.
    //   The "up cast" trades precision for range.
    microseconds d = time_point_cast<microseconds>(t1) - time_point_cast<microseconds>(t2);
    std::cout << d.count() << " microseconds\n";
}

int main()
{
    explore_limits();
    return 0;
}

ext: cpp

max_stars_repo_name: randolphwong/mcsema

max_issues_repo_name: randolphwong/mcsema

max_forks_repo_name: randolphwong/mcsema


text:
///////////////////////////////////////////////////////////////////////////////
// bug2407.hpp
//
//  Copyright 2008 Eric Niebler. Distributed under the Boost
//  Software License, Version 1.0. (See accompanying file
//  LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

#include <iostream>
#include <boost/proto/proto.hpp>

namespace mpl = boost::mpl;
namespace proto = boost::proto;
using proto::_;

template<class E>
struct e;

struct g
  : proto::or_<
        proto::terminal<int>
      , proto::plus<g,g>
    >
{};

struct d
  : proto::domain<proto::generator<e>, g>
{};

template<class E>
struct e
  : proto::extends<E, e<E>, d>
{
    BOOST_MPL_ASSERT((proto::matches<E, g>));

    e(E const &x = E())
      : proto::extends<E, e<E>, d>(x)
    {}
};

e<proto::terminal<int>::type> i;

template<class E>
std::ostream &operator<<(std::ostream &sout, e<E> const &x)
{
    return sout;
}

int main()
{
    std::cout << (i+i);
}

max_stars_repo_name: zyiacas/boost-doc-zh

max_issues_repo_name: sdfict/boost-doc-zh

max_forks_repo_name: sdfict/boost-doc-zh


text:
#ifndef LEAF_ALL_HPP_INCLUDED
#define LEAF_ALL_HPP_INCLUDED

// Copyright (c) 2018-2019 Emil Dotchevski and Reverge Studios, Inc.

// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

#include <boost/leaf/config.hpp>
#include <boost/leaf/capture.hpp>
#include <boost/leaf/common.hpp>
#include <boost/leaf/context.hpp>
#include <boost/leaf/error.hpp>
#include <boost/leaf/exception.hpp>
#include <boost/leaf/handle_error.hpp>
#ifndef LEAF_NO_EXCEPTIONS
#	include <boost/leaf/handle_exception.hpp>
#endif
#include <boost/leaf/preload.hpp>
#include <boost/leaf/result.hpp>

#endif

ext: hpp


text:
#include <boost/cstdint.hpp>
using namespace boost;

//----------------------------------------------------------------------------
/** @file FuegoMainEngine.cpp
    See FuegoMainEngine.h
*/
//----------------------------------------------------------------------------

#include "SgSystem.h"
#include "FuegoMainEngine.h"

#include "FuegoMainUtil.h"
#include "GoGtpCommandUtil.h"
#include "GoUctPlayer.h"

#include "SpGreedyPlayer.h"
#include "Experiments/HCUBE_GoExperiment.h"

using namespace std;

//----------------------------------------------------------------------------

FuegoMainEngine::FuegoMainEngine(GtpInputStream& in, GtpOutputStream& out, std::string experimentPath,
                                 int fixedBoardSize, const char* programPath,
                                 bool noHandicap)
    :
    GoGtpEngine(in, out, fixedBoardSize, programPath, false, noHandicap),
    m_uctCommands(Board(), m_player),
    m_safetyCommands(Board())
{
    m_uctCommands.Register(*this);
    m_safetyCommands.Register(*this);
    Register("fuego-license", &FuegoMainEngine::CmdLicense, this);

    if(experimentPath.length())
    {
        experimentRun.setupExperimentInProgress(
            experimentPath,
            ""
        );

        int numGenerations = experimentRun.getPopulation()->getGenerationCount();

        goExperiment = (HCUBE::GoExperiment*)experimentRun.getExperiment()->clone();

        shared_ptr<NEAT::GeneticIndividual> indiv =
            shared_ptr<NEAT::GeneticIndividual>(
                new NEAT::GeneticIndividual(
                    *(experimentRun.getIndividual(numGenerations-1,0).get())
                )
            );

        goExperiment->addIndividualToGroup(indiv);
        goExperiment->populateSubstrate(indiv);

        SetPlayer(new HCUBE::HyperNEATPlayer(Board(),goExperiment));

    }
    else
    {
        cout << "COULD NOT FIND PLAYER\n";
        throw CREATE_LOCATEDEXCEPTION_INFO("OOPS");
        //SetPlayer(new SpGreedyPlayer(Board()));
    }
}

FuegoMainEngine::~FuegoMainEngine()
{
}

void FuegoMainEngine::CmdAnalyzeCommands(GtpCommand& cmd)
{
    GoGtpEngine::CmdAnalyzeCommands(cmd);
    m_uctCommands.AddGoGuiAnalyzeCommands(cmd);
    m_safetyCommands.AddGoGuiAnalyzeCommands(cmd);
    cmd << "string/Fuego License/fuego-license\n";
    string response = cmd.Response();
    cmd.SetResponse(GoGtpCommandUtil::SortResponseAnalyzeCommands(response));
}

void FuegoMainEngine::CmdLicense(GtpCommand& cmd)
{
    cmd << "\n" <<
        "Fuego " << FuegoMainUtil::Version() << "\n" <<
        "Copyright (C) 2009 by the authors of the Fuego project.\n"
        "See http://fuego.sf.net for information about Fuego. Fuego comes\n"
        "with NO WARRANTY to the extent permitted by law. This program is\n"
        "free software; you can redistribute it and/or modify it under the\n"
        "terms of the GNU Lesser General Public License as published by the\n"
        "Free Software Foundation - version 3. For more information about\n"
        "these matters, see the files named COPYING and COPYING.LESSER\n";
}

void FuegoMainEngine::CmdName(GtpCommand& cmd)
{
    cmd << "HyperNEATGoPlayer";
}

/** Return Fuego version.
    @see FuegoMainUtil::Version()
*/
void FuegoMainEngine::CmdVersion(GtpCommand& cmd)
{
    cmd << FuegoMainUtil::Version();
}

//----------------------------------------------------------------------------

max_stars_repo_name: LMBernardo/HyperNEAT

max_issues_repo_name: LMBernardo/HyperNEAT

max_forks_repo_name: LMBernardo/HyperNEAT


text:
/* Copyright 2008 (C) Nicira, Inc.
 *
 * This file is part of NOX.
 *
 * NOX is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * NOX is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with NOX.  If not, see <http://www.gnu.org/licenses/>.
 */
#ifndef PYSTORAGE_COMMON_HH
#define PYSTORAGE_COMMON_HH 1

#include "pystorage.hh"

#include <Python.h>

#include <stdexcept>
#include <string>

#include <boost/foreach.hpp>
#include <boost/intrusive_ptr.hpp>
#include <boost/variant/apply_visitor.hpp>
#include <boost/variant/static_visitor.hpp>

#include "pyrt/pyrt.hh"

namespace boost {

/* Functions for the intrusive_ptr to manage the Python object
   references */
inline
void intrusive_ptr_add_ref(PyObject* pobj) {
    Py_INCREF(pobj);
}

inline
void intrusive_ptr_release(PyObject* pobj) {
    Py_DECREF(pobj);
}

}

namespace vigil {
namespace applications {
namespace storage {

typedef boost::intrusive_ptr<PyObject> PyObject_ptr;

template<typename T>
T from_python(PyObject*);

template<typename T>
PyObject* to_python(const T&);

/* Translate Python strings and unicode strings to UTF-8 strings */
template<>
inline
std::string from_python(PyObject* s) {
    if (s && PyString_Check(s)) {
        return std::string(PyString_AsString(s));
    } else if (s && PyUnicode_Check(s)) {
        PyObject* u = PyUnicode_AsUTF8String(s);
        if (!u) {
            throw std::runtime_error("Invalid string (unable to encode "
                                     "as UTF-8)");
        }

        std::string r(PyString_AsString(u));
        Py_DECREF(u);
        return r;
    }

    throw std::runtime_error("Invalid string");
}

template<>
inline
bool from_python(PyObject* b) {
    if (!b || !(b == Py_True || b == Py_False)) {
        throw std::runtime_error("Invalid bool");
    }

    return b == Py_True;
}

template<>
inline
PyObject* to_python(const bool& v) {
    PyObject* ret = v ? Py_True : Py_False;
    Py_INCREF(ret);
    return ret;
}

template<>
inline
int32_t from_python(PyObject* i) {
    if (!i || !PyInt_Check(i)) {
        throw std::runtime_error("Invalid int32");
    }

    return PyLong_AsLong(i);
}

template<>
inline
int64_t from_python(PyObject* i) {
    if (!i || !(PyLong_Check(i) || PyInt_Check(i))) {
        throw std::runtime_error("Invalid int64");
    }

    return PyLong_AsLongLong(i);
}

template<>
inline
PyObject* to_python(const int64_t& i) {
    return PyLong_FromLongLong(i);
}

template<>
inline
PyObject* to_python(const int32_t& i) {
    return PyInt_FromLong(i);
}

template<>
inline
double from_python(PyObject* d) {
    if (!d || !PyFloat_Check(d)) {
        throw std::runtime_error("Invalid double object");
    }

    return PyFloat_AsDouble(d);
}

template<>
inline
PyObject* to_python(const double& d) {
    return PyFloat_FromDouble(d);
}

template<>
inline
PyObject* to_python(const std::string& s) {
    PyObject* str = PyUnicode_DecodeUTF8(s.c_str(), s.length(), 0);
    if (!str) {
        throw std::runtime_error("Decoding an UTF-8 string to unicode string "
                                 "failed:\n" + pretty_print_python_exception());
    }
    return str;
}

template<>
inline
PyObject* to_python(const Trigger_reason& reason) {
    return PyInt_FromLong(reason);
}

template<>
inline
GUID from_python(PyObject* g_) {
    uint8_t* buffer;
    Py_ssize_t length;

    if (!PyTuple_Check(g_) || PyTuple_GET_SIZE(g_) != 2) {
        throw std::runtime_error("GUID not a 2-tuple");
    }

    PyObject* g = PyTuple_GET_ITEM(g_, 1);
    if (!PyString_Check(g) || PyString_GET_SIZE(g) != GUID::GUID_SIZE ||
        PyString_AsStringAndSize(g, (char**)&buffer, &length) == -1) {
        PyErr_Clear();
        throw std::runtime_error("GUID not a proper binary string");
    }

    return GUID(buffer);
}

template<>
inline
PyObject* to_python(const GUID& guid) {
    PyObject* obj = PyTuple_New(2);
    PyTuple_SET_ITEM(obj, 0, to_python<std::string>("GUID"));
    PyTuple_SET_ITEM(obj, 1, PyString_FromStringAndSize((const char*)guid.guid,
                                                        GUID::GUID_SIZE));
    return obj;

}

template<>
inline
Reference from_python(PyObject* ref) {
    if (!ref || !PyTuple_Check(ref) || PyTuple_GET_SIZE(ref) != 3) {
        throw std::runtime_error("Invalid Reference object");
    }

    return Reference(PyInt_AsLong(PyTuple_GetItem(ref, 0)),
                     from_python<GUID>(PyTuple_GET_ITEM(ref, 1)),
                     from_python<bool>(PyTuple_GET_ITEM(ref, 2)));
}

template<>
inline
PyObject* to_python(const Reference& ref) {
    PyObject* py_ref = PyTuple_New(3);
    try {
        PyTuple_SET_ITEM(py_ref, 0, PyInt_FromLong(ref.version));
        PyTuple_SET_ITEM(py_ref, 1, to_python(ref.guid));
        PyObject* py_wildcard =
            ref.wildcard ? Py_True : Py_False; Py_INCREF(py_wildcard);
        PyTuple_SET_ITEM(py_ref, 2, py_wildcard);

        return py_ref;
    }
    catch (...) {
        Py_DECREF(py_ref);
        throw;
    }
}

template<>
inline
Trigger_id from_python(PyObject* tid) {
    if (!tid || !PyTuple_Check(tid) || PyTuple_GET_SIZE(tid) != 4) {
        throw std::runtime_error("Invalid Trigger id object");
    }

    return Trigger_id(from_python<bool>(PyTuple_GET_ITEM(tid, 0)),
                      from_python<std::string>(PyTuple_GET_ITEM(tid, 1)),
                      from_python<Reference>(PyTuple_GET_ITEM(tid, 2)),
                      from_python<int64_t>(PyTuple_GET_ITEM(tid, 3)));
}

template<>
inline
PyObject* to_python(const Trigger_id& tid) {
    PyObject* py_tid = PyTuple_New(4);
    try {
        PyTuple_SET_ITEM(py_tid, 0, to_python(tid.for_table));
        PyTuple_SET_ITEM(py_tid, 1, to_python(tid.ring));
        PyTuple_SET_ITEM(py_tid, 2, to_python(tid.ref));
        PyTuple_SET_ITEM(py_tid, 3, to_python(tid.tid));

        return py_tid;
    }
    catch (...) {
        Py_DECREF(py_tid);
        throw;
    }
}

template<>
inline
Column_value_map from_python(PyObject* cv) {
    if (!cv || !PyDict_Check(cv)) {
        throw std::runtime_error("Invalid column names or values");
    }

    Column_value_map m;
    PyObject* keys = PyDict_Keys(cv);

    try {
        for (Py_ssize_t i = 0; i < PyList_GET_SIZE(keys); ++i) {
            PyObject* k = PyList_GET_ITEM(keys, i);
            const Column_name name = from_python<std::string>(k);

            PyObject* v = PyDict_GetItem(cv, k);
            Column_value value;
            if (PyString_Check(v)) {
                m[name] = from_python<std::string>(v);
            } else if (PyUnicode_Check(v)) {
                PyObject_ptr str(PyUnicode_AsUTF8String(v), false);
                if (!str.get()) {
                    throw std::runtime_error("Encoding an unicode string to "
                                             "UTF-8 failed:\n" +
                                             pretty_print_python_exception());
                }

                m[name] = from_python<std::string>(str.get());
            } else if (PyInt_Check(v)) {
                m[name] = from_python<int64_t>(v);
            } else if (PyLong_Check(v)) {
                m[name] = from_python<int64_t>(v);
            } else if (PyFloat_Check(v)) {
                m[name] = from_python<double>(v);
            } else if (PyTuple_Check(v) && PyTuple_GET_SIZE(v) == 2) {
                m[name] = from_python<GUID>(v);
            } else if (PyType_Check(v)) {
                if (PyType_IsSubtype((PyTypeObject*)v, &PyString_Type)) {
                    m[name] = "";
                } else if (PyType_IsSubtype((PyTypeObject*)v, &PyInt_Type)) {
                    m[name] = (int64_t)0;
                } else if (PyType_IsSubtype((PyTypeObject*)v, &PyFloat_Type)) {
                    m[name] = (double)0;
                } else if (PyType_IsSubtype((PyTypeObject*)v, &PyTuple_Type)) {
                    m[name] = GUID();
                } else {
                    throw std::runtime_error("unknown Python type object");
                }
            } else {
                throw std::runtime_error("unknown Python column type");
            }
        }

        Py_DECREF(keys);
        return m;
    }
    catch (...) {
        Py_DECREF(keys);
        throw;
    }
}

/* Visitor for computing a SHA-1 digest out of a set of column values. */
struct Column_value_converter
    : public boost::static_visitor<>
{
    void operator()(const int64_t& i) const { obj = to_python(i); }
    void operator()(const std::string& s) const { obj = to_python(s); }
    void operator()(const double& d) const { obj = to_python(d); }
    void operator()(const GUID& guid) const { obj = to_python(guid); }

    mutable PyObject* obj;
};

template <>
inline
PyObject* to_python(const Column_value_map& row) {
    PyObject* r = PyDict_New();
    try {
        Column_value_converter converter;

        BOOST_FOREACH(Column_value_map::value_type v, row) {
            boost::apply_visitor(converter, v.second);
            PyObject* key = to_python(v.first);
            PyObject* value = converter.obj;
            PyDict_SetItem(r, key, value); Py_DECREF(key); Py_DECREF(value);
        }

        return r;
    }
    catch (...) {
        Py_DECREF(r);
        throw;
    }
}

template<>
inline
Index_list from_python(PyObject* indices) {
    if (!indices || !PyTuple_Check(indices)) {
        throw std::runtime_error("Indices not defined as a tuple.");
    }

    Index_list r;
    for (Py_ssize_t j = 0; j < PyTuple_GET_SIZE(indices); ++j) {
        PyObject* index = PyTuple_GET_ITEM(indices, j);
        if (!index || !PyTuple_Check(index) || PyTuple_GET_SIZE(index) != 2) {
            throw std::runtime_error("Index not defined as a tuple or "
                                     "has wrong length");
        }

        Index i;
        i.name = from_python<std::string>(PyTuple_GET_ITEM(index, 0));
        PyObject* columns = PyTuple_GET_ITEM(index, 1);
        if (!columns || !PyTuple_Check(columns)) {
            throw std::runtime_error("Index columns not defined as a tuple");
        }

        for (Py_ssize_t k = 0; k < PyTuple_GET_SIZE(columns); ++k) {
            Column_name n =
                from_python<std::string>(PyTuple_GET_ITEM(columns, k));
            i.columns.push_back(n);
        }

        r.push_back(i);
    }

    return r;
}

/* Convert the Python storage context to C++ context */
template<>
inline
Context from_python(PyObject* ctxt) {
    if (!ctxt || !PyTuple_Check(ctxt) || PyTuple_GET_SIZE(ctxt) != 5) {
        throw std::runtime_error("Invalid Context object");
    }

    Table_name table = from_python<Table_name>(PyTuple_GET_ITEM(ctxt, 0));
    Index_name index = from_python<Index_name>(PyTuple_GET_ITEM(ctxt, 1));
    Reference index_row = from_python<Reference>(PyTuple_GET_ITEM(ctxt, 2));
    Reference initial_row = from_python<Reference>(PyTuple_GET_ITEM(ctxt, 3));
    Reference current_row = from_python<Reference>(PyTuple_GET_ITEM(ctxt, 4));

    Context c(table);
    c.table = table;
    c.index = index;
    c.index_row = index_row;
    c.initial_row = initial_row;
    c.current_row = current_row;
    return c;
}

template<>
inline
PyObject* to_python(const Context& ctxt)  {
    PyObject* py_ctxt = PyTuple_New(5);

    try {
        PyTuple_SET_ITEM(py_ctxt, 0, to_python(ctxt.table));
        PyTuple_SET_ITEM(py_ctxt, 1, to_python(ctxt.index));
        PyTuple_SET_ITEM(py_ctxt, 2, to_python(ctxt.index_row));
        PyTuple_SET_ITEM(py_ctxt, 3, to_python(ctxt.initial_row));
        PyTuple_SET_ITEM(py_ctxt, 4, to_python(ctxt.current_row));

        return py_ctxt;
    }
    catch (...) {
        Py_DECREF(py_ctxt);
        throw;
    }
}

/* Convert the C++ result to Python result */
template<>
inline
PyObject* to_python(const Result& result) {
    PyObject* py_result = PyTuple_New(2);

    try {
        PyTuple_SET_ITEM(py_result, 0, PyInt_FromLong(result.code));
        PyTuple_SET_ITEM(py_result, 1, to_python(result.message));

        return py_result;
    }
    catch (...) {
        Py_DECREF(py_result);
        throw;
    }
}

}
}
}

#endif /* PYSTORAGE_COMMON_HH */

ext: hh

max_stars_repo_name: ayjazz/OESS

max_issues_repo_name: ayjazz/OESS

max_forks_repo_name: ayjazz/OESS


text:
// Copyright Louis Dionne 2013-2016
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)

#include <boost/hana/assert.hpp>
#include <boost/hana/equal.hpp>
#include <boost/hana/integral_constant.hpp>
#include <boost/hana/less.hpp>
#include <boost/hana/take_while.hpp>
#include <boost/hana/tuple.hpp>
namespace hana = boost::hana;
using namespace hana::literals;

BOOST_HANA_CONSTANT_CHECK(
    hana::take_while(hana::tuple_c<int, 0, 1, 2, 3>, hana::less.than(2_c))
    ==
    hana::tuple_c<int, 0, 1>
);

int main() { }

max_stars_repo_name: nxplatform/nx-mobile

max_issues_repo_name: nxplatform/nx-mobile

max_forks_repo_name: nxplatform/nx-mobile


text:
#include <iostream>
#include <boost/asio.hpp>

#ifndef TCPCONNECTION
#define TCPCONNECTION

struct tcpThreadArguments{
  std::string serverAddress; //the address of the server to connect to
  int portNumber;
  std::string message;
} typedef tcpThreadArguments_t;

void* startServer_baseline(void *);
void* startClient_baseline(void *);

std::string serializePtCloud(std::vector<idxXYZ> data);
std::vector<idxXYZ> deserializePtCloud(std::string str);

#endif

max_stars_repo_name: amir-keramatian/MAD-C

max_issues_repo_name: amir-keramatian/MAD-C

max_forks_repo_name: amir-keramatian/MAD-C


text:
// Copyright 2019 Lawrence Livermore National Security, LLC and other Metall Project Developers.
// See the top-level COPYRIGHT file for details.
//
// SPDX-License-Identifier: (Apache-2.0 OR MIT)

#include <iostream>
#include <boost/container/scoped_allocator.hpp>
#include <boost/container/string.hpp>
#include <boost/container/map.hpp>
#include <metall/metall.hpp>

// String with Metall
using persistent_string = boost::container::basic_string<char,
                                                         std::char_traits<char>,
                                                         metall::manager::allocator_type<char>>;
// Example of string-int map
void string_int_map() {
  using value_type = std::pair<const persistent_string, // Key type
                               int>; // Mapped type
  using string_int_map = boost::container::map<persistent_string,
                                               int,
                                               std::less<persistent_string>,
                                               metall::manager::allocator_type<value_type>>;

  {
    metall::manager manager(metall::create_only, "/tmp/datastore");
    auto pmap = manager.construct<string_int_map>("string-int-map")(manager.get_allocator<>());

    pmap->insert(value_type(persistent_string("zero", manager.get_allocator<>()), 0));

    (*pmap)[persistent_string("one", manager.get_allocator<>())] = 1;
  }

  {
    metall::manager manager(metall::open_only, "/tmp/datastore");
    auto pmap = manager.find<string_int_map>("string-int-map").first;

    std::cout << pmap->at(persistent_string("zero", manager.get_allocator<>())) << std::endl; // Will print "0"
    std::cout << pmap->at(persistent_string("one", manager.get_allocator<>())) << std::endl; // Will print "1"
  }
}

// Example of int-string map
// This is also considered as an example of a container of containers
void int_string_map() {
  using value_type = std::pair<const int, // Key type
                               persistent_string>; // Mapped type

  // Scoped allocator is required to force to use user given allocators in inner containers
  using map_allocator_type = boost::container::scoped_allocator_adaptor<metall::manager::allocator_type<value_type>>;
  using int_string_map = boost::container::map<int,
                                               persistent_string,
                                               std::less<int>,
                                               map_allocator_type>;

  {
    metall::manager manager(metall::create_only, "/tmp/datastore");
    auto pmap = manager.construct<int_string_map>("int-string-map")(manager.get_allocator<>());

    pmap->insert(value_type(0, persistent_string("zero", manager.get_allocator<>())));

    (*pmap)[1] = persistent_string("one", manager.get_allocator<>());

    // Mapped objects (persistent_string) use an allocator stored in pmap automatically thanks to scoped allocator
    (*pmap)[2].push_back('t');
    (*pmap)[2].push_back('w');
    (*pmap)[2].push_back('o');
  }

  {
    metall::manager manager(metall::open_only, "/tmp/datastore");
    auto pmap = manager.find<int_string_map>("int-string-map").first;

    std::cout << pmap->at(0) << std::endl; // Will print "zero"
    std::cout << pmap->at(1) << std::endl; // Will print "one"
    std::cout << pmap->at(2) << std::endl; // Will print "two"
  }
}

int main() {

  string_int_map();
  int_string_map();

  return 0;
}

max_stars_repo_name: dice-group/metall

max_issues_repo_name: dice-group/metall

max_forks_repo_name: dice-group/metall


text:
#include <alya.h>

#include <string.h>

#include <algorithm>
#include <chrono>
#include <cstring>
#include <fstream>
#include <thread>
#include <vector>
#include <unordered_map>
#include <unordered_set>
#include <map>

//#include <boost/algorithm/string/case_conv.hpp>

template <class T>
struct is_const
{
    static constexpr bool value = std::is_const<typename std::remove_reference<T>::type>::value;
};

TRACE_CLASS(A);

template <class T>
void f(T&& x)
{
    COUTF(std::is_reference<T>::value);
    COUTF(std::is_rvalue_reference<T>::value);
    COUTF(is_const<T>::value);
    COUTF(std::is_reference<decltype(x)>::value);
    COUTF(std::is_rvalue_reference<decltype(x)>::value);
    COUTF(is_const<decltype(x)>::value);
}

template <class T>
void ff(T x)
{
    COUTF(std::is_reference<T>::value);
    COUTF(std::is_rvalue_reference<T>::value);
    COUTF(is_const<T>::value);
    COUTF(std::is_reference<decltype(x)>::value);
    COUTF(std::is_rvalue_reference<decltype(x)>::value);
    COUTF(is_const<decltype(x)>::value);
}

__attribute__((noinline)) void test(size_t c)
{
    std::string_view str ("Hello world\nHello world\nHello world\nHello world\nHello world\nHello world\nHello world\nHello world\nHello world\nHello world\n");
    for (size_t i = 0; i < c; i++)
        std::cout << str;
}

__attribute__((noinline)) uint64_t test_internal(uint64_t *arr, uint64_t size)
{
    uint64_t res = 0;
    for (size_t i = 0; i < size; i++)
        res += __builtin_ctzl(arr[i]);
    return res;
}

__attribute__((noinline)) uint64_t wrap_test(uint64_t *arr, uint64_t size)
{
    alya::CTimer t;
    t.Start();
    uint64_t res = test_internal(arr, size);
    t.Stop();
    COUTF(t.Mrps(size));
    return res;
}

constexpr size_t SIZE = 1024 * 1024;
uint64_t arr[SIZE];

void prn(uint64_t v)
{
    std::cout << "--------------------" << std::endl;
    for (size_t i = 0; i < 8; i++) {
        for (size_t j = 0; j < 8; j++) {
            std::cout << ((v & (1ull << (i * 8  + j))) ? "o " : ". ");
        }
        std::cout << std::endl;
    }
}

uint64_t move(uint64_t& v)
{
    uint64_t res = v;
    uint64_t t;
    t  = (v >> 1) & 0x7F7F7F7F7F7F7F7Full;
    t |= (v << 1) & 0xFEFEFEFEFEFEFEFEull;
    res |= t << 16;
    res |= t >> 16;
    t  = (v >> 2) & 0x3F3F3F3F3F3F3F3Full;
    t |= (v << 2) & 0xFCFCFCFCFCFCFCFCull;
    res |= t << 8;
    res |= t >> 8;
    return res;
}

int main()
{
    uint64_t v = 1ull << 36;
    prn(v);
    for (size_t i = 0; i < 6; i++) {
        v = move(v);
        prn(v);
    }

    return 0;

    for (size_t i = 0; i < SIZE; i++)
        arr[i] = rand() + 1;
    COUTF(wrap_test(arr, SIZE));
    return 0;

    srand(time(0));
    size_t M = 1000;
    size_t N = 100;
    size_t R = 0;
    for (size_t i = 0; i < M; i++)
    {
        if (size_t(rand()) >= size_t((M - N) * RAND_MAX / M))
            R++;
    }
    COUTF(R);
    return 0;

    std::ios::sync_with_stdio(false);
    test(100000);

    return 0;

    A a;
    const A& b = std::move(a);
    A&& c = std::move(a);
    COUTF(std::is_reference<A&&>::value);
    COUTF(std::is_rvalue_reference<A&&>::value);
    COUTF(std::is_reference<decltype(b)>::value);
    COUTF(std::is_rvalue_reference<decltype(b)>::value);
    COUTF(is_const<decltype(b)>::value);
    COUTF(std::is_reference<decltype(c)>::value);
    COUTF(std::is_rvalue_reference<decltype(c)>::value);
    COUTF(is_const<decltype(c)>::value);

    COUT("---val---");
    f(a);
    COUT("---ref---");
    f(b);
    COUT("---move---");
    f(std::move(c));

    COUT("---val---");
    ff(a);
    COUT("---ref---");
    ff(b);
    COUT("---move---");
    ff(std::move(c));

    return 0;
    const char rus[33][2][4] =
    {
        {"а", "А"},
        {"б", "Б"},
        {"в", "В"},
        {"г", "Г"},
        {"д", "Д"},
        {"е", "Е"},
        {"ё", "Ё"},
        {"ж", "Ж"},
        {"з", "З"},
        {"и", "И"},
        {"й", "Й"},
        {"к", "К"},
        {"л", "Л"},
        {"м", "М"},
        {"н", "Н"},
        {"о", "О"},
        {"п", "П"},
        {"р", "Р"},
        {"с", "С"},
        {"т", "Т"},
        {"у", "У"},
        {"ф", "Ф"},
        {"х", "Х"},
        {"ц", "Ц"},
        {"ч", "Ч"},
        {"ш", "Ш"},
        {"щ", "Щ"},
        {"ъ", "Ъ"},
        {"ы", "Ы"},
        {"ь", "Ь"},
        {"э", "Э"},
        {"ю", "Ю"},
        {"я", "Я"}
    };
    for (size_t i = 0; i < 33; i++)
    {
        std::cout << rus[i][0] << rus[i][1];
        for (size_t j = 0; j < 2; j++)
        {
            std::cout << ": ";
            for (size_t k = 0; k < 4; k++)
            {
                if (rus[i][j][k] == 0)
                    break;
                int a = (int)(unsigned char)rus[i][j][k];
                std::cout << a << " ";
            }
        }
        std::cout << "\n";
    }

    /*
    COUTF(std::setlocale(LC_ALL, "ru_RU.UTF-8"));
    const char* base = "aAbBzZаАёЁйЙяЯ";
    wchar_t base_w[30];
    wchar_t low_w[30];
    char low[30];

    size_t len_w = mbstowcs(base_w, base, strlen(base));
    if (len_w == static_cast<size_t>(-1))
        COUTF(len_w);
    base_w[len_w] = 0;

    for (size_t i = 0; i < len_w; i++)
        low_w[i] = std::towlower(base_w[i]);
    low_w[len_w] = 0;

    size_t len = wcstombs(low, low_w, len_w);
    low[len] = 0;

    std::cout << base << std::endl;
    std::wcout << base_w << std::endl;
    std::wcout << low_w << std::endl;
    std::cout << low << std::endl;
    */
}

max_stars_repo_name: alyapunov/tests

max_issues_repo_name: alyapunov/tests

max_forks_repo_name: alyapunov/tests


text:
#include <string.h>
#include "I2PEndian.h"
#include <fstream>
#include <vector>
#include <boost/asio.hpp>
#include <cryptopp/gzip.h>
#include "base64.h"
#include "Log.h"
#include "Timestamp.h"
#include "I2NPProtocol.h"
#include "Tunnel.h"
#include "Transports.h"
#include "RouterContext.h"
#include "Garlic.h"
#include "NetDb.h"
#include "util.h"

using namespace i2p::transport;

namespace i2p
{
namespace data
{
	const char NetDb::m_NetDbPath[] = "netDb";
	NetDb netdb;

	NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr)
	{
	}

	NetDb::~NetDb ()
	{
		Stop ();
		delete m_Reseeder;
	}

	void NetDb::Start ()
	{
		Load ();
		if (m_RouterInfos.size () < 25) // reseed if # of router less than 50
		{
			// try SU3 first
			Reseed ();

			// deprecated
			if (m_Reseeder)
			{
				// if still not enough download .dat files
				int reseedRetries = 0;
				while (m_RouterInfos.size () < 25 && reseedRetries < 5)
				{
					m_Reseeder->reseedNow();
					reseedRetries++;
					Load ();
				}
			}
		}
		m_IsRunning = true;
		m_Thread = new std::thread (std::bind (&NetDb::Run, this));
	}

	void NetDb::Stop ()
	{
		if (m_IsRunning)
		{
			for (auto it: m_RouterInfos)
				it.second->SaveProfile ();
			DeleteObsoleteProfiles ();
			m_RouterInfos.clear ();
			m_Floodfills.clear ();
			if (m_Thread)
			{
				m_IsRunning = false;
				m_Queue.WakeUp ();
				m_Thread->join ();
				delete m_Thread;
				m_Thread = 0;
			}
			m_LeaseSets.clear();
			m_Requests.Stop ();
		}
	}

	void NetDb::Run ()
	{
		uint32_t lastSave = 0, lastPublish = 0, lastExploratory = 0, lastManageRequest = 0;
		while (m_IsRunning)
		{
			try
			{
				auto msg = m_Queue.GetNextWithTimeout (15000); // 15 sec
				if (msg)
				{
					int numMsgs = 0;
					while (msg)
					{
						switch (msg->GetTypeID ())
						{
							case eI2NPDatabaseStore:
								LogPrint ("DatabaseStore");
								HandleDatabaseStoreMsg (msg);
							break;
							case eI2NPDatabaseSearchReply:
								LogPrint ("DatabaseSearchReply");
								HandleDatabaseSearchReplyMsg (msg);
							break;
							case eI2NPDatabaseLookup:
								LogPrint ("DatabaseLookup");
								HandleDatabaseLookupMsg (msg);
							break;
							default: // WTF?
								LogPrint (eLogError, "NetDb: unexpected message type ", msg->GetTypeID ());
								//i2p::HandleI2NPMessage (msg);
						}
						if (numMsgs > 100) break;
						msg = m_Queue.Get ();
						numMsgs++;
					}
				}
				if (!m_IsRunning) break;

				uint64_t ts = i2p::util::GetSecondsSinceEpoch ();
				if (ts - lastManageRequest >= 15) // manage requests every 15 seconds
				{
					m_Requests.ManageRequests ();
					lastManageRequest = ts;
				}
				if (ts - lastSave >= 60) // save routers, manage leasesets and validate subscriptions every minute
				{
					if (lastSave)
					{
						SaveUpdated ();
						ManageLeaseSets ();
					}
					lastSave = ts;
				}
				if (ts - lastPublish >= 2400) // publish every 40 minutes
				{
					Publish ();
					lastPublish = ts;
				}
				if (ts - lastExploratory >= 30) // exploratory every 30 seconds
				{
					auto numRouters = m_RouterInfos.size ();
					if (numRouters < 2500 || ts - lastExploratory >= 90)
					{
						numRouters = 800/numRouters;
						if (numRouters < 1) numRouters = 1;
						if (numRouters > 9) numRouters = 9;
						m_Requests.ManageRequests ();
						Explore (numRouters);
						lastExploratory = ts;
					}
				}
			}
			catch (std::exception& ex)
			{
				LogPrint ("NetDb: ", ex.what ());
			}
		}
	}

	void NetDb::AddRouterInfo (const uint8_t * buf, int len)
	{
		IdentityEx identity;
		if (identity.FromBuffer (buf, len))
			AddRouterInfo (identity.GetIdentHash (), buf, len);
	}

	void NetDb::AddRouterInfo (const IdentHash& ident, const uint8_t * buf, int len)
	{
		auto r = FindRouter (ident);
		if (r)
		{
			auto ts = r->GetTimestamp ();
			r->Update (buf, len);
			if (r->GetTimestamp () > ts)
				LogPrint ("RouterInfo updated");
		}
		else
		{
			LogPrint ("New RouterInfo added");
			r = std::make_shared<RouterInfo> (buf, len);
			{
				std::unique_lock<std::mutex> l(m_RouterInfosMutex);
				m_RouterInfos[r->GetIdentHash ()] = r;
			}
			if (r->IsFloodfill ())
			{
				std::unique_lock<std::mutex> l(m_FloodfillsMutex);
				m_Floodfills.push_back (r);
			}
		}
		// take care about requested destination
		m_Requests.RequestComplete (ident, r);
	}

	void NetDb::AddLeaseSet (const IdentHash& ident, const uint8_t * buf, int len,
		std::shared_ptr<i2p::tunnel::InboundTunnel> from)
	{
		if (!from) // unsolicited LS must be received directly
		{
			auto it = m_LeaseSets.find(ident);
			if (it != m_LeaseSets.end ())
			{
				it->second->Update (buf, len);
				if (it->second->IsValid ())
					LogPrint (eLogInfo, "LeaseSet updated");
				else
				{
					LogPrint (eLogInfo, "LeaseSet update failed");
					m_LeaseSets.erase (it);
				}
			}
			else
			{
				auto leaseSet = std::make_shared<LeaseSet> (buf, len);
				if (leaseSet->IsValid ())
				{
					LogPrint (eLogInfo, "New LeaseSet added");
					m_LeaseSets[ident] = leaseSet;
				}
				else
					LogPrint (eLogError, "New LeaseSet validation failed");
			}
		}
	}

	std::shared_ptr<RouterInfo> NetDb::FindRouter (const IdentHash& ident) const
	{
		std::unique_lock<std::mutex> l(m_RouterInfosMutex);
		auto it = m_RouterInfos.find (ident);
		if (it != m_RouterInfos.end ())
			return it->second;
		else
			return nullptr;
	}

	std::shared_ptr<LeaseSet> NetDb::FindLeaseSet (const IdentHash& destination) const
	{
		auto it = m_LeaseSets.find (destination);
		if (it != m_LeaseSets.end ())
			return it->second;
		else
			return nullptr;
	}

	void NetDb::SetUnreachable (const IdentHash& ident, bool unreachable)
	{
		auto it = m_RouterInfos.find (ident);
		if (it != m_RouterInfos.end ())
			return it->second->SetUnreachable (unreachable);
	}

	// TODO: Move to reseed and/or scheduled tasks. (In java version, scheduler fix this as well as sort RIs.)
	bool NetDb::CreateNetDb(boost::filesystem::path directory)
	{
		LogPrint (directory.string(), " doesn't exist, trying to create it.");
		if (!boost::filesystem::create_directory (directory))
		{
			LogPrint("Failed to create directory ", directory.string());
			return false;
		}

		// list of chars might appear in base64 string
		const char * chars = GetBase64SubstitutionTable (); // 64 bytes
		boost::filesystem::path suffix;
		for (int i = 0; i < 64; i++)
		{
#ifndef _WIN32
			suffix = std::string ("/r") + chars[i];
#else
			suffix = std::string ("\\r") + chars[i];
#endif
			if (!boost::filesystem::create_directory( boost::filesystem::path (directory / suffix) )) return false;
		}
		return true;
	}

	void NetDb::Reseed ()
	{
		if (!m_Reseeder)
		{
			m_Reseeder = new Reseeder ();
			m_Reseeder->LoadCertificates (); // we need certificates for SU3 verification
		}
		int reseedRetries = 0;
		while (reseedRetries < 10 && !m_Reseeder->ReseedNowSU3 ())
			reseedRetries++;
		if (reseedRetries >= 10)
			LogPrint (eLogWarning, "Failed to reseed after 10 attempts");
	}

	void NetDb::Load ()
	{
		boost::filesystem::path p(i2p::util::filesystem::GetDataDir() / m_NetDbPath);
		if (!boost::filesystem::exists (p))
		{
			// seems netDb doesn't exist yet
			if (!CreateNetDb(p)) return;
		}
		// make sure we cleanup netDb from previous attempts
		m_RouterInfos.clear ();
		m_Floodfills.clear ();

		// load routers now
		uint64_t ts = i2p::util::GetMillisecondsSinceEpoch ();
		int numRouters = 0;
		boost::filesystem::directory_iterator end;
		for (boost::filesystem::directory_iterator it (p); it != end; ++it)
		{
			if (boost::filesystem::is_directory (it->status()))
			{
				for (boost::filesystem::directory_iterator it1 (it->path ()); it1 != end; ++it1)
				{
#if BOOST_VERSION > 10500
					const std::string& fullPath = it1->path().string();
#else
					const std::string& fullPath = it1->path();
#endif
					auto r = std::make_shared<RouterInfo>(fullPath);
					if (!r->IsUnreachable () && (!r->UsesIntroducer () || ts < r->GetTimestamp () + 3600*1000LL)) // 1 hour
					{
						r->DeleteBuffer ();
						r->ClearProperties (); // properties are not used for regular routers
						m_RouterInfos[r->GetIdentHash ()] = r;
						if (r->IsFloodfill ())
							m_Floodfills.push_back (r);
						numRouters++;
					}
					else
					{
						if (boost::filesystem::exists (fullPath))
							boost::filesystem::remove (fullPath);
					}
				}
			}
		}
		LogPrint (numRouters, " routers loaded");
		LogPrint (m_Floodfills.size (), " floodfills loaded");
	}

	void NetDb::SaveUpdated ()
	{
		auto GetFilePath = [](const boost::filesystem::path& directory, const RouterInfo * routerInfo)
		{
			std::string s(routerInfo->GetIdentHashBase64());
			return directory / (std::string("r") + s[0]) / ("routerInfo-" + s + ".dat");
		};

		boost::filesystem::path fullDirectory (i2p::util::filesystem::GetDataDir() / m_NetDbPath);
		int count = 0, deletedCount = 0;
		auto total = m_RouterInfos.size ();
		uint64_t ts = i2p::util::GetMillisecondsSinceEpoch ();
		for (auto it: m_RouterInfos)
		{
			if (it.second->IsUpdated ())
			{
				std::string f = GetFilePath(fullDirectory, it.second.get()).string();
				it.second->SaveToFile (f);
				it.second->SetUpdated (false);
				it.second->SetUnreachable (false);
				it.second->DeleteBuffer ();
				count++;
			}
			else
			{
				// RouterInfo expires after 1 hour if uses introducer
				if (it.second->UsesIntroducer () && ts > it.second->GetTimestamp () + 3600*1000LL) // 1 hour
					it.second->SetUnreachable (true);
				else if (total > 75 && ts > (i2p::context.GetStartupTime () + 600)*1000LL) // routers don't expire if less than 25 or uptime is less than 10 minutes
				{
					if (i2p::context.IsFloodfill ())
					{
						if (ts > it.second->GetTimestamp () + 3600*1000LL) // 1 hours
						{
							it.second->SetUnreachable (true);
							total--;
						}
					}
					else if (total > 300)
					{
						if (ts > it.second->GetTimestamp () + 30*3600*1000LL) // 30 hours
						{
							it.second->SetUnreachable (true);
							total--;
						}
					}
					else if (total > 120)
					{
						if (ts > it.second->GetTimestamp () + 72*3600*1000LL) // 72 hours
						{
							it.second->SetUnreachable (true);
							total--;
						}
					}
				}

				if (it.second->IsUnreachable ())
				{
					total--;
					// delete RI file
					if (boost::filesystem::exists (GetFilePath (fullDirectory, it.second.get ())))
					{
						boost::filesystem::remove (GetFilePath (fullDirectory, it.second.get ()));
						deletedCount++;
					}
					// delete from floodfills list
					if (it.second->IsFloodfill ())
					{
						std::unique_lock<std::mutex> l(m_FloodfillsMutex);
						m_Floodfills.remove (it.second);
					}
				}
			}
		}
		if (count > 0)
			LogPrint (count," new/updated routers saved");
		if (deletedCount > 0)
		{
			LogPrint (deletedCount," routers deleted");
			// clean up RouterInfos table
			std::unique_lock<std::mutex> l(m_RouterInfosMutex);
			for (auto it = m_RouterInfos.begin (); it != m_RouterInfos.end ();)
			{
				if (it->second->IsUnreachable ())
				{
					it->second->SaveProfile ();
					it = m_RouterInfos.erase (it);
				}
				else
					it++;
			}
		}
	}

	void NetDb::RequestDestination (const IdentHash& destination, RequestedDestination::RequestComplete requestComplete)
	{
		auto dest = m_Requests.CreateRequest (destination, false, requestComplete); // non-exploratory
		if (!dest)
		{
			LogPrint (eLogWarning, "Destination ", destination.ToBase64(), " is requested already");
			return;
		}

		auto floodfill = GetClosestFloodfill (destination, dest->GetExcludedPeers ());
		if (floodfill)
			transports.SendMessage (floodfill->GetIdentHash (), dest->CreateRequestMessage (floodfill->GetIdentHash ()));
		else
		{
			LogPrint (eLogError, "No floodfills found");
			m_Requests.RequestComplete (destination, nullptr);
		}
	}

	void NetDb::HandleDatabaseStoreMsg (std::shared_ptr<const I2NPMessage> m)
	{
		const uint8_t * buf = m->GetPayload ();
		size_t len = m->GetSize ();
		IdentHash ident (buf + DATABASE_STORE_KEY_OFFSET);
		if (ident.IsZero ())
		{
			LogPrint (eLogError, "Database store with zero ident. Dropped");
			return;
		}
		uint32_t replyToken = bufbe32toh (buf + DATABASE_STORE_REPLY_TOKEN_OFFSET);
		size_t offset = DATABASE_STORE_HEADER_SIZE;
		if (replyToken)
		{
			auto deliveryStatus = CreateDeliveryStatusMsg (replyToken);
			uint32_t tunnelID = bufbe32toh (buf + offset);
			offset += 4;
			if (!tunnelID) // send response directly
				transports.SendMessage (buf + offset, deliveryStatus);
			else
			{
				auto pool = i2p::tunnel::tunnels.GetExploratoryPool ();
				auto outbound = pool ? pool->GetNextOutboundTunnel () : nullptr;
				if (outbound)
					outbound->SendTunnelDataMsg (buf + offset, tunnelID, deliveryStatus);
				else
					LogPrint (eLogError, "No outbound tunnels for DatabaseStore reply found");
			}
			offset += 32;

			if (context.IsFloodfill ())
			{
				// flood it
				auto floodMsg = ToSharedI2NPMessage (NewI2NPShortMessage ());
				uint8_t * payload = floodMsg->GetPayload ();
				memcpy (payload, buf, 33); // key + type
				htobe32buf (payload + DATABASE_STORE_REPLY_TOKEN_OFFSET, 0); // zero reply token
				memcpy (payload + DATABASE_STORE_HEADER_SIZE, buf + offset, len - offset);
				floodMsg->len += DATABASE_STORE_HEADER_SIZE + len -offset;
				floodMsg->FillI2NPMessageHeader (eI2NPDatabaseStore);
				std::set<IdentHash> excluded;
				for (int i = 0; i < 3; i++)
				{
					auto floodfill = GetClosestFloodfill (ident, excluded);
					if (floodfill)
						transports.SendMessage (floodfill->GetIdentHash (), floodMsg);
				}
			}
		}

		if (buf[DATABASE_STORE_TYPE_OFFSET]) // type
		{
			LogPrint ("LeaseSet");
			AddLeaseSet (ident, buf + offset, len - offset, m->from);
		}
		else
		{
			LogPrint ("RouterInfo");
			size_t size = bufbe16toh (buf + offset);
			offset += 2;
			if (size > 2048 || size > len - offset)
			{
				LogPrint ("Invalid RouterInfo length ", (int)size);
				return;
			}
			try
			{
				CryptoPP::Gunzip decompressor;
				decompressor.Put (buf + offset, size);
				decompressor.MessageEnd();
				uint8_t uncompressed[2048];
				size_t uncomressedSize = decompressor.MaxRetrievable ();
				if (uncomressedSize <= 2048)
				{
					decompressor.Get (uncompressed, uncomressedSize);
					AddRouterInfo (ident, uncompressed, uncomressedSize);
				}
				else
					LogPrint ("Invalid RouterInfo uncomressed length ", (int)uncomressedSize);
			}
			catch (CryptoPP::Exception& ex)
			{
				LogPrint (eLogError, "DatabaseStore: ", ex.what ());
			}
		}
	}

	void NetDb::HandleDatabaseSearchReplyMsg (std::shared_ptr<const I2NPMessage> msg)
	{
		const uint8_t * buf = msg->GetPayload ();
		char key[48];
		int l = i2p::data::ByteStreamToBase64 (buf, 32, key, 48);
		key[l] = 0;
		int num = buf[32]; // num
		LogPrint ("DatabaseSearchReply for ", key, " num=", num);
		IdentHash ident (buf);
		auto dest = m_Requests.FindRequest (ident);
		if (dest)
		{
			bool deleteDest = true;
			if (num > 0)
			{
				auto pool = i2p::tunnel::tunnels.GetExploratoryPool ();
				auto outbound = pool ? pool->GetNextOutboundTunnel () : nullptr;
				auto inbound = pool ? pool->GetNextInboundTunnel () : nullptr;
				if (!dest->IsExploratory ())
				{
					// reply to our destination. Try other floodfills
					if (outbound && inbound )
					{
						std::vector<i2p::tunnel::TunnelMessageBlock> msgs;
						auto count = dest->GetExcludedPeers ().size ();
						if (count < 7)
						{
							auto nextFloodfill = GetClosestFloodfill (dest->GetDestination (), dest->GetExcludedPeers ());
							if (nextFloodfill)
							{
								// tell floodfill about us
								msgs.push_back (i2p::tunnel::TunnelMessageBlock
									{
										i2p::tunnel::eDeliveryTypeRouter,
										nextFloodfill->GetIdentHash (), 0,
										ToSharedI2NPMessage (CreateDatabaseStoreMsg ())
									});

								// request destination
								LogPrint ("Try ", key, " at ", count, " floodfill ", nextFloodfill->GetIdentHash ().ToBase64 ());
								auto msg = dest->CreateRequestMessage (nextFloodfill, inbound);
								msgs.push_back (i2p::tunnel::TunnelMessageBlock
									{
										i2p::tunnel::eDeliveryTypeRouter,
										nextFloodfill->GetIdentHash (), 0, msg
									});
								deleteDest = false;
							}
						}
						else
							LogPrint (key, " was not found on 7 floodfills");

						if (msgs.size () > 0)
							outbound->SendTunnelDataMsg (msgs);
					}
				}

				if (deleteDest)
					// no more requests for the destinationation. delete it
					m_Requests.RequestComplete (ident, nullptr);
			}
			else
				// no more requests for detination possible. delete it
				m_Requests.RequestComplete (ident, nullptr);
		}
		else
			LogPrint ("Requested destination for ", key, " not found");

		// try responses
		for (int i = 0; i < num; i++)
		{
			const uint8_t * router = buf + 33 + i*32;
			char peerHash[48];
			int l1 = i2p::data::ByteStreamToBase64 (router, 32, peerHash, 48);
			peerHash[l1] = 0;
			LogPrint (i,": ", peerHash);

			auto r = FindRouter (router);
			if (!r || i2p::util::GetMillisecondsSinceEpoch () > r->GetTimestamp () + 3600*1000LL)
			{
				// router with ident not found or too old (1 hour)
				LogPrint ("Found new/outdated router. Requesting RouterInfo ...");
				RequestDestination (router);
			}
			else
				LogPrint ("Bayan");
		}
	}

	void NetDb::HandleDatabaseLookupMsg (std::shared_ptr<const I2NPMessage> msg)
	{
		const uint8_t * buf = msg->GetPayload ();
		IdentHash ident (buf);
		if (ident.IsZero ())
		{
			LogPrint (eLogError, "DatabaseLookup for zero ident. Ignored");
			return;
		}
		char key[48];
		int l = i2p::data::ByteStreamToBase64 (buf, 32, key, 48);
		key[l] = 0;
		uint8_t flag = buf[64];
		LogPrint ("DatabaseLookup for ", key, " recieved flags=", (int)flag);
		uint8_t lookupType = flag & DATABASE_LOOKUP_TYPE_FLAGS_MASK;
		const uint8_t * excluded = buf + 65;
		uint32_t replyTunnelID = 0;
		if (flag & DATABASE_LOOKUP_DELIVERY_FLAG) //reply to tunnel
		{
			replyTunnelID = bufbe32toh (buf + 64);
			excluded += 4;
		}
		uint16_t numExcluded = bufbe16toh (excluded);
		excluded += 2;
		if (numExcluded > 512)
		{
			LogPrint ("Number of excluded peers", numExcluded, " exceeds 512");
			numExcluded = 0; // TODO:
		}

		std::shared_ptr<I2NPMessage> replyMsg;
		if (lookupType == DATABASE_LOOKUP_TYPE_EXPLORATORY_LOOKUP)
		{
			LogPrint ("Exploratory close to  ", key, " ", numExcluded, " excluded");
			std::set<IdentHash> excludedRouters;
			for (int i = 0; i < numExcluded; i++)
			{
				excludedRouters.insert (excluded);
				excluded += 32;
			}
			std::vector<IdentHash> routers;
			for (int i = 0; i < 3; i++)
			{
				auto r = GetClosestNonFloodfill (ident, excludedRouters);
				if (r)
				{
					routers.push_back (r->GetIdentHash ());
					excludedRouters.insert (r->GetIdentHash ());
				}
			}
			replyMsg = ToSharedI2NPMessage (CreateDatabaseSearchReply (ident, routers));
		}
		else
		{
			if (lookupType == DATABASE_LOOKUP_TYPE_ROUTERINFO_LOOKUP  ||
			    lookupType == DATABASE_LOOKUP_TYPE_NORMAL_LOOKUP)
			{
				auto router = FindRouter (ident);
				if (router)
				{
					LogPrint ("Requested RouterInfo ", key, " found");
					router->LoadBuffer ();
					if (router->GetBuffer ())
						replyMsg = ToSharedI2NPMessage (CreateDatabaseStoreMsg (router));
				}
			}

			if (!replyMsg && (lookupType == DATABASE_LOOKUP_TYPE_LEASESET_LOOKUP  ||
			    lookupType == DATABASE_LOOKUP_TYPE_NORMAL_LOOKUP))
			{
				auto leaseSet = FindLeaseSet (ident);
				if (leaseSet) // we don't send back our LeaseSets
				{
					LogPrint ("Requested LeaseSet ", key, " found");
					replyMsg = ToSharedI2NPMessage (CreateDatabaseStoreMsg (leaseSet));
				}
			}

			if (!replyMsg)
			{
				LogPrint ("Requested ", key, " not found. ", numExcluded, " excluded");
				std::set<IdentHash> excludedRouters;
				for (int i = 0; i < numExcluded; i++)
				{
					excludedRouters.insert (excluded);
					excluded += 32;
				}
				replyMsg = ToSharedI2NPMessage (CreateDatabaseSearchReply (ident, GetClosestFloodfills (ident, 3, excludedRouters)));
			}
		}

		if (replyMsg)
		{
			if (replyTunnelID)
			{
				// encryption might be used though tunnel only
				if (flag & DATABASE_LOOKUP_ENCYPTION_FLAG) // encrypted reply requested
				{
					const uint8_t * sessionKey = excluded;
					uint8_t numTags = sessionKey[32];
					if (numTags > 0)
					{
						const uint8_t * sessionTag = sessionKey + 33; // take first tag
						i2p::garlic::GarlicRoutingSession garlic (sessionKey, sessionTag);
						replyMsg = garlic.WrapSingleMessage (replyMsg);
					}
				}
				auto exploratoryPool = i2p::tunnel::tunnels.GetExploratoryPool ();
				auto outbound = exploratoryPool ? exploratoryPool->GetNextOutboundTunnel () : nullptr;
				if (outbound)
					outbound->SendTunnelDataMsg (buf+32, replyTunnelID, replyMsg);
				else
					transports.SendMessage (buf+32, i2p::CreateTunnelGatewayMsg (replyTunnelID, replyMsg));
			}
			else
				transports.SendMessage (buf+32, replyMsg);
		}
	}

	void NetDb::Explore (int numDestinations)
	{
		// new requests
		auto exploratoryPool = i2p::tunnel::tunnels.GetExploratoryPool ();
		auto outbound = exploratoryPool ? exploratoryPool->GetNextOutboundTunnel () : nullptr;
		auto inbound = exploratoryPool ? exploratoryPool->GetNextInboundTunnel () : nullptr;
		bool throughTunnels = outbound && inbound;

		CryptoPP::RandomNumberGenerator& rnd = i2p::context.GetRandomNumberGenerator ();
		uint8_t randomHash[32];
		std::vector<i2p::tunnel::TunnelMessageBlock> msgs;
		std::set<const RouterInfo *> floodfills;
		LogPrint ("Exploring new ", numDestinations, " routers ...");
		for (int i = 0; i < numDestinations; i++)
		{
			rnd.GenerateBlock (randomHash, 32);
			auto dest = m_Requests.CreateRequest (randomHash, true); // exploratory
			if (!dest)
			{
				LogPrint (eLogWarning, "Exploratory destination is requested already");
				return;
			}
			auto floodfill = GetClosestFloodfill (randomHash, dest->GetExcludedPeers ());
			if (floodfill && !floodfills.count (floodfill.get ())) // request floodfill only once
			{
				floodfills.insert (floodfill.get ());
				if (i2p::transport::transports.IsConnected (floodfill->GetIdentHash ()))
					throughTunnels = false;
				if (throughTunnels)
				{
					msgs.push_back (i2p::tunnel::TunnelMessageBlock
						{
							i2p::tunnel::eDeliveryTypeRouter,
							floodfill->GetIdentHash (), 0,
							ToSharedI2NPMessage (CreateDatabaseStoreMsg ()) // tell floodfill about us
						});
					msgs.push_back (i2p::tunnel::TunnelMessageBlock
						{
							i2p::tunnel::eDeliveryTypeRouter,
							floodfill->GetIdentHash (), 0,
							dest->CreateRequestMessage (floodfill, inbound) // explore
						});
				}
				else
					i2p::transport::transports.SendMessage (floodfill->GetIdentHash (), dest->CreateRequestMessage (floodfill->GetIdentHash ()));
			}
			else
				m_Requests.RequestComplete (randomHash, nullptr);
		}
		if (throughTunnels && msgs.size () > 0)
			outbound->SendTunnelDataMsg (msgs);
	}

	void NetDb::Publish ()
	{
		std::set<IdentHash> excluded; // TODO: fill up later
		for (int i = 0; i < 2; i++)
		{
			auto floodfill = GetClosestFloodfill (i2p::context.GetRouterInfo ().GetIdentHash (), excluded);
			if (floodfill)
			{
				uint32_t replyToken = i2p::context.GetRandomNumberGenerator ().GenerateWord32 ();
				LogPrint ("Publishing our RouterInfo to ", floodfill->GetIdentHashAbbreviation (), ". reply token=", replyToken);
				transports.SendMessage (floodfill->GetIdentHash (), ToSharedI2NPMessage (CreateDatabaseStoreMsg (i2p::context.GetSharedRouterInfo (), replyToken)));
				excluded.insert (floodfill->GetIdentHash ());
			}
		}
	}

	std::shared_ptr<const RouterInfo> NetDb::GetRandomRouter () const
	{
		return GetRandomRouter (
			[](std::shared_ptr<const RouterInfo> router)->bool
			{
				return !router->IsHidden ();
			});
	}

	std::shared_ptr<const RouterInfo> NetDb::GetRandomRouter (std::shared_ptr<const RouterInfo> compatibleWith) const
	{
		return GetRandomRouter (
			[compatibleWith](std::shared_ptr<const RouterInfo> router)->bool
			{
				return !router->IsHidden () && router != compatibleWith &&
					router->IsCompatible (*compatibleWith);
			});
	}

	std::shared_ptr<const RouterInfo> NetDb::GetRandomPeerTestRouter () const
	{
		return GetRandomRouter (
			[](std::shared_ptr<const RouterInfo> router)->bool
			{
				return !router->IsHidden () && router->IsPeerTesting ();
			});
	}

	std::shared_ptr<const RouterInfo> NetDb::GetRandomIntroducer () const
	{
		return GetRandomRouter (
			[](std::shared_ptr<const RouterInfo> router)->bool
			{
				return !router->IsHidden () && router->IsIntroducer ();
			});
	}

	std::shared_ptr<const RouterInfo> NetDb::GetHighBandwidthRandomRouter (std::shared_ptr<const RouterInfo> compatibleWith) const
	{
		return GetRandomRouter (
			[compatibleWith](std::shared_ptr<const RouterInfo> router)->bool
			{
				return !router->IsHidden () && router != compatibleWith &&
					router->IsCompatible (*compatibleWith) &&
					(router->GetCaps () & RouterInfo::eHighBandwidth);
			});
	}

	template<typename Filter>
	std::shared_ptr<const RouterInfo> NetDb::GetRandomRouter (Filter filter) const
	{
		CryptoPP::RandomNumberGenerator& rnd = i2p::context.GetRandomNumberGenerator ();
		uint32_t ind = rnd.GenerateWord32 (0, m_RouterInfos.size () - 1);
		for (int j = 0; j < 2; j++)
		{
			uint32_t i = 0;
			std::unique_lock<std::mutex> l(m_RouterInfosMutex);
			for (auto it: m_RouterInfos)
			{
				if (i >= ind)
				{
					if (!it.second->IsUnreachable () && filter (it.second))
						return it.second;
				}
				else
					i++;
			}
			// we couldn't find anything, try second pass
			ind = 0;
		}
		return nullptr; // seems we have too few routers
	}

	void NetDb::PostI2NPMsg (std::shared_ptr<const I2NPMessage> msg)
	{
		if (msg) m_Queue.Put (msg);
	}

	std::shared_ptr<const RouterInfo> NetDb::GetClosestFloodfill (const IdentHash& destination,
		const std::set<IdentHash>& excluded) const
	{
		std::shared_ptr<const RouterInfo> r;
		XORMetric minMetric;
		IdentHash destKey = CreateRoutingKey (destination);
		minMetric.SetMax ();
		std::unique_lock<std::mutex> l(m_FloodfillsMutex);
		for (auto it: m_Floodfills)
		{
			if (!it->IsUnreachable ())
			{
				XORMetric m = destKey ^ it->GetIdentHash ();
				if (m < minMetric && !excluded.count (it->GetIdentHash ()))
				{
					minMetric = m;
					r = it;
				}
			}
		}
		return r;
	}

	std::vector<IdentHash> NetDb::GetClosestFloodfills (const IdentHash& destination, size_t num,
		std::set<IdentHash>& excluded) const
	{
		struct Sorted
		{
			std::shared_ptr<const RouterInfo> r;
			XORMetric metric;
			bool operator< (const Sorted& other) const { return metric < other.metric; };
		};

		std::set<Sorted> sorted;
		IdentHash destKey = CreateRoutingKey (destination);
		{
			std::unique_lock<std::mutex> l(m_FloodfillsMutex);
			for (auto it: m_Floodfills)
			{
				if (!it->IsUnreachable ())
				{
					XORMetric m = destKey ^ it->GetIdentHash ();
					if (sorted.size () < num)
						sorted.insert ({it, m});
					else if (m < sorted.rbegin ()->metric)
					{
						sorted.insert ({it, m});
						sorted.erase (std::prev (sorted.end ()));
					}
				}
			}
		}

		std::vector<IdentHash> res;
		size_t i = 0;
		for (auto it: sorted)
		{
			if (i < num)
			{
				auto& ident = it.r->GetIdentHash ();
				if (!excluded.count (ident))
				{
					res.push_back (ident);
					i++;
				}
			}
			else
				break;
		}
		return res;
	}

	std::shared_ptr<const RouterInfo> NetDb::GetClosestNonFloodfill (const IdentHash& destination,
		const std::set<IdentHash>& excluded) const
	{
		std::shared_ptr<const RouterInfo> r;
		XORMetric minMetric;
		IdentHash destKey = CreateRoutingKey (destination);
		minMetric.SetMax ();
		// must be called from NetDb thread only
		for (auto it: m_RouterInfos)
		{
			if (!it.second->IsFloodfill ())
			{
				XORMetric m = destKey ^ it.first;
				if (m < minMetric && !excluded.count (it.first))
				{
					minMetric = m;
					r = it.second;
				}
			}
		}
		return r;
	}

	void NetDb::ManageLeaseSets ()
	{
		for (auto it = m_LeaseSets.begin (); it != m_LeaseSets.end ();)
		{
			if (!it->second->HasNonExpiredLeases ()) // all leases expired
			{
				LogPrint ("LeaseSet ", it->second->GetIdentHash ().ToBase64 (), " expired");
				it = m_LeaseSets.erase (it);
			}
			else
				it++;
		}
	}
}
}


text:
// [[Rcpp::depends(BH)]]

#include <Rcpp.h>
#include <boost/algorithm/string.hpp>
#include <boost/geometry.hpp>

#include "wkt.h"
#include "googlePolylines.h"
#include "variants.h"

using namespace Rcpp;
namespace bg = boost::geometry;
namespace bgm = bg::model;

void addLonLatToWKTStream(std::ostringstream& os, float lon, float lat ) {
  os << std::to_string(lon) << " " << std::to_string(lat);
}

void geom_type(const char *cls, int *tp = NULL) {

  int type = 0;
  if (strcmp(cls, "POINT") == 0)
    type = POINT;
  else if (strcmp(cls, "MULTIPOINT") == 0)
    type = MULTIPOINT;
  else if (strcmp(cls, "LINESTRING") == 0)
    type = LINESTRING;
  else if (strcmp(cls, "POLYGON") == 0)
    type = POLYGON;
  else if (strcmp(cls, "MULTILINESTRING") == 0)
    type = MULTILINESTRING;
  else if (strcmp(cls, "MULTIPOLYGON") == 0)
    type = MULTIPOLYGON;
  else
    type = UNKNOWN;
  if (tp != NULL)
    *tp = type;
}

void beginWKT(std::ostringstream& os, Rcpp::CharacterVector cls) {

  int tp;
  geom_type(cls[1], &tp);
  //geom_type(cls[0], &tp);  // iff removing XY and sfg attributs

  switch( tp ) {
  case POINT:
    os << "POINT ";
    break;
  case MULTIPOINT:
    os << "MULTIPOINT (";
    break;
  case LINESTRING:
    os << "LINESTRING ";
    break;
  case MULTILINESTRING:
    os << "MULTILINESTRING (";
    break;
  case POLYGON:
    os << "POLYGON (";
    break;
  case MULTIPOLYGON:
    os << "MULTIPOLYGON ((";
    break;
  default: {
      Rcpp::stop("Unknown geometry type");
    }
  }
}

void endWKT(std::ostringstream& os, Rcpp::CharacterVector cls) {

  int tp;
  geom_type(cls[1], &tp);
  //geom_type(cls[0], &tp);

  switch( tp ) {
  case POINT:
    os << "";
    break;
  case MULTIPOINT:
    os << ")";
    break;
  case LINESTRING:
    os << "";
    break;
  case MULTILINESTRING:
    os << ")";
    break;
  case POLYGON:
    os << ")";
    break;
  case MULTIPOLYGON:
    os << "))";
    break;
  default: {
      Rcpp::stop("Unknown geometry type");
    }
  }
}

void coordSeparateWKT(std::ostringstream& os) {
  os << ", ";
}

// [[Rcpp::export]]
Rcpp::StringVector rcpp_polyline_to_wkt(Rcpp::List sfencoded) {

  unsigned int nrow = sfencoded.size();
  Rcpp::StringVector res(nrow);
  std::string stdspl;
  Rcpp::CharacterVector cls;
  Rcpp::StringVector pl;
  Rcpp::String spl;

  for (size_t i = 0; i < nrow; i++ ){

    std::ostringstream os;
    stdspl.clear();

    pl = sfencoded[i];

    if(!Rf_isNull(pl.attr("sfc"))){
      cls = pl.attr("sfc");
    }else{
      Rcpp::stop("No geometry attribute found");
    }

    beginWKT(os, cls);
    unsigned int n =  pl.size();

    for(size_t j = 0; j < n; j ++ ) {

      spl = pl[j];

      if(spl == SPLIT_CHAR){
        os << "),(";
      }else{
        stdspl = spl;
        os << "(";
        polylineToWKT(os, stdspl);
        os << ")";
        if(n > 1 && j < (n - 1)){
          if(pl[j+1] != SPLIT_CHAR){
            os << ",";
          }
        }
      }

    }
    endWKT(os, cls);
    res[i] = os.str();
  }

  return res;

}

void polylineToWKT(std::ostringstream& os, std::string encoded){

  int len = encoded.size();
  int index = 0;
  float lat = 0;
  float lng = 0;

  while (index < len){
    char b;
    unsigned int shift = 0;
    int result = 0;
    do {
      b = encoded.at(index++) - 63;
      result |= (b & 0x1f) << shift;
      shift += 5;
    } while (b >= 0x20);
    float dlat = ((result & 1) ? ~(result >> 1) : (result >> 1));
    lat += dlat;

    shift = 0;
    result = 0;
    do {
      b = encoded.at(index++) - 63;
      result |= (b & 0x1f) << shift;
      shift += 5;
    } while (b >= 0x20);
    float dlng = ((result & 1) ? ~(result >> 1) : (result >> 1));
    lng += dlng;

    addLonLatToWKTStream(os, lng* (float)1e-5, lat* (float)1e-5);

    if(index < len) {
      coordSeparateWKT(os);
    }
  }
}

/*
void ReplaceStringInPlace(std::string& subject, const std::string& search,
                          const std::string& replace) {
  size_t pos = 0;
  while ((pos = subject.find(search, pos)) != std::string::npos) {
    subject.replace(pos, search.length(), replace);
    pos += replace.length();
  }
}
*/

/**
 * Finds the 'GEOMETRY' text
 */
std::string geomFromWKT(std::string& pl) {
  size_t s = pl.find_first_of("(");
  std::string geom = pl.substr(0, s);
  boost::trim(geom);
//  pl.replace(0, s, "");

  return geom;
}

template <typename Point>
void encode_wkt_point(Point const& p, std::ostringstream& os) {

  global_vars::lons.clear();
  global_vars::lats.clear();

  global_vars::lons.push_back(bg::get<0>(p));
  global_vars::lats.push_back(bg::get<1>(p));

  global_vars::encodedString = encode_polyline();

  addToStream(os);
}

template <typename MultiPoint>
void encode_wkt_multipoint(MultiPoint const& mp, std::ostringstream& os) {

  typedef typename boost::range_iterator
  <
    multi_point_type const
  >::type iterator_type;

  for (iterator_type it = boost::begin(mp);
       it != boost::end(mp);
       ++it)
  {
    encode_wkt_point(*it, os);
  }

}

template <typename LineString>
void encode_wkt_linestring(LineString const& ls, std::ostringstream& os) {

  // works for RINGS (because it's templated)

  typedef typename boost::range_iterator
    <
      linestring_type const
    >::type iterator_type;

  int i = 0;
  for (iterator_type it = boost::begin(ls);
       it != boost::end(ls);
       ++it)
  {
    global_vars::lons.push_back(bg::get<0>(*it));
    global_vars::lats.push_back(bg::get<1>(*it));
    i++;
  }
  global_vars::encodedString = encode_polyline();
  addToStream(os);
}

template <typename MultiLinestring>
void encode_wkt_multi_linestring(MultiLinestring const& mls, std::ostringstream& os) {

  typedef typename boost::range_iterator
  <
    multi_linestring_type const
  >::type iterator_type;

  for (iterator_type it = boost::begin(mls);
       it != boost::end(mls);
       ++it)
  {
    encode_wkt_linestring(*it, os);
  }
}

template <typename Polygon>
void encode_wkt_polygon(Polygon const& pl, std::ostringstream& os) {

  encode_wkt_linestring(pl.outer(), os);

  // perhaps use
  // https://stackoverflow.com/questions/7722087/getting-the-coordinates-of-points-from-a-boost-geometry-polygon/7748091#7748091
  // https://stackoverflow.com/questions/37071031/how-to-get-a-polygon-from-boostgeometrymodelpolygon
  for (auto& i: pl.inners() ) {
    encode_wkt_linestring(i, os);
  }
}

template <typename MultiPolygon>
void encode_wkt_multi_polygon(MultiPolygon const& mpl, std::ostringstream& os) {

  typedef typename boost::range_iterator
  <
    multi_polygon_type const
  >::type iterator_type;

  global_vars::encodedString = SPLIT_CHAR;
  for (iterator_type it = boost::begin(mpl);
       it != boost::end(mpl);
       ++it)
  {
    encode_wkt_polygon(*it, os);
    addToStream(os);
  }
}

// [[Rcpp::export]]
Rcpp::List rcpp_wkt_to_polyline(Rcpp::StringVector wkt) {

  size_t n = wkt.length();
  Rcpp::String r_wkt;
  std::string str_wkt;
  std::string geomType;
  std::vector<std::string> cls;
  Rcpp::CharacterVector sv;

  Rcpp::List resultPolylines(n);
  int lastItem;
  unsigned int i;

  for (i = 0; i < n; i++ ) {

    std::ostringstream os;

    r_wkt = wkt[i];
    str_wkt = r_wkt;
    geomType = geomFromWKT(str_wkt);

    cls.clear();
    cls.push_back("XY");
    cls.push_back(geomType);
    cls.push_back("sfg");

    //Rcpp::CharacterVector cls(1);
    //cls[0] = geomType;

    if (geomType == "POINT" ) {

      point_type pt;
      bg::read_wkt(str_wkt, pt);
      encode_wkt_point(pt, os);

    }else if (geomType == "MULTIPOINT" ) {

      multi_point_type mp;
      bg::read_wkt(str_wkt, mp);
      encode_wkt_multipoint(mp, os);

    }else if (geomType == "LINESTRING" ) {

      linestring_type ls;
      bg::read_wkt(str_wkt, ls);
      encode_wkt_linestring(ls, os);

    }else if (geomType == "MULTILINESTRING" ) {

      multi_linestring_type mls;
      bg::read_wkt(str_wkt, mls);
      encode_wkt_multi_linestring(mls, os);

    }else if (geomType == "POLYGON" ) {

      polygon_type pl;
      bg::read_wkt(str_wkt, pl);
      encode_wkt_polygon(pl, os);

    }else if (geomType == "MULTIPOLYGON" ) {

      multi_polygon_type mpl;
      bg::read_wkt(str_wkt, mpl);
      encode_wkt_multi_polygon(mpl, os);
    }

    std::string str = os.str();
    split(str, ' ');

    if(global_vars::elems.size() > 1) {
      lastItem = global_vars::elems.size() - 1;

      if (global_vars::elems[lastItem] == "-") {
        global_vars::elems.erase(global_vars::elems.end() - 1);
      }
    }

    sv = wrap(global_vars::elems);
    sv.attr("sfc") = cls;
    resultPolylines[i] = sv;
  }

  return resultPolylines;
}

max_stars_repo_name: SymbolixAU/googlePolylines

max_issues_repo_name: SymbolixAU/googlePolylines

max_forks_repo_name: SymbolixAU/googlePolylines