<commit_before>/// Copyright 2016 Pinterest Inc.
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0

/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.

//
// @author bol (<EMAIL>)
//

#pragma once

#include <algorithm>
#include <functional>
#include <map>
#include <memory>
#include <random>
#include <set>
#include <string>
#include <vector>
#include <unordered_map>
#include <utility>

#include "common/file_watcher.h"
#include "common/availability_zone.h"
#include "common/thrift_client_pool.h"
#include "folly/RWSpinLock.h"
#include "folly/SocketAddress.h"
#include "folly/ThreadLocal.h"

DECLARE_int32(min_client_reconnect_interval_seconds);
DECLARE_int64(client_connect_timeout_millis);

namespace common {

namespace detail {

enum class Role { MASTER, SLAVE, ANY };
enum class Quantity { ONE, TWO, ALL };
enum ReturnCode { OK, NOT_FOUND, BAD_HOST, UNKNOWN_SEGMENT, UNKNOWN_SHARD };
using ShardID = uint32_t;
using SegmentName = std::string;

struct Host {
  bool operator<(const Host& other) const {
    return addr < other.addr;
  }

  folly::SocketAddress addr;
  std::string az;  // availability zone
};

struct SegmentInfo {
  // there are totally shard_to_hosts.size() shards in this segment.
  // shard_to_hosts[i] contains all host info for shard i.
  // Host* refers to a host in ClusterLayout.all_hosts
  std::vector<std::vector<std::pair<const Host*, Role>>> shard_to_hosts;
};

struct ClusterLayout {
  std::map<SegmentName, SegmentInfo> segments;
  std::set<Host> all_hosts;
};

}  // namespace detail

/*
 * A router for sharded thrift services.
 * It returns thrift client objects based on the requested conditions.
 * All interfaces of ThriftRouter are thread safe.
 */
template <typename ClientType>
class ThriftRouter {
 public:
  using Role = detail::Role;
  using Quantity = detail::Quantity;
  using ReturnCode = detail::ReturnCode;
  using ShardID = detail::ShardID;
  using SegmentName = detail::SegmentName;
  using Host = detail::Host;
  using SegmentInfo = detail::SegmentInfo;
  using ClusterLayout = detail::ClusterLayout;

  /*
   * @param config_path  The config file path
   * @param parser       User provided parser to parse config file content into
   *                     an in memory READONLY ClusterLayout structure.
   */
  ThriftRouter(
      const std::string& local_az,
      const std::string& config_path,
      std::function<std::unique_ptr<const ClusterLayout>(std::string)> parser)
      : config_path_(config_path)
      , parser_(std::move(parser))
      , layout_rwlock_()
      , cluster_layout_()
      , local_client_map_(local_az) {
    CHECK(common::FileWatcher::Instance()->AddFile(
      config_path_,
      [this] (std::string content) {
        std::shared_ptr<const ClusterLayout> new_layout(
          parser_(std::move(content)));
        {
          folly::RWSpinLock::WriteHolder write_guard(layout_rwlock_);
          cluster_layout_.swap(new_layout);
        }
      }))
    << "Failed to watch " << config_path_;
  }
<commit_msg>[rocksplicator] log local az used by ThriftRouter<commit_after>/// Copyright 2016 Pinterest Inc.
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0

/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.

//
// @author bol (<EMAIL>)
//

#pragma once

#include <algorithm>
#include <functional>
#include <map>
#include <memory>
#include <random>
#include <set>
#include <string>
#include <vector>
#include <unordered_map>
#include <utility>

#include "common/file_watcher.h"
#include "common/availability_zone.h"
#include "common/thrift_client_pool.h"
#include "folly/RWSpinLock.h"
#include "folly/SocketAddress.h"
#include "folly/ThreadLocal.h"

DECLARE_int32(min_client_reconnect_interval_seconds);
DECLARE_int64(client_connect_timeout_millis);

namespace common {

namespace detail {

enum class Role { MASTER, SLAVE, ANY };
enum class Quantity { ONE, TWO, ALL };
enum ReturnCode { OK, NOT_FOUND, BAD_HOST, UNKNOWN_SEGMENT, UNKNOWN_SHARD };
using ShardID = uint32_t;
using SegmentName = std::string;

struct Host {
  bool operator<(const Host& other) const {
    return addr < other.addr;
  }

  folly::SocketAddress addr;
  std::string az;  // availability zone
};

struct SegmentInfo {
  // there are totally shard_to_hosts.size() shards in this segment.
  // shard_to_hosts[i] contains all host info for shard i.
  // Host* refers to a host in ClusterLayout.all_hosts
  std::vector<std::vector<std::pair<const Host*, Role>>> shard_to_hosts;
};

struct ClusterLayout {
  std::map<SegmentName, SegmentInfo> segments;
  std::set<Host> all_hosts;
};

}  // namespace detail

/*
 * A router for sharded thrift services.
 * It returns thrift client objects based on the requested conditions.
 * All interfaces of ThriftRouter are thread safe.
 */
template <typename ClientType>
class ThriftRouter {
 public:
  using Role = detail::Role;
  using Quantity = detail::Quantity;
  using ReturnCode = detail::ReturnCode;
  using ShardID = detail::ShardID;
  using SegmentName = detail::SegmentName;
  using Host = detail::Host;
  using SegmentInfo = detail::SegmentInfo;
  using ClusterLayout = detail::ClusterLayout;

  /*
   * @param config_path  The config file path
   * @param parser       User provided parser to parse config file content into
   *                     an in memory READONLY ClusterLayout structure.
   */
  ThriftRouter(
      const std::string& local_az,
      const std::string& config_path,
      std::function<std::unique_ptr<const ClusterLayout>(std::string)> parser)
      : config_path_(config_path)
      , parser_(std::move(parser))
      , layout_rwlock_()
      , cluster_layout_()
      , local_client_map_(local_az) {
    CHECK(common::FileWatcher::Instance()->AddFile(
      config_path_,
      [this] (std::string content) {
        std::shared_ptr<const ClusterLayout> new_layout(
          parser_(std::move(content)));
        {
          folly::RWSpinLock::WriteHolder write_guard(layout_rwlock_);
          cluster_layout_.swap(new_layout);
        }
      }))
    << "Failed to watch " << config_path_;
    LOG(INFO) << "Local AZ used by ThriftRouter: " << local_az;
  }

{'fraction_non_alphanumeric': 0.11269208878770631, 'fraction_numerical': 0.00398406374501992, 'mean_word_length': 3.8643598615916956, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6574859', 'n_tokens_mistral': 2159, 'n_tokens_neox': 2086, 'n_words': 637}