<commit_before>/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */


#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <fcntl.h>

#include <arpa/inet.h>
#include <sys/ioctl.h>
#include <net/if.h>
#include <net/if_arp.h>
#include <sstream>

#include "LightNetwork.h"
#include "paddle/utils/Util.h"
#include "paddle/utils/StringUtil.h"
#include "RDMANetwork.h"

/// quick ack can reduce the latency of small message
P_DEFINE_bool(small_messages, false,
              "if message size is small, recommend set it True to enable quick "
              "ack and no delay");

/// reasonable sock_send_buf_size can control the traffic injected into switch
/// network. Injecting too many data into traffic could cause packets loss which
/// cause long latency and degrade the efficiency of communication.
P_DEFINE_int32(sock_send_buf_size, 1024 * 1024 * 40,
               "restrict sock send buff size, can reduce network congestion if "
               "set carefully");

/// reasonable size can hold bursted packets and reduce packets loss
P_DEFINE_int32(sock_recv_buf_size, 1024 * 1024 * 40,
               "restrict sock recv buff size");

namespace paddle {

/**
 * @brief get ip address from interface name
 *
 * @param[in] device device interface name
 */
std::string getIpAddr(std::string &device) {
  int sock;
  struct sockaddr_in sin;
  struct ifreq ifr;

  sock = socket(AF_INET, SOCK_DGRAM, 0);
  CHECK(sock >= 0) << "Create socket error.";

  strncpy(ifr.ifr_name, device.c_str(), IFNAMSIZ);
  ifr.ifr_name[IFNAMSIZ - 1] = 0;

  CHECK_GE(ioctl(sock, SIOCGIFADDR, &ifr), 0);
  memcpy(&sin, &ifr.ifr_addr, sizeof(sin));
  close(sock);
  return std::string(inet_ntoa(sin.sin_addr));
}

/**
 * @brief set sock option
 *
 * @param[in] sockfd sock file descriptor
 *
 * @note adjust some default sock option for better performance
 */
void setOption(int sockfd) {
  int sendSize = FLAGS_sock_send_buf_size;
  int recvSize = FLAGS_sock_recv_buf_size;
  CHECK_GE(
      setsockopt(sockfd, SOL_SOCKET, SO_RCVBUF, &recvSize, sizeof(recvSize)),
      0);
  CHECK_GE(
      setsockopt(sockfd, SOL_SOCKET, SO_SNDBUF, &sendSize, sizeof(sendSize)),
      0);
  if (FLAGS_small_messages) {<commit_msg>auto-tuning SND/REV buff size on MAC OS<commit_after>/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */


#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <fcntl.h>

#include <arpa/inet.h>
#include <sys/ioctl.h>
#include <net/if.h>
#include <net/if_arp.h>
#include <sstream>

#include "LightNetwork.h"
#include "paddle/utils/Util.h"
#include "paddle/utils/StringUtil.h"
#include "RDMANetwork.h"

/// quick ack can reduce the latency of small message
P_DEFINE_bool(small_messages, false,
              "if message size is small, recommend set it True to enable quick "
              "ack and no delay");

/// reasonable sock_send_buf_size can control the traffic injected into switch
/// network. Injecting too many data into traffic could cause packets loss which
/// cause long latency and degrade the efficiency of communication.
P_DEFINE_int32(sock_send_buf_size, 1024 * 1024 * 40,
               "restrict sock send buff size, can reduce network congestion if "
               "set carefully");

/// reasonable size can hold bursted packets and reduce packets loss
P_DEFINE_int32(sock_recv_buf_size, 1024 * 1024 * 40,
               "restrict sock recv buff size");

namespace paddle {

/**
 * @brief get ip address from interface name
 *
 * @param[in] device device interface name
 */
std::string getIpAddr(std::string &device) {
  int sock;
  struct sockaddr_in sin;
  struct ifreq ifr;

  sock = socket(AF_INET, SOCK_DGRAM, 0);
  CHECK(sock >= 0) << "Create socket error.";

  strncpy(ifr.ifr_name, device.c_str(), IFNAMSIZ);
  ifr.ifr_name[IFNAMSIZ - 1] = 0;

  CHECK_GE(ioctl(sock, SIOCGIFADDR, &ifr), 0);
  memcpy(&sin, &ifr.ifr_addr, sizeof(sin));
  close(sock);
  return std::string(inet_ntoa(sin.sin_addr));
}

/**
 * @brief set sock option
 *
 * @param[in] sockfd sock file descriptor
 *
 * @note adjust some default sock option for better performance
 */
void setOption(int sockfd) {
#if !defined(__APPLE__) && !defined(__OSX__)
  int sendSize = FLAGS_sock_send_buf_size;
  int recvSize = FLAGS_sock_recv_buf_size;
  CHECK_GE(
      setsockopt(sockfd, SOL_SOCKET, SO_RCVBUF, &recvSize, sizeof(recvSize)),
      0);
  CHECK_GE(
      setsockopt(sockfd, SOL_SOCKET, SO_SNDBUF, &sendSize, sizeof(sendSize)),
      0);
#endif

  if (FLAGS_small_messages) {
{'fraction_non_alphanumeric': 0.10517028410093524, 'fraction_numerical': 0.013763896241397565, 'mean_word_length': 3.941586748038361, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5606086', 'n_tokens_mistral': 1909, 'n_tokens_neox': 1810, 'n_words': 614}