<commit_before>// Copyright © 2016 by <NAME> <<EMAIL>>
// Available under the MIT License. See LICENSE for details.

#include "base/fd.h"

#include <fcntl.h>
#include <sys/socket.h>
#include <sys/syscall.h>
#include <sys/types.h>
#include <unistd.h>

#include <cerrno>
#include <cstdlib>
#include <exception>

#include "base/logging.h"
#include "base/mutex.h"

namespace base {

static void invoke(FDHolder::HookFn hook) noexcept {
  try {
    hook();
  } catch (...) {
    LOG_EXCEPTION(std::current_exception());
  }
}

FDHolder::FDHolder(int fd) noexcept : fd_(fd) {
  VLOG(2) << "FDHolder: obtained ownership of fd " << fd;
}

FDHolder::~FDHolder() noexcept {
  int fd = release_internal(true);
  if (fd != -1) {
    VLOG(3) << "FDHolder::~FDHolder performing close of fd " << fd;
    ::close(fd);
  }
}

void FDHolder::on_close(HookFn hook) {
  auto lock = acquire_write(rwmu_);
  if (fd_ == -1) {
    lock.unlock();
    invoke(std::move(hook));
  } else {
    hooks_.push_back(std::move(hook));
  }
}

int FDHolder::release_internal(bool for_close) noexcept {
  auto lock = acquire_write(rwmu_);
  int fd = -1;
  std::swap(fd, fd_);
  std::vector<HookFn> hooks = std::move(hooks_);
  lock.unlock();
  VLOG(2) << "FDHolder: relinquished ownership of fd " << fd << ", "
          << "for_close=" << std::boolalpha << for_close;
  for (auto& hook : hooks) {
    invoke(std::move(hook));
  }
  return fd;
}

int FDHolder::release_fd() noexcept { return release_internal(false); }

Result FDHolder::close() {
  int fd = release_internal(true);
  Result r;
  int rc = ::close(fd);
  if (rc != 0) {
    int err_no = errno;
    r = Result::from_errno(err_no, "close(2)");
  }
  return r;
}

Result make_pipe(Pipe* out) {
  *DCHECK_NOTNULL(out) = Pipe();
  int fds[2] = {-1, -1};
  int rc = ::pipe2(fds, O_NONBLOCK | O_CLOEXEC);
  if (rc != 0) {
    int err_no = errno;
    return Result::from_errno(err_no, "pipe2(2)");
  }
  *out = Pipe(wrapfd(fds[0]), wrapfd(fds[1]));
  return Result();
}

Result make_socketpair(SocketPair* out, int domain, int type, int protocol) {
  *DCHECK_NOTNULL(out) = SocketPair();
  type |= SOCK_NONBLOCK;
  type |= SOCK_CLOEXEC;
  int fds[2] = {-1, -1};
  int rc = ::socketpair(domain, type, protocol, fds);
  if (rc != 0) {
    int err_no = errno;
    return Result::from_errno(err_no, "socketpair(2)");
  }
  *out = SocketPair(wrapfd(fds[0]), wrapfd(fds[1]));
  return Result();
}

Result set_blocking(FD fd, bool value) {
  auto pair = DCHECK_NOTNULL(fd)->acquire_fd();
  int flags = ::fcntl(pair.first, F_GETFL);
  if (flags == -1) {
    int err_no = errno;
    return Result::from_errno(err_no, "fcntl(2)");
  }
  if (value)
    flags &= ~O_NONBLOCK;
  else
    flags |= O_NONBLOCK;
  int n = ::fcntl(pair.first, F_SETFL, flags);
  if (n == -1) {
    int err_no = errno;
    return Result::from_errno(err_no, "fcntl(2)");
  }
  return Result();
}

Result shutdown(FD fd, int how) {
  auto pair = DCHECK_NOTNULL(fd)->acquire_fd();
  int rc = ::shutdown(pair.first, how);
  if (rc != 0) {
    int err_no = errno;
    return Result::from_errno(err_no, "shutdown(2)");
  }
  return Result();
}

Result seek(off_t* out, FD fd, off_t offset, int whence) {
  auto pair = DCHECK_NOTNULL(fd)->acquire_fd();
  off_t n = ::lseek(pair.first, offset, whence);
  if (n == static_cast<off_t>(-1)) {
    int err_no = errno;
    return Result::from_errno(err_no, "lseek(2)");
  }
  if (out) *out = n;
  return Result();
}

Result readdir_all(std::vector<DEntry>* out, FD fd, const char* what) {
  CHECK_NOTNULL(out);
  CHECK_NOTNULL(fd);

  std::vector<char> buf(4096);
  long nread;
  auto fdpair = fd->acquire_fd();
  while (true) {
    nread = syscall(SYS_getdents, fdpair.first, buf.data(), buf.size());
    if (nread < 0) {
      int err_no = errno;
      return Result::from_errno(err_no, "getdents(2) from ", what);
    }
    if (nread == 0) break;

    const char* ptr = buf.data();
    const char* end = ptr + nread;
    while (ptr != end) {
      unsigned long ino;
      unsigned short reclen;
      unsigned char type;
      ::memcpy(&ino, ptr, sizeof(unsigned long));
      ::memcpy(&reclen, ptr + 2 * sizeof(unsigned long),
               sizeof(unsigned short));
      type = *(ptr + reclen - 1);
      const char* p = ptr + 2 * sizeof(unsigned long) + sizeof(unsigned short);
      const char* q = ptr + reclen - 2;
      out->emplace_back(ino, type, std::string(p, q - p));
      ptr += reclen;
    }
  }
  return Result();<commit_msg>Deal with getdents(2) returning extra null bytes in the filename.<commit_after>// Copyright © 2016 by <NAME> <<EMAIL>>
// Available under the MIT License. See LICENSE for details.

#include "base/fd.h"

#include <fcntl.h>
#include <sys/socket.h>
#include <sys/syscall.h>
#include <sys/types.h>
#include <unistd.h>

#include <cerrno>
#include <cstdlib>
#include <exception>

#include "base/logging.h"
#include "base/mutex.h"

namespace base {

static void invoke(FDHolder::HookFn hook) noexcept {
  try {
    hook();
  } catch (...) {
    LOG_EXCEPTION(std::current_exception());
  }
}

FDHolder::FDHolder(int fd) noexcept : fd_(fd) {
  VLOG(2) << "FDHolder: obtained ownership of fd " << fd;
}

FDHolder::~FDHolder() noexcept {
  int fd = release_internal(true);
  if (fd != -1) {
    VLOG(3) << "FDHolder::~FDHolder performing close of fd " << fd;
    ::close(fd);
  }
}

void FDHolder::on_close(HookFn hook) {
  auto lock = acquire_write(rwmu_);
  if (fd_ == -1) {
    lock.unlock();
    invoke(std::move(hook));
  } else {
    hooks_.push_back(std::move(hook));
  }
}

int FDHolder::release_internal(bool for_close) noexcept {
  auto lock = acquire_write(rwmu_);
  int fd = -1;
  std::swap(fd, fd_);
  std::vector<HookFn> hooks = std::move(hooks_);
  lock.unlock();
  VLOG(2) << "FDHolder: relinquished ownership of fd " << fd << ", "
          << "for_close=" << std::boolalpha << for_close;
  for (auto& hook : hooks) {
    invoke(std::move(hook));
  }
  return fd;
}

int FDHolder::release_fd() noexcept { return release_internal(false); }

Result FDHolder::close() {
  int fd = release_internal(true);
  Result r;
  int rc = ::close(fd);
  if (rc != 0) {
    int err_no = errno;
    r = Result::from_errno(err_no, "close(2)");
  }
  return r;
}

Result make_pipe(Pipe* out) {
  *DCHECK_NOTNULL(out) = Pipe();
  int fds[2] = {-1, -1};
  int rc = ::pipe2(fds, O_NONBLOCK | O_CLOEXEC);
  if (rc != 0) {
    int err_no = errno;
    return Result::from_errno(err_no, "pipe2(2)");
  }
  *out = Pipe(wrapfd(fds[0]), wrapfd(fds[1]));
  return Result();
}

Result make_socketpair(SocketPair* out, int domain, int type, int protocol) {
  *DCHECK_NOTNULL(out) = SocketPair();
  type |= SOCK_NONBLOCK;
  type |= SOCK_CLOEXEC;
  int fds[2] = {-1, -1};
  int rc = ::socketpair(domain, type, protocol, fds);
  if (rc != 0) {
    int err_no = errno;
    return Result::from_errno(err_no, "socketpair(2)");
  }
  *out = SocketPair(wrapfd(fds[0]), wrapfd(fds[1]));
  return Result();
}

Result set_blocking(FD fd, bool value) {
  auto pair = DCHECK_NOTNULL(fd)->acquire_fd();
  int flags = ::fcntl(pair.first, F_GETFL);
  if (flags == -1) {
    int err_no = errno;
    return Result::from_errno(err_no, "fcntl(2)");
  }
  if (value)
    flags &= ~O_NONBLOCK;
  else
    flags |= O_NONBLOCK;
  int n = ::fcntl(pair.first, F_SETFL, flags);
  if (n == -1) {
    int err_no = errno;
    return Result::from_errno(err_no, "fcntl(2)");
  }
  return Result();
}

Result shutdown(FD fd, int how) {
  auto pair = DCHECK_NOTNULL(fd)->acquire_fd();
  int rc = ::shutdown(pair.first, how);
  if (rc != 0) {
    int err_no = errno;
    return Result::from_errno(err_no, "shutdown(2)");
  }
  return Result();
}

Result seek(off_t* out, FD fd, off_t offset, int whence) {
  auto pair = DCHECK_NOTNULL(fd)->acquire_fd();
  off_t n = ::lseek(pair.first, offset, whence);
  if (n == static_cast<off_t>(-1)) {
    int err_no = errno;
    return Result::from_errno(err_no, "lseek(2)");
  }
  if (out) *out = n;
  return Result();
}

Result readdir_all(std::vector<DEntry>* out, FD fd, const char* what) {
  CHECK_NOTNULL(out);
  CHECK_NOTNULL(fd);

  std::vector<char> buf(4096);
  long nread;
  auto fdpair = fd->acquire_fd();
  while (true) {
    nread = syscall(SYS_getdents, fdpair.first, buf.data(), buf.size());
    if (nread < 0) {
      int err_no = errno;
      return Result::from_errno(err_no, "getdents(2) from ", what);
    }
    if (nread == 0) break;

    const char* ptr = buf.data();
    const char* end = ptr + nread;
    while (ptr != end) {
      constexpr std::size_t kUL = sizeof(unsigned long);
      constexpr std::size_t kUS = sizeof(unsigned short);

      unsigned long ino;
      unsigned short reclen;
      unsigned char type;
      ::memcpy(&ino, ptr, kUL);
      ::memcpy(&reclen, ptr + 2 * kUL, kUS);
      type = *(ptr + reclen - 1);
      const char* p = ptr + 2 * kUL + kUS;
      out->emplace_back(ino, type, std::string(p));
      ptr += reclen;
    }
  }
  return Result();
{'fraction_non_alphanumeric': 0.17229050279329608, 'fraction_numerical': 0.010502793296089385, 'mean_word_length': 3.4979899497487437, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6554381', 'n_tokens_mistral': 3478, 'n_tokens_neox': 3416, 'n_words': 831}