<commit_before>// This file is part of the "libxzero" project
//   (c) 2009-2015 <NAME> <https://github.com/christianparpart>
//   (c) 2014-2015 <NAME> <https://github.com/paulasmuth>
//
// libxzero is free software: you can redistribute it and/or modify it under
// the terms of the GNU Affero General Public License v3.0.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

#include <xzero/io/FileUtil.h>
#include <xzero/io/FileView.h>
#include <xzero/io/FileDescriptor.h>
#include <xzero/io/File.h>
#include <xzero/RuntimeError.h>
#include <xzero/Buffer.h>
#include <xzero/logging.h>
#include <xzero/sysconfig.h>

#include <system_error>
#include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h>
#include <unistd.h>
#include <fcntl.h>
#include <limits.h>
#include <stdlib.h>
#include <pwd.h>
#include <grp.h>

namespace xzero {

static const char PathSeperator = '/';

char FileUtil::pathSeperator() noexcept {
  return PathSeperator;
}

std::string FileUtil::currentWorkingDirectory() {
  char buf[PATH_MAX];
  if (getcwd(buf, sizeof(buf)))
    return buf;

  RAISE_ERRNO(errno);
}

std::string FileUtil::realpath(const std::string& relpath) {
  if (relpath.find(PathSeperator) == std::string::npos)
    return joinPaths(currentWorkingDirectory(), relpath);

  char result[PATH_MAX];
  if (::realpath(relpath.c_str(), result) == nullptr)
    RAISE_ERRNO(errno);

  return result;
}

bool FileUtil::exists(const std::string& path) {
  struct stat st;
  return stat(path.c_str(), &st) == 0;
}

bool FileUtil::isDirectory(const std::string& path) {
  struct stat st;
  return stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode);
}

bool FileUtil::isRegular(const std::string& path) {
  struct stat st;
  return stat(path.c_str(), &st) == 0 && S_ISREG(st.st_mode);
}

size_t FileUtil::size(const std::string& path) {
  struct stat st;

  if (stat(path.c_str(), &st) == 0)
    return st.st_size;

  RAISE_ERRNO(errno);
}

size_t FileUtil::sizeRecursive(const std::string& path) {
  struct stat st;
  if (stat(path.c_str(), &st) < 0)
    RAISE_ERRNO(errno);

  if (S_ISREG(st.st_mode))
    return st.st_size;

  if (!S_ISDIR(st.st_mode))
    return 0;

  size_t totalSize = 0;
  FileUtil::ls(path, [&totalSize](const std::string& childpath) -> bool {
    totalSize += sizeRecursive(childpath);
    return true;
  });
  return totalSize;
}

void FileUtil::ls(const std::string& path,
                  std::function<bool(const std::string&)> callback) {
  DIR* dir = opendir(path.c_str());
  if (!dir)
    RAISE_ERRNO(errno);

  int len = offsetof(dirent, d_name) + pathconf(path.c_str(), _PC_NAME_MAX);
  dirent* dep = (dirent*) new unsigned char[len + 1];
  dirent* res = nullptr;
  Buffer buf;

  std::string filename;
  filename = joinPaths(path, "/");
  std::size_t baseFileNameLength = filename.size();

  while (readdir_r(dir, dep, &res) == 0 && res) {
    // skip "."
    if (dep->d_name[0] == '.' && dep->d_name[1] == 0)
      continue;

    // skip ".."
    if (dep->d_name[0] == '.' && dep->d_name[1] == '.' && dep->d_name[2] == 0)
      continue;

    // prepare filename
    filename += static_cast<char*>(dep->d_name);

    // callback
    if (!callback(filename))
      break;

    // reset filename to its base-length
    filename.resize(baseFileNameLength);
  }

  // free resources
  delete[] dep;
  closedir(dir);
}

std::string FileUtil::joinPaths(const std::string& base,
                                const std::string& append) {
  if (base.empty())
    return append;

  if (append.empty())
    return base;

  if (base.back() == PathSeperator) {
    if (append.front() == PathSeperator) {
      return base + append.substr(1);
    } else {
      return base + append;
    }
  } else {
    if (append.front() == PathSeperator) {
      return base + append;
    } else {
      return base + '/' + append;
    }
  }
}

void FileUtil::seek(int fd, off_t offset) {
  off_t rv = ::lseek(fd, offset, SEEK_SET);
  if (rv == (off_t) -1)
    RAISE_ERRNO(errno);
}

void FileUtil::read(int fd, Buffer* output) {
  struct stat st;
  if (fstat(fd, &st) < 0)
    RAISE_ERRNO(errno);

  size_t beg = output->size();
  output->reserve(beg + st.st_size + 1);
  ssize_t nread = ::pread(fd, output->data() + beg, st.st_size, 0);
  if (nread < 0)
    RAISE_ERRNO(errno);

  output->data()[beg + nread] = '\0';
  output->resize(beg + nread);
}

void FileUtil::read(File& file, Buffer* output) {
  FileDescriptor fd = file.createPosixChannel(File::Read);
  read(fd, output);
}

void FileUtil::read(const std::string& path, Buffer* output) {
  FileDescriptor fd = open(path.c_str(), O_RDONLY);
  if (fd < 0)
    RAISE_ERRNO(errno);
<commit_msg>FileUtil: read(int fd, Buffer* out) must handle a special case where fstat() doesn't yield the file size in advance, such as files in /proc<commit_after>// This file is part of the "libxzero" project
//   (c) 2009-2015 <NAME> <https://github.com/christianparpart>
//   (c) 2014-2015 <NAME> <https://github.com/paulasmuth>
//
// libxzero is free software: you can redistribute it and/or modify it under
// the terms of the GNU Affero General Public License v3.0.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

#include <xzero/io/FileUtil.h>
#include <xzero/io/FileView.h>
#include <xzero/io/FileDescriptor.h>
#include <xzero/io/File.h>
#include <xzero/RuntimeError.h>
#include <xzero/Buffer.h>
#include <xzero/logging.h>
#include <xzero/sysconfig.h>

#include <system_error>
#include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h>
#include <unistd.h>
#include <fcntl.h>
#include <limits.h>
#include <stdlib.h>
#include <pwd.h>
#include <grp.h>

namespace xzero {

static const char PathSeperator = '/';

char FileUtil::pathSeperator() noexcept {
  return PathSeperator;
}

std::string FileUtil::currentWorkingDirectory() {
  char buf[PATH_MAX];
  if (getcwd(buf, sizeof(buf)))
    return buf;

  RAISE_ERRNO(errno);
}

std::string FileUtil::realpath(const std::string& relpath) {
  if (relpath.find(PathSeperator) == std::string::npos)
    return joinPaths(currentWorkingDirectory(), relpath);

  char result[PATH_MAX];
  if (::realpath(relpath.c_str(), result) == nullptr)
    RAISE_ERRNO(errno);

  return result;
}

bool FileUtil::exists(const std::string& path) {
  struct stat st;
  return stat(path.c_str(), &st) == 0;
}

bool FileUtil::isDirectory(const std::string& path) {
  struct stat st;
  return stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode);
}

bool FileUtil::isRegular(const std::string& path) {
  struct stat st;
  return stat(path.c_str(), &st) == 0 && S_ISREG(st.st_mode);
}

size_t FileUtil::size(const std::string& path) {
  struct stat st;

  if (stat(path.c_str(), &st) == 0)
    return st.st_size;

  RAISE_ERRNO(errno);
}

size_t FileUtil::sizeRecursive(const std::string& path) {
  struct stat st;
  if (stat(path.c_str(), &st) < 0)
    RAISE_ERRNO(errno);

  if (S_ISREG(st.st_mode))
    return st.st_size;

  if (!S_ISDIR(st.st_mode))
    return 0;

  size_t totalSize = 0;
  FileUtil::ls(path, [&totalSize](const std::string& childpath) -> bool {
    totalSize += sizeRecursive(childpath);
    return true;
  });
  return totalSize;
}

void FileUtil::ls(const std::string& path,
                  std::function<bool(const std::string&)> callback) {
  DIR* dir = opendir(path.c_str());
  if (!dir)
    RAISE_ERRNO(errno);

  int len = offsetof(dirent, d_name) + pathconf(path.c_str(), _PC_NAME_MAX);
  dirent* dep = (dirent*) new unsigned char[len + 1];
  dirent* res = nullptr;
  Buffer buf;

  std::string filename;
  filename = joinPaths(path, "/");
  std::size_t baseFileNameLength = filename.size();

  while (readdir_r(dir, dep, &res) == 0 && res) {
    // skip "."
    if (dep->d_name[0] == '.' && dep->d_name[1] == 0)
      continue;

    // skip ".."
    if (dep->d_name[0] == '.' && dep->d_name[1] == '.' && dep->d_name[2] == 0)
      continue;

    // prepare filename
    filename += static_cast<char*>(dep->d_name);

    // callback
    if (!callback(filename))
      break;

    // reset filename to its base-length
    filename.resize(baseFileNameLength);
  }

  // free resources
  delete[] dep;
  closedir(dir);
}

std::string FileUtil::joinPaths(const std::string& base,
                                const std::string& append) {
  if (base.empty())
    return append;

  if (append.empty())
    return base;

  if (base.back() == PathSeperator) {
    if (append.front() == PathSeperator) {
      return base + append.substr(1);
    } else {
      return base + append;
    }
  } else {
    if (append.front() == PathSeperator) {
      return base + append;
    } else {
      return base + '/' + append;
    }
  }
}

void FileUtil::seek(int fd, off_t offset) {
  off_t rv = ::lseek(fd, offset, SEEK_SET);
  if (rv == (off_t) -1)
    RAISE_ERRNO(errno);
}

void FileUtil::read(int fd, Buffer* output) {
  struct stat st;
  if (fstat(fd, &st) < 0)
    RAISE_ERRNO(errno);

  if (st.st_size > 0) {
    size_t beg = output->size();
    output->reserve(beg + st.st_size + 1);
    ssize_t nread = ::pread(fd, output->data() + beg, st.st_size, 0);
    if (nread < 0)
      RAISE_ERRNO(errno);

    output->data()[beg + nread] = '\0';
    output->resize(beg + nread);
  } else {
    // XXX some files do not yield informations via stat, such as files in /proc.
    // So fallback to standard read() until EOF is reached.
    output->reserve(output->size() + 4096);
    for (;;) {
      ssize_t nread = ::read(fd, output->end(), 4096);
      if (nread > 0) {
        output->resize(output->size() + nread);
      } else if (nread == 0) {
        break;
      } else if (errno == EINTR) {
        continue;
      } else {
        RAISE_ERRNO(errno);
      }
    }
  }
}

void FileUtil::read(File& file, Buffer* output) {
  FileDescriptor fd = file.createPosixChannel(File::Read);
  read(fd, output);
}

void FileUtil::read(const std::string& path, Buffer* output) {
  FileDescriptor fd = open(path.c_str(), O_RDONLY);
  if (fd < 0)
    RAISE_ERRNO(errno);

{'fraction_non_alphanumeric': 0.16440244142547747, 'fraction_numerical': 0.009352234691868478, 'mean_word_length': 3.424651567944251, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 84, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1326537', 'n_tokens_mistral': 3840, 'n_tokens_neox': 3681, 'n_words': 891}