<commit_before>/******************************************************************************\

          This file is part of the C! library.  A.K.A the cbang library.

              Copyright (c) 2003-2015, Cauldron Development LLC
                 Copyright (c) 2003-2015, Stanford University
                             All rights reserved.

        The C! library is free software: you can redistribute it and/or
        modify it under the terms of the GNU Lesser General Public License
        as published by the Free Software Foundation, either version 2.1 of
        the License, or (at your option) any later version.

        The C! library is distributed in the hope that it will be useful,
        but WITHOUT ANY WARRANTY; without even the implied warranty of
        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
        Lesser General Public License for more details.

        You should have received a copy of the GNU Lesser General Public
        License along with the C! library.  If not, see
        <http://www.gnu.org/licenses/>.

        In addition, BSD licensing may be granted on a case by case basis
        by written permission from at least one of the copyright holders.
        You may request written permission by emailing the authors.

                For information regarding this software email:
                               <NAME>
                        <EMAIL>

\******************************************************************************/

#include "SystemUtilities.h"

#include "Subprocess.h"
#include "DirectoryWalker.h"
#include "SysError.h"
#include "Win32Utilities.h"

#include <cbang/time/Timer.h>
#include <cbang/time/Time.h>
#include <cbang/log/Logger.h>
#include <cbang/util/SmartLock.h>
#include <cbang/net/URI.h>

#include <cbang/io/File.h>

#include <cbang/Exception.h>
#include <cbang/String.h>
#include <cbang/SmartPointer.h>

#include <errno.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>

#ifdef _WIN32
#define WINDOWS_LEAN_AND_MEAN
#include <windows.h>
#include <io.h>
#include <direct.h> // For _chdir
#include <process.h>
#define getpid _getpid

#else // _WIN32
#include <unistd.h>
#include <termios.h>
#include <signal.h>
#include <sys/resource.h>
#include <sys/times.h>
#include <sys/wait.h>
#include <pwd.h>
#endif // _WIN32

#ifdef __FreeBSD__
#include <sys/sysctl.h>
#endif // __FeeBSD__

#ifdef __APPLE__
#include <mach-o/dyld.h>
#endif // __APPLE__

#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>

#include <sstream>
#include <fstream>
#include <iomanip>
#include <vector>

#define BOOST_SYSTEM_NO_DEPRECATED

#include <boost/system/error_code.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/iostreams/device/file_descriptor.hpp>

using namespace std;
using namespace cb;
using namespace cb::SystemUtilities;

namespace fs = boost::filesystem;
namespace sys = boost::system;
namespace io = boost::iostreams;


namespace cb {
  namespace SystemUtilities {
#ifdef _WIN32
    const string path_separators  = "/\\";
    const char path_separator  = '\\';
    const char path_delimiter  = ';';
    const string library_path = "PATH";
#else
    const string path_separators = "/";
    const char path_separator = '/';
    const char path_delimiter = ':';

#if defined(__APPLE__) || defined(__FreeBSD__)
    const string library_path = "DYLD_LIBRARY_PATH";
#else
    const string library_path = "LD_LIBRARY_PATH";
#endif // defined(__APPLE__) || defined(__FreeBSD__)

#endif

    bool useHardLinks = true;


    string basename(const string &path) {
      if (path.empty()) return path;
      string::size_type pos = path.find_last_of(path_separators);

      if (pos == string::npos) return path;
      else return path.substr(pos + 1);
    }


    string dirname(const string &path) {
      if (path.empty()) return path;
      string::size_type pos = path.find_last_of(path_separators);

      if (pos == path.length() - 1)
        pos = path.find_last_of(path_separators, pos - 1);

      if (pos == string::npos) return ".";
      else if (pos == 0) return "/";
      else return path.substr(0, pos);
    }


    string::size_type getExtensionPosition(const string &path) {
      string::size_type lastSep = path.find_last_of(path_separators);
      string::size_type pos = path.find_last_of('.');

      // Don't count '.' in directory names
      if (lastSep != string::npos && pos != string::npos && pos < lastSep)
        return string::npos;

      return pos;
    }


    bool hasExtension(const string &path) {
      return getExtensionPosition(path) != string::npos;
    }


    string extension(const string &path) {
      string::size_type pos = getExtensionPosition(path);
      return (pos == string::npos) ? string() : path.substr(pos + 1);
    }


    string swapExtension(const string &path, const string &ext) {
      string::size_type pos = getExtensionPosition(path);
      return path.substr(0, pos) + "." + ext;
    }


    vector<string> splitExt(const string &path) {
      vector<string> result;

      string::size_type pos = getExtensionPosition(path);
      if (pos == string::npos) {
        result.push_back(path);
        result.push_back(string());

      } else {
        result.push_back(path.substr(0, pos));
        result.push_back(path.substr(pos + 1));
      }

      return result;
    }


    bool isAbsolute(const string &path) {
      return fs::path(path).is_complete();
    }


    string relative(const string &base, const string &target,
                    unsigned maxDotDot) {
      // Find directory
      string dir;
      if (!exists(base) || isDirectory(base)) dir = base;
      else dir = dirname(base);

      // Split
      vector<string> parts1;
      vector<string> parts2;
      splitPath(getCanonicalPath(dir), parts1);
      splitPath(getCanonicalPath(target), parts2);

      // Find common parts
      unsigned i = 0;
      while (i < parts1.size() && i < parts2.size() && parts1[i] == parts2[i])
        i++;

      // Check number of ..'s
      if (maxDotDot < parts1.size() - i) return absolute(target);

      // Create relative path
      vector<string> parts;
      for (unsigned j = i; j < parts1.size(); j++) parts.push_back("..");
      for (unsigned j = i; j < parts2.size(); j++) parts.push_back(parts2[j]);

      return joinPath(parts);
    }


    string absolute(const string &base, const string &target) {
      if (isAbsolute(target)) return target;

      if (!exists(base) || isDirectory(base)) return base + "/" + target;
      else return dirname(base) + "/" + target;
    }


    string absolute(const string &path) {
      return absolute(getcwd(), path);
    }


    string getCanonicalPath(const string &_path) {
      if (_path.empty()) return _path;

      string path = _path;

      // Expand tilde
      if (path[0] == '~') {
        string name;
        unsigned i;
        for (i = 1; i < path.length() && path[i] != '/'; i++)
          name.append(1, path[i]);

#ifdef _WIN32
        if (name.empty()) {
#endif // _WIN32

          string home = getUserHome(name);
          if (i < path.length()) path = home + "/" + path.substr(i + 1);
          else path = home;

#ifdef _WIN32
        }
#endif // _WIN32
      }

      fs::path p = fs::system_complete(path);

      // Save root path
#if BOOST_FILESYSTEM_VERSION < 3
      string root = p.root_path();
#else
      string root = p.root_path().string();
#endif

      // Get relative part
      p = p.relative_path();

      // Normalize
      vector<string> parts;
      fs::path::iterator it;
      for (it = p.begin(); it != p.end(); it++)
        if (*it == ".." && !parts.empty()) parts.pop_back();
        else if (*it != "" && *it != "." && *it != "..") {
#if BOOST_FILESYSTEM_VERSION < 3
          parts.push_back(*it);
#else
          parts.push_back(it->string());
#endif
        }

      // Reassemble
      string result = root;
      for (unsigned i = 0; i < parts.size(); i++) {
        if (i) result += "/";
        result += parts[i];
      }

      return result;<commit_msg>Attempt to fix root path for old boost<commit_after>/******************************************************************************\

          This file is part of the C! library.  A.K.A the cbang library.

              Copyright (c) 2003-2015, Cauldron Development LLC
                 Copyright (c) 2003-2015, Stanford University
                             All rights reserved.

        The C! library is free software: you can redistribute it and/or
        modify it under the terms of the GNU Lesser General Public License
        as published by the Free Software Foundation, either version 2.1 of
        the License, or (at your option) any later version.

        The C! library is distributed in the hope that it will be useful,
        but WITHOUT ANY WARRANTY; without even the implied warranty of
        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
        Lesser General Public License for more details.

        You should have received a copy of the GNU Lesser General Public
        License along with the C! library.  If not, see
        <http://www.gnu.org/licenses/>.

        In addition, BSD licensing may be granted on a case by case basis
        by written permission from at least one of the copyright holders.
        You may request written permission by emailing the authors.

                For information regarding this software email:
                               <NAME>
                        <EMAIL>

\******************************************************************************/

#include "SystemUtilities.h"

#include "Subprocess.h"
#include "DirectoryWalker.h"
#include "SysError.h"
#include "Win32Utilities.h"

#include <cbang/time/Timer.h>
#include <cbang/time/Time.h>
#include <cbang/log/Logger.h>
#include <cbang/util/SmartLock.h>
#include <cbang/net/URI.h>

#include <cbang/io/File.h>

#include <cbang/Exception.h>
#include <cbang/String.h>
#include <cbang/SmartPointer.h>

#include <errno.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>

#ifdef _WIN32
#define WINDOWS_LEAN_AND_MEAN
#include <windows.h>
#include <io.h>
#include <direct.h> // For _chdir
#include <process.h>
#define getpid _getpid

#else // _WIN32
#include <unistd.h>
#include <termios.h>
#include <signal.h>
#include <sys/resource.h>
#include <sys/times.h>
#include <sys/wait.h>
#include <pwd.h>
#endif // _WIN32

#ifdef __FreeBSD__
#include <sys/sysctl.h>
#endif // __FeeBSD__

#ifdef __APPLE__
#include <mach-o/dyld.h>
#endif // __APPLE__

#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>

#include <sstream>
#include <fstream>
#include <iomanip>
#include <vector>

#define BOOST_SYSTEM_NO_DEPRECATED

#include <boost/system/error_code.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/iostreams/device/file_descriptor.hpp>

using namespace std;
using namespace cb;
using namespace cb::SystemUtilities;

namespace fs = boost::filesystem;
namespace sys = boost::system;
namespace io = boost::iostreams;


namespace cb {
  namespace SystemUtilities {
#ifdef _WIN32
    const string path_separators  = "/\\";
    const char path_separator  = '\\';
    const char path_delimiter  = ';';
    const string library_path = "PATH";
#else
    const string path_separators = "/";
    const char path_separator = '/';
    const char path_delimiter = ':';

#if defined(__APPLE__) || defined(__FreeBSD__)
    const string library_path = "DYLD_LIBRARY_PATH";
#else
    const string library_path = "LD_LIBRARY_PATH";
#endif // defined(__APPLE__) || defined(__FreeBSD__)

#endif

    bool useHardLinks = true;


    string basename(const string &path) {
      if (path.empty()) return path;
      string::size_type pos = path.find_last_of(path_separators);

      if (pos == string::npos) return path;
      else return path.substr(pos + 1);
    }


    string dirname(const string &path) {
      if (path.empty()) return path;
      string::size_type pos = path.find_last_of(path_separators);

      if (pos == path.length() - 1)
        pos = path.find_last_of(path_separators, pos - 1);

      if (pos == string::npos) return ".";
      else if (pos == 0) return "/";
      else return path.substr(0, pos);
    }


    string::size_type getExtensionPosition(const string &path) {
      string::size_type lastSep = path.find_last_of(path_separators);
      string::size_type pos = path.find_last_of('.');

      // Don't count '.' in directory names
      if (lastSep != string::npos && pos != string::npos && pos < lastSep)
        return string::npos;

      return pos;
    }


    bool hasExtension(const string &path) {
      return getExtensionPosition(path) != string::npos;
    }


    string extension(const string &path) {
      string::size_type pos = getExtensionPosition(path);
      return (pos == string::npos) ? string() : path.substr(pos + 1);
    }


    string swapExtension(const string &path, const string &ext) {
      string::size_type pos = getExtensionPosition(path);
      return path.substr(0, pos) + "." + ext;
    }


    vector<string> splitExt(const string &path) {
      vector<string> result;

      string::size_type pos = getExtensionPosition(path);
      if (pos == string::npos) {
        result.push_back(path);
        result.push_back(string());

      } else {
        result.push_back(path.substr(0, pos));
        result.push_back(path.substr(pos + 1));
      }

      return result;
    }


    bool isAbsolute(const string &path) {
      return fs::path(path).is_complete();
    }


    string relative(const string &base, const string &target,
                    unsigned maxDotDot) {
      // Find directory
      string dir;
      if (!exists(base) || isDirectory(base)) dir = base;
      else dir = dirname(base);

      // Split
      vector<string> parts1;
      vector<string> parts2;
      splitPath(getCanonicalPath(dir), parts1);
      splitPath(getCanonicalPath(target), parts2);

      // Find common parts
      unsigned i = 0;
      while (i < parts1.size() && i < parts2.size() && parts1[i] == parts2[i])
        i++;

      // Check number of ..'s
      if (maxDotDot < parts1.size() - i) return absolute(target);

      // Create relative path
      vector<string> parts;
      for (unsigned j = i; j < parts1.size(); j++) parts.push_back("..");
      for (unsigned j = i; j < parts2.size(); j++) parts.push_back(parts2[j]);

      return joinPath(parts);
    }


    string absolute(const string &base, const string &target) {
      if (isAbsolute(target)) return target;

      if (!exists(base) || isDirectory(base)) return base + "/" + target;
      else return dirname(base) + "/" + target;
    }


    string absolute(const string &path) {
      return absolute(getcwd(), path);
    }


    string getCanonicalPath(const string &_path) {
      if (_path.empty()) return _path;

      string path = _path;

      // Expand tilde
      if (path[0] == '~') {
        string name;
        unsigned i;
        for (i = 1; i < path.length() && path[i] != '/'; i++)
          name.append(1, path[i]);

#ifdef _WIN32
        if (name.empty()) {
#endif // _WIN32

          string home = getUserHome(name);
          if (i < path.length()) path = home + "/" + path.substr(i + 1);
          else path = home;

#ifdef _WIN32
        }
#endif // _WIN32
      }

      fs::path p = fs::system_complete(path);
      string root = p.root_path().string();

      // Get relative part
      p = p.relative_path();

      // Normalize
      vector<string> parts;
      fs::path::iterator it;
      for (it = p.begin(); it != p.end(); it++)
        if (*it == ".." && !parts.empty()) parts.pop_back();
        else if (*it != "" && *it != "." && *it != "..") {
#if BOOST_FILESYSTEM_VERSION < 3
          parts.push_back(*it);
#else
          parts.push_back(it->string());
#endif
        }

      // Reassemble
      string result = root;
      for (unsigned i = 0; i < parts.size(); i++) {
        if (i) result += "/";
        result += parts[i];
      }

      return result;
{'fraction_non_alphanumeric': 0.14763231197771587, 'fraction_numerical': 0.00798514391829155, 'mean_word_length': 2.698717948717949, 'pattern_counts': {'":': 0, '<': 116, '<?xml version=': 0, '>': 97, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4490866', 'n_tokens_mistral': 5122, 'n_tokens_neox': 4885, 'n_words': 1411}