<commit_before>#ifndef CROMBIE_FILESYSTEM_H
#define CROMBIE_FILESYSTEM_H

#include <cstdlib>
#include <cstring>
#include <vector>
#include <string>
#include <dirent.h>
#include <sys/stat.h>
#include <libgen.h>

#include "crombie/Types.h"
#include "crombie/Misc.h"

namespace crombie {
  namespace FileSystem {

    std::string basename(const std::string& name) {
      auto output = Misc::split(name, "/").back();
      Debug::Debug(__PRETTY_FUNCTION__, name, output);
      return output;
    }

    std::string dirname(const std::string& name) {
      std::string modname = name;
      while (modname.back() == '/')
        modname.pop_back();

      auto output = modname.substr(0, modname.size() - basename(modname).size());
      Debug::Debug(__PRETTY_FUNCTION__, name, output);
      return output;
    }

    namespace {
      // Is the path for xrd
      bool is_xrd(const std::string& path) {
        return path.find("root://") == 0;
      }

      // Do the xrdfs call
      Types::strings xrd_ls(const std::string& path) {
        // Cache for listed directories to prevent too many calls
        static Types::map<Types::strings> xrd_dir_contents;

        auto iter = xrd_dir_contents.find(path);
        if (iter != xrd_dir_contents.end())
          return iter->second;

        // Door will be between two "//" sequences when user is sane
        auto parts = Misc::split(path, "//");
        if (parts.size() != 3)
          throw std::runtime_error{std::string("xrd path (") + path + ") doesn't seem to have a good pattern"};

        // Set the cache to the split long listing of xrdfs and return it
        return xrd_dir_contents[path] = 
          Misc::split(Misc::shell(std::string("xrdfs root://") + parts[1] + "/ ls -l /" + parts[2]));
      }

      // Get the xrd_ls call and return the file portion of the line
      Types::strings xrd_list(const std::string& path) {
        return Misc::comprehension<std::string>
          (xrd_ls(path),
           [] (auto& line) {
            return basename(Misc::tokenize(line)[4]);
          });
      }

      // Get the listing of the directory, and then extract the size
      unsigned long xrd_get_size(const std::string& name) {
        // There is some tricky business about the "//" in ls outputs, so just look for the file
        std::string filename = basename(name);<commit_msg>Allow access through redirector<commit_after>#ifndef CROMBIE_FILESYSTEM_H
#define CROMBIE_FILESYSTEM_H

#include <cstdlib>
#include <cstring>
#include <vector>
#include <string>
#include <dirent.h>
#include <sys/stat.h>
#include <libgen.h>

#include "crombie/Types.h"
#include "crombie/Misc.h"

namespace crombie {
  namespace FileSystem {

    std::string basename(const std::string& name) {
      auto output = Misc::split(name, "/").back();
      Debug::Debug(__PRETTY_FUNCTION__, name, output);
      return output;
    }

    std::string dirname(const std::string& name) {
      std::string modname = name;
      while (modname.back() == '/')
        modname.pop_back();

      auto output = modname.substr(0, modname.size() - basename(modname).size());
      Debug::Debug(__PRETTY_FUNCTION__, name, output);
      return output;
    }

    namespace {
      // Is the path for xrd
      bool is_xrd(const std::string& path) {
        return path.find("root://") == 0;
      }

      // Do the xrdfs call
      Types::strings xrd_ls(const std::string& path) {
        // Cache for listed directories to prevent too many calls
        static Types::map<Types::strings> xrd_dir_contents;

        auto iter = xrd_dir_contents.find(path);
        if (iter != xrd_dir_contents.end())
          return iter->second;

        // Door will be between two "//" sequences when user is sane
        auto parts = Misc::split(path, "//");
        if (parts.size() != 3)
          throw std::runtime_error{std::string("xrd path (") + path + ") doesn't seem to have a good pattern"};

        // Set the cache to the split long listing of xrdfs and return it
        return xrd_dir_contents[path] = 
          Misc::split(Misc::shell(std::string("xrdfs root://") + parts[1] + "/ ls -l /" + parts[2] + " | sort -u"));
      }

      // Get the xrd_ls call and return the file portion of the line
      Types::strings xrd_list(const std::string& path) {
        return Misc::comprehension<std::string>
          (xrd_ls(path),
           [] (auto& line) {
            return basename(Misc::tokenize(line)[4]);
          });
      }

      // Get the listing of the directory, and then extract the size
      unsigned long xrd_get_size(const std::string& name) {
        // There is some tricky business about the "//" in ls outputs, so just look for the file
        std::string filename = basename(name);
{'fraction_non_alphanumeric': 0.13396424815983177, 'fraction_numerical': 0.002523659305993691, 'mean_word_length': 2.681114551083591, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6793556', 'n_tokens_mistral': 1504, 'n_tokens_neox': 1463, 'n_words': 416}