<commit_before>/*
  Utility to do a parallel sync of remote ROOT files locally. As added value
  it allows to only sync the parts (TKeys) of a ROOT file which is match a
  provided regular expression.
 */

#include <sys/types.h>
#include <sys/wait.h>
#include <signal.h>
#include <unistd.h>
#include <time.h>
#include <cstdlib>
#include <cassert>
#include <string>
#include <cstdio>
#include <cstddef>
#include <stdint.h>
#include <vector>
#include <list>
#include <algorithm>
#include <sys/stat.h>
#include <limits.h>
#include <libgen.h>

#include "RConfig.h"
#include "TFile.h"
#include "THashList.h"
#include "TKey.h"
#include "TRegexp.h"
#include "TTree.h"
#include "TObjString.h"
#include "Riostream.h"
#include "TClass.h"
#include "TSystem.h"
#include "TGrid.h"
#include "TFile.h"

#include "TFileMerger.h"

static const char *USAGE = 
      "usage: alisync [-h] [options] [@]<input file> [<input file>]\n"
      "Synchronizes remote ROOT files locally, atomically and in parallel\n"
      "If <input file> is prepended with @ it will be considered"
      " a list of other files to be merged.\n"
      "\n"
      "The following options can be specified:\n"
      "   -v [<level>]\n"
      "      Set verbosity level to <level>. If <level> is not provided defaults to maximum verbosity (99).\n"
      "   -o <output directory>\n"
      "      Set the output directory, i.e. where the files should be copied to. Defaults to the current one.\n"
      "   -j <# jobs>\n"
      "      Use <# jobs> parallel jobs to do the fetching (1 by default). \n"
      "   -t <# seconds>\n"
      "      Set per-file timeout while fetching to <# seconds> seconds.\n"
      "   -n <# retries>\n"
      "      Retry fetching for <# retries> retries before considering the file as not available.\n"
      "   -s <success rate>\n"
      "      Do not bother fetching more files after success rate has been reached. \n"
      "      E.g. if you are fine with only 90\% of the files being transferred, specify `-s 0.9'.\n"
      "   -i <regex>\n"
      "      Include TKeys whose name matches <regex>. This option can be used multiple times.\n";

const char * OPT_STRING = ":v:j:t:n:s:i:o:";

static int gVerbosity = 0;

// Helper to print out verbosity aware messages.
void log(int level, const char *fmt, ...)
{
  if (level > gVerbosity)
    return;

  va_list args;
  va_start(args, fmt);
  vprintf(fmt, args);
  va_end(args);
}

// Helper to print out a message and exit with exit code `exit`.
void dieIf(bool condition, int exitCode, const char *fmt, ...)
{
  if (!condition)
    return;
  va_list args;
  va_start(args, fmt);
  vprintf(fmt, args);
  va_end(args);
  exit(exitCode);
}

// Helper for recursive directory creation
static void recursiveLocalMkdir(const char *dir) {
  char tmp[PATH_MAX];
  char *p = NULL;
  size_t len;
  
  snprintf(tmp, sizeof(tmp),"%s",dir);
  len = strlen(tmp);
  if(tmp[len - 1] == '/')
          tmp[len - 1] = 0;
  for(p = tmp + 1; *p; p++)
          if(*p == '/') {
                  *p = 0;
                  mkdir(tmp, S_IRWXU);
                  *p = '/';
          }
  mkdir(tmp, S_IRWXU);
}

struct JobInfo {
  std::string filename;
  time_t startTime;
  pid_t pid;
  int exitCode;
  int retries;
  bool running;
};

struct SamePidAs {
  SamePidAs(pid_t pid) : pid(pid) {}
  pid_t pid;
  bool operator()(const JobInfo &info) { return info.pid == this->pid;}
};

struct MatchPathSeparator
{
  bool operator()( char ch ) const
  {
    return ch == '/';
  }
};

struct NotMatchPathSeparator
{
  bool operator()( char ch ) const
  {
    return ch != '/';
  }
};

static std::vector<JobInfo> gJobs;

// Downloading of files:
// - Connect to alien if the filename starts with "alien".
// - Create a temporary destination file.
// - TFile::Cp the file to the output directory.
void downloadFile(const JobInfo &info, 
                  const std::string &outdir,
                  const std::vector<TRegexp *> &regexps) {
  gSystem->Load("libTreePlayer");

  static const std::string prefix = "alien://"; 
  if (info.filename.compare(0, prefix.size(), prefix) == 0
      || outdir.compare(0, prefix.size(), prefix) == 0)
    TGrid::Connect("alien://");
  bool useAtomic = true;
  if (outdir.compare(0, prefix.size(), prefix) == 0)
    useAtomic = false;

  char *relDir = strdup(dirname(strdup(info.filename.c_str())));
  char *tmpFilename = strdup(basename(strdup(info.filename.c_str())));
  std::string tmpdest = outdir + "/" + relDir + "/." + tmpFilename + ".tmp";
  char *tmpoutdestCpy = strdup(tmpdest.c_str());
  char *tmpoutdir = strdup(dirname(tmpoutdestCpy));
  std::string dest = outdir + "/" + info.filename;
  // If destination is a local file, remove old temporary file.
  // else we really copy directly where we want to be.
  if (useAtomic) {
    recursiveLocalMkdir(tmpoutdir);
    unlink(tmpdest.c_str());
  } else {
    assert(gGrid);
    dieIf(gGrid->Mkdir(tmpoutdir, "-p") != 0, 1, "Unable to create %s\n", tmpoutdir);
    tmpdest = dest;<commit_msg>Correct check of TGrid::Mkdir exit code in alisync<commit_after>/*
  Utility to do a parallel sync of remote ROOT files locally. As added value
  it allows to only sync the parts (TKeys) of a ROOT file which is match a
  provided regular expression.
 */

#include <sys/types.h>
#include <sys/wait.h>
#include <signal.h>
#include <unistd.h>
#include <time.h>
#include <cstdlib>
#include <cassert>
#include <string>
#include <cstdio>
#include <cstddef>
#include <stdint.h>
#include <vector>
#include <list>
#include <algorithm>
#include <sys/stat.h>
#include <limits.h>
#include <libgen.h>

#include "RConfig.h"
#include "TFile.h"
#include "THashList.h"
#include "TKey.h"
#include "TRegexp.h"
#include "TTree.h"
#include "TObjString.h"
#include "Riostream.h"
#include "TClass.h"
#include "TSystem.h"
#include "TGrid.h"
#include "TFile.h"

#include "TFileMerger.h"

static const char *USAGE = 
      "usage: alisync [-h] [options] [@]<input file> [<input file>]\n"
      "Synchronizes remote ROOT files locally, atomically and in parallel\n"
      "If <input file> is prepended with @ it will be considered"
      " a list of other files to be merged.\n"
      "\n"
      "The following options can be specified:\n"
      "   -v [<level>]\n"
      "      Set verbosity level to <level>. If <level> is not provided defaults to maximum verbosity (99).\n"
      "   -o <output directory>\n"
      "      Set the output directory, i.e. where the files should be copied to. Defaults to the current one.\n"
      "   -j <# jobs>\n"
      "      Use <# jobs> parallel jobs to do the fetching (1 by default). \n"
      "   -t <# seconds>\n"
      "      Set per-file timeout while fetching to <# seconds> seconds.\n"
      "   -n <# retries>\n"
      "      Retry fetching for <# retries> retries before considering the file as not available.\n"
      "   -s <success rate>\n"
      "      Do not bother fetching more files after success rate has been reached. \n"
      "      E.g. if you are fine with only 90\% of the files being transferred, specify `-s 0.9'.\n"
      "   -i <regex>\n"
      "      Include TKeys whose name matches <regex>. This option can be used multiple times.\n";

const char * OPT_STRING = ":v:j:t:n:s:i:o:";

static int gVerbosity = 0;

// Helper to print out verbosity aware messages.
void log(int level, const char *fmt, ...)
{
  if (level > gVerbosity)
    return;

  va_list args;
  va_start(args, fmt);
  vprintf(fmt, args);
  va_end(args);
}

// Helper to print out a message and exit with exit code `exit`.
void dieIf(bool condition, int exitCode, const char *fmt, ...)
{
  if (!condition)
    return;
  va_list args;
  va_start(args, fmt);
  vprintf(fmt, args);
  va_end(args);
  exit(exitCode);
}

// Helper for recursive directory creation
static void recursiveLocalMkdir(const char *dir) {
  char tmp[PATH_MAX];
  char *p = NULL;
  size_t len;
  
  snprintf(tmp, sizeof(tmp),"%s",dir);
  len = strlen(tmp);
  if(tmp[len - 1] == '/')
          tmp[len - 1] = 0;
  for(p = tmp + 1; *p; p++)
          if(*p == '/') {
                  *p = 0;
                  mkdir(tmp, S_IRWXU);
                  *p = '/';
          }
  mkdir(tmp, S_IRWXU);
}

struct JobInfo {
  std::string filename;
  time_t startTime;
  pid_t pid;
  int exitCode;
  int retries;
  bool running;
};

struct SamePidAs {
  SamePidAs(pid_t pid) : pid(pid) {}
  pid_t pid;
  bool operator()(const JobInfo &info) { return info.pid == this->pid;}
};

struct MatchPathSeparator
{
  bool operator()( char ch ) const
  {
    return ch == '/';
  }
};

struct NotMatchPathSeparator
{
  bool operator()( char ch ) const
  {
    return ch != '/';
  }
};

static std::vector<JobInfo> gJobs;

// Downloading of files:
// - Connect to alien if the filename starts with "alien".
// - Create a temporary destination file.
// - TFile::Cp the file to the output directory.
void downloadFile(const JobInfo &info, 
                  const std::string &outdir,
                  const std::vector<TRegexp *> &regexps) {
  gSystem->Load("libTreePlayer");

  static const std::string prefix = "alien://"; 
  if (info.filename.compare(0, prefix.size(), prefix) == 0
      || outdir.compare(0, prefix.size(), prefix) == 0)
    TGrid::Connect("alien://");
  bool useAtomic = true;
  if (outdir.compare(0, prefix.size(), prefix) == 0)
    useAtomic = false;

  char *relDir = strdup(dirname(strdup(info.filename.c_str())));
  char *tmpFilename = strdup(basename(strdup(info.filename.c_str())));
  std::string tmpdest = outdir + "/" + relDir + "/." + tmpFilename + ".tmp";
  char *tmpoutdestCpy = strdup(tmpdest.c_str());
  char *tmpoutdir = strdup(dirname(tmpoutdestCpy));
  std::string dest = outdir + "/" + info.filename;
  // If destination is a local file, remove old temporary file.
  // else we really copy directly where we want to be.
  if (useAtomic) {
    recursiveLocalMkdir(tmpoutdir);
    unlink(tmpdest.c_str());
  } else {
    assert(gGrid);
    dieIf(gGrid->Mkdir(tmpoutdir, "-p") == 0, 1, "Unable to create %s\n", tmpoutdir);
    tmpdest = dest;
{'fraction_non_alphanumeric': 0.14520793856587214, 'fraction_numerical': 0.004188690535554004, 'mean_word_length': 2.923317683881064, 'pattern_counts': {'":': 2, '<': 73, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5972430', 'n_tokens_mistral': 3534, 'n_tokens_neox': 3457, 'n_words': 1070}