<commit_before>// @(#)root/base:$Id$
// Author: <NAME>, <NAME>  04/06/07

/*************************************************************************
 * Copyright (C) 1995-2007, <NAME> and <NAME>.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

//////////////////////////////////////////////////////////////////////////
//                                                                      //
// TFileCollection                                                      //
//                                                                      //
// Class that contains a list of TFileInfo's and accumulated meta       //
// data information about its entries. This class is used to describe   //
// file sets as stored by Grid file catalogs, by PROOF or any other     //
// collection of TFile names.                                           //
//                                                                      //
//////////////////////////////////////////////////////////////////////////

#include "TFileCollection.h"
#include "THashList.h"
#include "TFileInfo.h"
#include "TIterator.h"
#include "TMap.h"
#include "TObjString.h"
#include "TUri.h"
#include "TUrl.h"
#include "TSystem.h"
#include "Riostream.h"
#include "TRegexp.h"



ClassImp(TFileCollection)

//______________________________________________________________________________
TFileCollection::TFileCollection(const char *name, const char *title,
                                 const char *textfile, Int_t nfiles, Int_t firstfile)
   : TNamed(name, title), fList(0), fMetaDataList(0), fDefaultTree(),
     fTotalSize(0), fNFiles(0), fNStagedFiles(0), fNCorruptFiles(0)
{
   // TFileCollection constructor. Specify a name and title describing
   // the list. If textfile is specified the file is opened and a
   // TFileCollection is created containing the files in the textfile.

   fList = new THashList();
   fList->SetOwner();

   fMetaDataList = new TList;
   fMetaDataList->SetOwner();

   AddFromFile(textfile, nfiles, firstfile);
}

//______________________________________________________________________________
TFileCollection::~TFileCollection()
{
   // Cleanup.

   delete fList;
   delete fMetaDataList;
}

//______________________________________________________________________________
Int_t TFileCollection::Add(TFileInfo *info)
{
   // Add TFileInfo to the collection.

   if (fList && info) {
      if (!fList->FindObject(info->GetName())) {
         fList->Add(info);
         return 1;
      } else {
         Warning("Add", "file: '%s' already in the list - ignoring",
                        info->GetCurrentUrl()->GetUrl());
      }
   }
   return 0;
}

//______________________________________________________________________________
Int_t TFileCollection::Add(TFileCollection *coll)
{
   // Add content of the TFileCollection to this collection.

   if (fList && coll && coll->GetList()) {
      TIter nxfi(coll->GetList());
      TFileInfo *fi = 0;
      while ((fi = (TFileInfo *) nxfi())) {
         fList->Add(new TFileInfo(*fi));
      }
      return 1;
   } else {
      return 0;
   }
}

//______________________________________________________________________________
Int_t TFileCollection::AddFromFile(const char *textfile, Int_t nfiles, Int_t firstfile)
{
   // Add file names contained in the specified text file.
   // The file should contain one url per line; empty lines or lines starting with '#'
   // (commented lines) are ignored.
   // If nfiles > 0 only nfiles files are added, starting from file 'firstfile' (>= 1).
   // The method returns the number of added files.

   if (!fList)
     return 0;

   Int_t nf = 0;
   if (textfile && *textfile) {
      ifstream f;
      f.open(gSystem->ExpandPathName(textfile));
      if (f.is_open()) {
         Bool_t all = (nfiles <= 0) ? kTRUE : kFALSE;
         Int_t ff = (!all && (firstfile < 1)) ? 1 : firstfile;
         Int_t nn = 0;
         while (f.good() && (all || nf < nfiles)) {
            TString line;
            line.ReadToDelim(f);
            // Skip commented or empty lines
            if (!line.IsWhitespace() && !line.BeginsWith("#")) {
               nn++;
               if (all || nn >= ff) {
                  fList->Add(new TFileInfo(line));
                  nf++;
               }
            }
         }
         f.close();
         Update();
      } else
         Error("AddFromFile", "unable to open file %s", textfile);
   }
   return nf;
}

//______________________________________________________________________________
Int_t TFileCollection::Add(const char *dir)
{
   // Add all files matching the specified pattern to the collection.
   // 'dir' can include wildcards after the last slash, which causes all
   // matching files in that directory to be added.
   // If dir is the full path of a file, only one element is added.
   // Return value is the number of added files.

   Int_t nf = 0;

   if (!fList)
      return nf;

   if (!dir || !*dir) {
      Error("Add", "input dir undefined");
      return nf;
   }

   FileStat_t st;
   FileStat_t tmp;
   TString baseDir = gSystem->DirName(dir);
   // if the 'dir' or its base dir exist
   if (gSystem->GetPathInfo(dir, st) == 0 ||
       gSystem->GetPathInfo(baseDir, tmp) == 0) {
      // If 'dir' points to a single file, add to the list and exit
      if (R_ISREG(st.fMode)) {
         // regular, single file
         TFileInfo *info = new TFileInfo(dir);
         info->SetBit(TFileInfo::kStaged);
         Add(info);
         nf++;
         Update();
         return nf;
      } else {
         void *dataSetDir = gSystem->OpenDirectory(gSystem->DirName(dir));
         if (!dataSetDir) {
            // directory cannot be opened
            Error("Add", "directory %s cannot be opened",
                  gSystem->DirName(dir));
         } else {
            const char *ent;
            TString filesExp(gSystem->BaseName(dir));<commit_msg>From <NAME>; In the present implementation, TFileCollection::Add("dir/file*.root") would not only match<commit_after>// @(#)root/base:$Id$
// Author: <NAME>, <NAME>  04/06/07

/*************************************************************************
 * Copyright (C) 1995-2007, <NAME> and <NAME>.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

//////////////////////////////////////////////////////////////////////////
//                                                                      //
// TFileCollection                                                      //
//                                                                      //
// Class that contains a list of TFileInfo's and accumulated meta       //
// data information about its entries. This class is used to describe   //
// file sets as stored by Grid file catalogs, by PROOF or any other     //
// collection of TFile names.                                           //
//                                                                      //
//////////////////////////////////////////////////////////////////////////

#include "TFileCollection.h"
#include "THashList.h"
#include "TFileInfo.h"
#include "TIterator.h"
#include "TMap.h"
#include "TObjString.h"
#include "TUri.h"
#include "TUrl.h"
#include "TSystem.h"
#include "Riostream.h"
#include "TRegexp.h"



ClassImp(TFileCollection)

//______________________________________________________________________________
TFileCollection::TFileCollection(const char *name, const char *title,
                                 const char *textfile, Int_t nfiles, Int_t firstfile)
   : TNamed(name, title), fList(0), fMetaDataList(0), fDefaultTree(),
     fTotalSize(0), fNFiles(0), fNStagedFiles(0), fNCorruptFiles(0)
{
   // TFileCollection constructor. Specify a name and title describing
   // the list. If textfile is specified the file is opened and a
   // TFileCollection is created containing the files in the textfile.

   fList = new THashList();
   fList->SetOwner();

   fMetaDataList = new TList;
   fMetaDataList->SetOwner();

   AddFromFile(textfile, nfiles, firstfile);
}

//______________________________________________________________________________
TFileCollection::~TFileCollection()
{
   // Cleanup.

   delete fList;
   delete fMetaDataList;
}

//______________________________________________________________________________
Int_t TFileCollection::Add(TFileInfo *info)
{
   // Add TFileInfo to the collection.

   if (fList && info) {
      if (!fList->FindObject(info->GetName())) {
         fList->Add(info);
         return 1;
      } else {
         Warning("Add", "file: '%s' already in the list - ignoring",
                        info->GetCurrentUrl()->GetUrl());
      }
   }
   return 0;
}

//______________________________________________________________________________
Int_t TFileCollection::Add(TFileCollection *coll)
{
   // Add content of the TFileCollection to this collection.

   if (fList && coll && coll->GetList()) {
      TIter nxfi(coll->GetList());
      TFileInfo *fi = 0;
      while ((fi = (TFileInfo *) nxfi())) {
         fList->Add(new TFileInfo(*fi));
      }
      return 1;
   } else {
      return 0;
   }
}

//______________________________________________________________________________
Int_t TFileCollection::AddFromFile(const char *textfile, Int_t nfiles, Int_t firstfile)
{
   // Add file names contained in the specified text file.
   // The file should contain one url per line; empty lines or lines starting with '#'
   // (commented lines) are ignored.
   // If nfiles > 0 only nfiles files are added, starting from file 'firstfile' (>= 1).
   // The method returns the number of added files.

   if (!fList)
     return 0;

   Int_t nf = 0;
   if (textfile && *textfile) {
      ifstream f;
      f.open(gSystem->ExpandPathName(textfile));
      if (f.is_open()) {
         Bool_t all = (nfiles <= 0) ? kTRUE : kFALSE;
         Int_t ff = (!all && (firstfile < 1)) ? 1 : firstfile;
         Int_t nn = 0;
         while (f.good() && (all || nf < nfiles)) {
            TString line;
            line.ReadToDelim(f);
            // Skip commented or empty lines
            if (!line.IsWhitespace() && !line.BeginsWith("#")) {
               nn++;
               if (all || nn >= ff) {
                  fList->Add(new TFileInfo(line));
                  nf++;
               }
            }
         }
         f.close();
         Update();
      } else
         Error("AddFromFile", "unable to open file %s", textfile);
   }
   return nf;
}

//______________________________________________________________________________
Int_t TFileCollection::Add(const char *dir)
{
   // Add all files matching the specified pattern to the collection.
   // 'dir' can include wildcards after the last slash, which causes all
   // matching files in that directory to be added.
   // If dir is the full path of a file, only one element is added.
   // Return value is the number of added files.

   Int_t nf = 0;

   if (!fList)
      return nf;

   if (!dir || !*dir) {
      Error("Add", "input dir undefined");
      return nf;
   }

   FileStat_t st;
   FileStat_t tmp;
   TString baseDir = gSystem->DirName(dir);
   // if the 'dir' or its base dir exist
   if (gSystem->GetPathInfo(dir, st) == 0 ||
       gSystem->GetPathInfo(baseDir, tmp) == 0) {
      // If 'dir' points to a single file, add to the list and exit
      if (R_ISREG(st.fMode)) {
         // regular, single file
         TFileInfo *info = new TFileInfo(dir);
         info->SetBit(TFileInfo::kStaged);
         Add(info);
         nf++;
         Update();
         return nf;
      } else {
         void *dataSetDir = gSystem->OpenDirectory(gSystem->DirName(dir));
         if (!dataSetDir) {
            // directory cannot be opened
            Error("Add", "directory %s cannot be opened",
                  gSystem->DirName(dir));
         } else {
            const char *ent;
            TString filesExp(TString("^") + gSystem->BaseName(dir) + "$");
{'fraction_non_alphanumeric': 0.15412072658047116, 'fraction_numerical': 0.00571111287380027, 'mean_word_length': 1.8963932919825408, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1862460', 'n_tokens_mistral': 3360, 'n_tokens_neox': 3239, 'n_words': 982}