<commit_before>// @(#)root/html:$Id$
// Author: <NAME> (18/10/95), <NAME> (09/28/01)

/*************************************************************************
 * Copyright (C) 1995-2007, <NAME> and <NAME>.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

#include "THtml.h"
#include "RConfigure.h"
#include "Riostream.h"
#include "TBaseClass.h"
#include "TClass.h"
#include "TClassDocOutput.h"
#include "TClassEdit.h"
#include "TClassTable.h"
#include "TDataType.h"
#include "TDocInfo.h"
#include "TDocOutput.h"
#include "TEnv.h"
#include "TInterpreter.h"
#include "TObjString.h"
#include "TPRegexp.h"
#include "TRegexp.h"
#include "TROOT.h"
#include "TSystem.h"
#include "TThread.h"

#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <set>
#include <fstream>

THtml *gHtml = 0;

//______________________________________________________________________________
//______________________________________________________________________________
namespace {
   class THtmlThreadInfo {
   public:
      THtmlThreadInfo(THtml* html, bool force): fHtml(html), fForce(force) {}
      Bool_t GetForce() const {return fForce;}
      THtml* GetHtml() const {return fHtml;}

   private:
      THtml* fHtml;
      Bool_t fForce;
   };
};


//______________________________________________________________________________
THtml::THelperBase::~THelperBase()
{
   // Helper's destructor.
   // Check that no THtml object is attached to the helper - it might still need it!
   if (fHtml) {
      fHtml->HelperDeleted(this);
   }
}


//______________________________________________________________________________
void THtml::THelperBase::SetOwner(THtml* html) {
   // Set the THtml object owning this object; if it's already set to
   // a different THtml object than issue an error message and signal to
   // the currently set object that we are not belonging to it anymore.
   if (fHtml && html && html != fHtml) {
      Error("SetOwner()", "Object already owned by an THtml instance!");
      fHtml->HelperDeleted(this);
   }
   fHtml = html;
}


//______________________________________________________________________________
bool THtml::TModuleDefinition::GetModule(TClass* cl, TFileSysEntry* fse,
                                         TString& out_modulename) const
{
   // Set out_modulename to cl's module name; return true if it's valid.
   // If applicable, the module contains super modules separated by "/".
   //
   // ROOT takes the directory part of cl's implementation file name
   // (or declaration file name, if the implementation file name is empty),
   // removes the last subdirectory if it is "src/" or "inc/", and interprets
   // the remaining path as the module hierarchy, converting it to upper case.
   // hist/histpainter/src/THistPainter.cxx thus becomes the module
   // HIST/HISTPAINTER. (Node: some ROOT packages get special treatment.)
   // If the file cannot be mapped into this scheme, the class's library
   // name (without directories, leading "lib" prefix or file extensions)
   // ius taken as the module name. If the module cannot be determined it is
   // set to "USER" and false is returned.
   //
   // If your software cannot be mapped into this scheme then derive your
   // own class from TModuleDefinition and pass it to THtml::SetModuleDefinition().
   //
   // The fse parameter is used to determine the relevant part of the path, i.e.
   // to not include parent directories of a TFileSysRoot.

   out_modulename = "USER";
   if (!cl) return false;

   // Filename: impl or decl?
   TString filename;
   if (fse) fse->GetFullName(filename, kFALSE);
   else {
      if (!GetOwner()->GetImplFileName(cl, kFALSE, filename))
         if (!GetOwner()->GetDeclFileName(cl, kFALSE, filename))
            return false;
   }
   TString inputdir = GetOwner()->GetInputPath();
   TString tok;
   Ssiz_t start = 0;
   // For -Idir/sub and A.h in dir/sub/A.h, use sub as module name if
   // it would eb empty otehrwise.
   TString trailingInclude;
   while (inputdir.Tokenize(tok, start, THtml::GetDirDelimiter())) {
      if (filename.BeginsWith(tok)) {
         if (tok.EndsWith("/") || tok.EndsWith("\\"))
            tok.Remove(tok.Length() - 1);
         trailingInclude = gSystem->BaseName(tok);
         filename.Remove(0, tok.Length());
         break;
      }
   }

   // take the directory name without "/" or leading "."
   out_modulename = gSystem->DirName(filename);

   while (out_modulename[0] == '.')
      out_modulename.Remove(0, 1);
   out_modulename.ReplaceAll("\\", "/");
   while (out_modulename[0] == '/')
      out_modulename.Remove(0, 1);
   while (out_modulename.EndsWith("/"))
      out_modulename.Remove(out_modulename.Length() - 1);

   if (!out_modulename[0])
      out_modulename = trailingInclude;

   if (!out_modulename[0])
      out_modulename = trailingInclude;

   // remove "/src", "/inc"
   if (out_modulename.EndsWith("/src")
      || out_modulename.EndsWith("/inc"))
      out_modulename.Remove(out_modulename.Length() - 4, 4);
   else {
   // remove "/src/whatever", "/inc/whatever"
      Ssiz_t pos = out_modulename.Index("/src/");
      if (pos == kNPOS)
         pos = out_modulename.Index("/inc/");
      if (pos != kNPOS)
         out_modulename.Remove(pos);
   }

   while (out_modulename.EndsWith("/"))
      out_modulename.Remove(out_modulename.Length() - 1);

   // special treatment:
   if (out_modulename == "MATH/GENVECTOR")
      out_modulename = "MATHCORE";
   else if (out_modulename == "MATH/MATRIX")
      out_modulename = "SMATRIX";
   else if (!out_modulename.Length()) {
      const char* cname= cl->GetName();
      if (strstr(cname, "::SMatrix<") || strstr(cname, "::SVector<"))
         out_modulename = "SMATRIX";
      else if (strstr(cname, "::TArrayProxy<") || strstr(cname, "::TClaArrayProxy<")
               || strstr(cname, "::TImpProxy<") || strstr(cname, "::TClaImpProxy<"))
         out_modulename = "TREEPLAYER";
      else {
         // determine the module name from the library name:
         out_modulename = cl->GetSharedLibs();
         Ssiz_t pos = out_modulename.Index(' ');
         if (pos != kNPOS)
            out_modulename.Remove(pos, out_modulename.Length());
         if (out_modulename.BeginsWith("lib"))
            out_modulename.Remove(0,3);
         pos = out_modulename.Index('.');
         if (pos != kNPOS)
            out_modulename.Remove(pos, out_modulename.Length());

         if (!out_modulename.Length()) {
            out_modulename = "USER";
            return false;
         }
      }
   }

   return true;
}

//______________________________________________________________________________
void THtml::TFileDefinition::ExpandSearchPath(TString& path) const
{
   // Create all permutations of path and THtml's input path:
   // path being PP/ and THtml's input being .:include/:src/ gives
   // .:./PP/:include:include/PP/:src/:src/PP
   THtml* owner = GetOwner();
   if (!owner) return;

   TString pathext;
   TString inputdir = owner->GetInputPath();
   TString tok;
   Ssiz_t start = 0;
   while (inputdir.Tokenize(tok, start, THtml::GetDirDelimiter())) {
      if (pathext.Length())
         pathext += GetDirDelimiter();
      if (tok.EndsWith("\\"))
         tok.Remove(tok.Length() - 1);
      pathext += tok;
      if (path.BeginsWith(tok))
         pathext += GetDirDelimiter() + path;
      else
         pathext += GetDirDelimiter() + tok + "/" + path;
   }
   path = pathext;

}

//______________________________________________________________________________
void THtml::TFileDefinition::SplitClassIntoDirFile(const TString& clname, TString& dir,
                                                   TString& filename) const
{
   // Given a class name with a scope, split the class name into directory part
   // and file name: A::B::C becomes module B, filename C.
   TString token;
   Ssiz_t from = 0;
   filename = "";
   dir = "";
   while (clname.Tokenize(token, from, "::") ) {
      dir = filename;
      filename = token;
   }

   // convert from Scope, class to module, filename.h
   dir.ToLower();
}


//______________________________________________________________________________
bool THtml::TFileDefinition::GetDeclFileName(const TClass* cl, TString& out_filename,
                                             TString& out_fsys, TFileSysEntry** fse) const
{
   // Determine cl's declaration file name. Usually it's just
   // cl->GetDeclFileName(), but sometimes conversions need to be done
   // like include/ to abc/cde/inc/. If no declaration file name is
   // available, look for b/inc/C.h for class A::B::C. out_fsys will contain
   // the file system's (i.e. local machine's) full path name to the file.
   // The function returns false if the class's header file cannot be found.
   //
   // If your software cannot be mapped into this scheme then derive your
   // own class from TFileDefinition and pass it to THtml::SetFileDefinition().

   return GetFileName(cl, true, out_filename, out_fsys, fse);
}

//______________________________________________________________________________
bool THtml::TFileDefinition::GetImplFileName(const TClass* cl, TString& out_filename,
                                             TString& out_fsys, TFileSysEntry** fse) const
{
   // Determine cl's implementation file name. Usually it's just
   // cl->GetImplFileName(), but sometimes conversions need to be done.
   // If no implementation file name is available look for b/src/C.cxx for
   // class A::B::C. out_fsys will contain the file system's (i.e. local
   // machine's) full path name to the file.
   // The function returns false if the class's source file cannot be found.
   //
   // If your software cannot be mapped into this scheme then derive your
   // own class from TFileDefinition and pass it to THtml::SetFileDefinition().

   return GetFileName(cl, false, out_filename, out_fsys, fse);
}


//______________________________________________________________________________
void THtml::TFileDefinition::NormalizePath(TString& filename) const
{
   // Remove "/./" and collapse "/subdir/../" to "/"
   static const char* delim[] = {"/", "\\\\"};
   for (int i = 0; i < 2; ++i) {
      const char* d = delim[i];
      filename = filename.ReplaceAll(TString::Format("%c.%c", d[0], d[0]), TString(d[0]));
      TPRegexp reg(TString::Format("%s[^%s]+%s\\.\\.%s", d, d, d, d));
      while (reg.Substitute(filename, TString(d[0]), "", 0, 1)) {}
   }
   if (filename.BeginsWith("./") || filename.BeginsWith(".\\"))
      filename.Remove(0,2);
}


//______________________________________________________________________________
TString THtml::TFileDefinition::MatchFileSysName(TString& filename, TFileSysEntry** fse) const
{
   // Find filename in the list of system files; return the system file name
   // and change filename to the file name as included.
   // filename must be normalized (no "/./" etc) before calling.

   TList* bucket = GetOwner()->GetLocalFiles()->GetEntries().GetListForObject(gSystem->BaseName(filename));
   TString filesysname;
   if (bucket) {
      TIter iFS(bucket);
      TFileSysEntry* fsentry = 0;
      while ((fsentry = (TFileSysEntry*) iFS())) {
         if (!filename.EndsWith(fsentry->GetName()))
            continue;
         fsentry->GetFullName(filesysname, kTRUE); // get the short version
         if (!filename.EndsWith(filesysname)) {
            filesysname = "";
            continue;
         }
         filename = filesysname;
         fsentry->GetFullName(filesysname, kFALSE); // get the long version
         if (fse) *fse = fsentry;
         break;
      }
   }
   return filesysname;
}


//______________________________________________________________________________
bool THtml::TFileDefinition::GetFileName(const TClass* cl, bool decl,
                                         TString& out_filename, TString& out_fsys,
                                         TFileSysEntry** fse) const
{
   // Common implementation for GetDeclFileName(), GetImplFileName()

   out_fsys = "";

   if (!cl) {
      out_filename = "";
      return false;
   }

   TString possibleFileName;
   TString possiblePath;
   TString filesysname;

   TString clfile = decl ? cl->GetDeclFileName() : cl->GetImplFileName();
   NormalizePath(clfile);

   out_filename = clfile;
   if (clfile.Length()) {
      // check that clfile doesn't start with one of the include paths;
      // that's not what we want (include/TObject.h), we want the actual file
      // if it exists (core/base/inc/TObject.h)

      // special case for TMath namespace:
      if (clfile == "include/TMathBase.h") {
         clfile = "math/mathcore/inc/TMath.h";
         out_filename = clfile;
      }

      TString inclDir;
      TString inclPath(GetOwner()->GetPathInfo().fIncludePath);
      Ssiz_t pos = 0;
      Ssiz_t longestMatch = kNPOS;
      while (inclPath.Tokenize(inclDir, pos, GetOwner()->GetDirDelimiter())) {
         if (clfile.BeginsWith(inclDir) && (longestMatch == kNPOS || inclDir.Length() > longestMatch))
            longestMatch = inclDir.Length();
      }
      if (longestMatch != kNPOS) {
         clfile.Remove(0, longestMatch);
         if (clfile.BeginsWith("/") || clfile.BeginsWith("\\"))
            clfile.Remove(0, 1);
         TString asincl(clfile);
         GetOwner()->GetPathDefinition().GetFileNameFromInclude(asincl, clfile);
         out_filename = clfile;
      } else {
         // header file without a -Iinclude-dir prefix
         filesysname = MatchFileSysName(out_filename, fse);
         if (filesysname[0]) {
            clfile = out_filename;
         }
      }
   } else {
      // check for a file named like the class:
      filesysname = cl->GetName();
      int templateLevel = 0;
      Ssiz_t end = filesysname.Length();
      Ssiz_t start = end - 1;
      for (; start >= 0 && (templateLevel || filesysname[start] != ':'); --start) {
         if (filesysname[start] == '>')
            ++templateLevel;
         else if (filesysname[start] == '<') {
            --templateLevel;
            if (!templateLevel)
               end = start;
         }
      }
      filesysname = filesysname(start + 1, end - start - 1);
      if (decl)
         filesysname += ".h";
      else
         filesysname += ".cxx";
      out_filename = filesysname;
      filesysname = MatchFileSysName(out_filename, fse);
      if (filesysname[0]) {
         clfile = out_filename;
      }
   }

   if (!decl && !clfile.Length()) {
      // determine possible impl file name from the decl file name,
      // replacing ".whatever" by ".cxx", and looking for it in the known
      // file names
      TString declSysFileName;
      if (GetFileName(cl, true, filesysname, declSysFileName)) {
         filesysname = gSystem->BaseName(filesysname);
         Ssiz_t posExt = filesysname.Last('.');
         if (posExt != kNPOS)
            filesysname.Remove(posExt);
         filesysname += ".cxx";
         out_filename = filesysname;
         filesysname = MatchFileSysName(out_filename, fse);
         if (filesysname[0]) {
            clfile = out_filename;
         }
      }
   }

   if (clfile.Length() && !decl) {
      // Do not return the source file for these packages, even though we can find them.
      // THtml needs to have the class description in the source file if it finds the
      // source file, and these classes have their class descriptions in the header files.
      // THtml needs to be improved to collect all of a class' documentation before writing
      // it out, so it can take the class doc from the header even though a source exists.
      static const char* vetoClasses[] = {"math/mathcore/", "math/mathmore/", "math/genvector/",
                                          "math/minuit2/", "math/smatrix/"};
      for (unsigned int i = 0; i < sizeof(vetoClasses) / sizeof(char*); ++i) {
         if (clfile.Contains(vetoClasses[i])) {
            // of course there are exceptions from the exceptions:
            // TComplex and TRandom, TRandom1,...
            if (strcmp(cl->GetName(), "TComplex")
                && strcmp(cl->GetName(), "TMath")
                && strncmp(cl->GetName(), "TRandom", 7)) {
               out_filename = "";
               return false;
            } else break;
         }
      }
   }


   if (!clfile.Length()) {
      // determine possible decl file name from class + scope name:
      // A::B::C::myclass will result in possible file name myclass.h
      // in directory C/inc/
      out_filename = cl->GetName();
      if (!out_filename.Contains("::")) {
         out_filename = "";
         return false;
      }
      SplitClassIntoDirFile(out_filename, possiblePath, possibleFileName);

      // convert from Scope, class to module, filename.h
      if (possibleFileName.Length()) {
         if (decl)
            possibleFileName += ".h";
         else
            possibleFileName += ".cxx";
      }
      if (possiblePath.Length())
         possiblePath += "/";
      if (decl)
         possiblePath += "inc/";
      else
         possiblePath += "src/";
      out_filename = possiblePath + "/" + possibleFileName;
   } else {
      possiblePath = gSystem->DirName(clfile);
      possibleFileName = gSystem->BaseName(clfile);
   }

   if (possiblePath.Length())
      ExpandSearchPath(possiblePath);
   else possiblePath=".";

   out_fsys = gSystem->FindFile(possiblePath, possibleFileName, kReadPermission);
   if (out_fsys.Length()) {
      NormalizePath(out_fsys);
      return true;
   }
   out_filename = "";
   return false;
}

//______________________________________________________________________________
bool THtml::TPathDefinition::GetMacroPath(const TString& module, TString& out_dir) const
{
   // Determine the path to look for macros (see TDocMacroDirective) for
   // classes from a given module. If the path was sucessfully determined return true.
   // For ROOT, this directory is the "doc/macros" subdirectory of the module
   // directory; the path returned is GetDocDir(module) + "/macros".
   //
   // If your software cannot be mapped into this scheme then derive your
   // own class from TPathDefinition and pass it to THtml::SetPathDefinition().

   TString moduledoc;
   if (!GetDocDir(module, moduledoc))
      return false;
   if (moduledoc.EndsWith("\\"))
      moduledoc.Remove(moduledoc.Length() - 1);

   TString macropath(GetOwner()->GetMacroPath());
   TString macrodirpart;
   out_dir = "";
   Ssiz_t pos = 0;
   while (macropath.Tokenize(macrodirpart, pos, ":")) {
      out_dir += moduledoc + "/" + macrodirpart + ":";
   }
   return true;
}


//______________________________________________________________________________
bool THtml::TPathDefinition::GetDocDir(const TString& module, TString& doc_dir) const
{
   // Determine the module's documentation directory. If module is empty,
   // set doc_dir to the product's documentation directory.
   // If the path was sucessfuly determined return true.
   // For ROOT, this directory is the subdir "doc/" in the
   // module's path; the directory returned is module + "/doc".
   //
   // If your software cannot be mapped into this scheme then derive your
   // own class from TPathDefinition and pass it to THtml::SetPathDefinition().

   if (module.Length())
      doc_dir = module + "/";
   doc_dir += GetOwner()->GetPathInfo().fDocPath;
   return true;
}


//______________________________________________________________________________
bool THtml::TPathDefinition::GetIncludeAs(TClass* cl, TString& out_dir) const
{
   // Determine the path and filename used in an include statement for the
   // header file of the given class. E.g. the class ROOT::Math::Boost is
   // meant to be included as "Math/Genvector/Boost.h" - which is what
   // out_dir is set to. GetIncludeAs() returns whether the include
   // statement's path was successfully determined.
   //
   // Any leading directory part that is part of fIncludePath (see SetIncludePath)
   // will be removed. For ROOT, leading "include/" is removed; everything after
   // is the include path. Only classes from TMVA are different; they are included
   // as TMVA/ClassName.h.
   //
   // If your software cannot be mapped into this scheme then derive your
   // own class from TPathDefinition and pass it to THtml::SetPathDefinition().

   out_dir = "";
   if (!cl || !GetOwner()) return false;

   const char* clname = cl->GetName();
   TString hdr;
   if (!GetOwner()->GetDeclFileName(cl, kFALSE, hdr))
      return false;

   out_dir = hdr;
   bool includePathMatches = false;
   TString tok;
   Ssiz_t pos = 0;
   while (!includePathMatches && GetOwner()->GetPathInfo().fIncludePath.Tokenize(tok, pos, THtml::GetDirDelimiter()))
      if (out_dir.BeginsWith(tok)) {
         out_dir = hdr(tok.Length(), hdr.Length());
         if (out_dir[0] == '/' || out_dir[0] == '\\')
            out_dir.Remove(0, 1);
         includePathMatches = true;
      }

   if (!includePathMatches) {
      // We probably have a file super/module/inc/optional/filename.h.
      // That gets translated into optional/filename.h.
      // Assume that only one occurrence of "/inc/" exists in hdr.
      // If /inc/ is not part of the include file name then
      // just return the full path.
      // If we have matched any include path then this ROOT-only
      // algorithm is skipped!
      Ssiz_t posInc = hdr.Index("/inc/");
      if (posInc == kNPOS) return true;
      hdr.Remove(0, posInc + 5);
      out_dir = hdr;

      // TMVA and RooStats special treatment:
      // TMVA::Whatever claims to be in in math/tmva/inc/Whatever.h
      // but it needs to get included as TMVA/Whatever.h
      if (strstr(clname, "TMVA::"))
         out_dir.Prepend("TMVA/");
      if (strstr(clname, "RooStats::"))
         out_dir.Prepend("RooStats/");
   }

   return (out_dir.Length());
}


//______________________________________________________________________________
bool THtml::TPathDefinition::GetFileNameFromInclude(const char* included, TString& out_fsname) const
{
   // Set out_fsname to the full pathname corresponding to a file
   // included as "included". Return false if this file cannot be determined
   // or found. For ROOT, out_fsname corresponds to included prepended with
   // "include"; only THtml prefers to work on the original files, e.g.
   // core/base/inc/TObject.h instead of include/TObject.h, so the
   // default implementation searches the TFileSysDB for an entry with
   // basename(included) and with matching directory part, setting out_fsname
   // to the TFileSysEntry's path.

   if (!included) return false;

   out_fsname = included;

   if (!strncmp(included, "TMVA/", 5)) {
      out_fsname.Remove(0, 4);
      out_fsname.Prepend("tmva/inc");
      return true;
   }
   // special treatment for roostats (same as in TMVA)
   if (!strncmp(included, "RooStats/", 9)) {
      out_fsname.Remove(0, 8);
      out_fsname.Prepend("roofit/roostats/inc");
      return true;
   }

   TString incBase(gSystem->BaseName(included));
   TList* bucket = GetOwner()->GetLocalFiles()->GetEntries().GetListForObject(incBase);
   if (!bucket) return false;

   TString alldir(gSystem->DirName(included));
   TObjArray* arrSubDirs = alldir.Tokenize("/");
   TIter iEntry(bucket);<commit_msg>Fix up headers for HistFactory: they claim to be in include/RooStats/HistFactory/* which in fact maps to roofit/histfactory/inc. Fixes module doc RooFit/HistFactory and enabled doc of HistFactory classes: they were suppressed before because their header and source files couldn't be located.<commit_after>// @(#)root/html:$Id$
// Author: <NAME> (18/10/95), <NAME> (09/28/01)

/*************************************************************************
 * Copyright (C) 1995-2007, <NAME> and <NAME>.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

#include "THtml.h"
#include "RConfigure.h"
#include "Riostream.h"
#include "TBaseClass.h"
#include "TClass.h"
#include "TClassDocOutput.h"
#include "TClassEdit.h"
#include "TClassTable.h"
#include "TDataType.h"
#include "TDocInfo.h"
#include "TDocOutput.h"
#include "TEnv.h"
#include "TInterpreter.h"
#include "TObjString.h"
#include "TPRegexp.h"
#include "TRegexp.h"
#include "TROOT.h"
#include "TSystem.h"
#include "TThread.h"

#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <set>
#include <fstream>

THtml *gHtml = 0;

//______________________________________________________________________________
//______________________________________________________________________________
namespace {
   class THtmlThreadInfo {
   public:
      THtmlThreadInfo(THtml* html, bool force): fHtml(html), fForce(force) {}
      Bool_t GetForce() const {return fForce;}
      THtml* GetHtml() const {return fHtml;}

   private:
      THtml* fHtml;
      Bool_t fForce;
   };
};


//______________________________________________________________________________
THtml::THelperBase::~THelperBase()
{
   // Helper's destructor.
   // Check that no THtml object is attached to the helper - it might still need it!
   if (fHtml) {
      fHtml->HelperDeleted(this);
   }
}


//______________________________________________________________________________
void THtml::THelperBase::SetOwner(THtml* html) {
   // Set the THtml object owning this object; if it's already set to
   // a different THtml object than issue an error message and signal to
   // the currently set object that we are not belonging to it anymore.
   if (fHtml && html && html != fHtml) {
      Error("SetOwner()", "Object already owned by an THtml instance!");
      fHtml->HelperDeleted(this);
   }
   fHtml = html;
}


//______________________________________________________________________________
bool THtml::TModuleDefinition::GetModule(TClass* cl, TFileSysEntry* fse,
                                         TString& out_modulename) const
{
   // Set out_modulename to cl's module name; return true if it's valid.
   // If applicable, the module contains super modules separated by "/".
   //
   // ROOT takes the directory part of cl's implementation file name
   // (or declaration file name, if the implementation file name is empty),
   // removes the last subdirectory if it is "src/" or "inc/", and interprets
   // the remaining path as the module hierarchy, converting it to upper case.
   // hist/histpainter/src/THistPainter.cxx thus becomes the module
   // HIST/HISTPAINTER. (Node: some ROOT packages get special treatment.)
   // If the file cannot be mapped into this scheme, the class's library
   // name (without directories, leading "lib" prefix or file extensions)
   // ius taken as the module name. If the module cannot be determined it is
   // set to "USER" and false is returned.
   //
   // If your software cannot be mapped into this scheme then derive your
   // own class from TModuleDefinition and pass it to THtml::SetModuleDefinition().
   //
   // The fse parameter is used to determine the relevant part of the path, i.e.
   // to not include parent directories of a TFileSysRoot.

   out_modulename = "USER";
   if (!cl) return false;

   // Filename: impl or decl?
   TString filename;
   if (fse) fse->GetFullName(filename, kFALSE);
   else {
      if (!GetOwner()->GetImplFileName(cl, kFALSE, filename))
         if (!GetOwner()->GetDeclFileName(cl, kFALSE, filename))
            return false;
   }
   TString inputdir = GetOwner()->GetInputPath();
   TString tok;
   Ssiz_t start = 0;
   // For -Idir/sub and A.h in dir/sub/A.h, use sub as module name if
   // it would eb empty otehrwise.
   TString trailingInclude;
   while (inputdir.Tokenize(tok, start, THtml::GetDirDelimiter())) {
      if (filename.BeginsWith(tok)) {
         if (tok.EndsWith("/") || tok.EndsWith("\\"))
            tok.Remove(tok.Length() - 1);
         trailingInclude = gSystem->BaseName(tok);
         filename.Remove(0, tok.Length());
         break;
      }
   }

   // take the directory name without "/" or leading "."
   out_modulename = gSystem->DirName(filename);

   while (out_modulename[0] == '.')
      out_modulename.Remove(0, 1);
   out_modulename.ReplaceAll("\\", "/");
   while (out_modulename[0] == '/')
      out_modulename.Remove(0, 1);
   while (out_modulename.EndsWith("/"))
      out_modulename.Remove(out_modulename.Length() - 1);

   if (!out_modulename[0])
      out_modulename = trailingInclude;

   if (!out_modulename[0])
      out_modulename = trailingInclude;

   // remove "/src", "/inc"
   if (out_modulename.EndsWith("/src")
      || out_modulename.EndsWith("/inc"))
      out_modulename.Remove(out_modulename.Length() - 4, 4);
   else {
   // remove "/src/whatever", "/inc/whatever"
      Ssiz_t pos = out_modulename.Index("/src/");
      if (pos == kNPOS)
         pos = out_modulename.Index("/inc/");
      if (pos != kNPOS)
         out_modulename.Remove(pos);
   }

   while (out_modulename.EndsWith("/"))
      out_modulename.Remove(out_modulename.Length() - 1);

   // special treatment:
   if (out_modulename == "MATH/GENVECTOR")
      out_modulename = "MATHCORE";
   else if (out_modulename == "MATH/MATRIX")
      out_modulename = "SMATRIX";
   else if (!out_modulename.Length()) {
      const char* cname= cl->GetName();
      if (strstr(cname, "::SMatrix<") || strstr(cname, "::SVector<"))
         out_modulename = "SMATRIX";
      else if (strstr(cname, "::TArrayProxy<") || strstr(cname, "::TClaArrayProxy<")
               || strstr(cname, "::TImpProxy<") || strstr(cname, "::TClaImpProxy<"))
         out_modulename = "TREEPLAYER";
      else {
         // determine the module name from the library name:
         out_modulename = cl->GetSharedLibs();
         Ssiz_t pos = out_modulename.Index(' ');
         if (pos != kNPOS)
            out_modulename.Remove(pos, out_modulename.Length());
         if (out_modulename.BeginsWith("lib"))
            out_modulename.Remove(0,3);
         pos = out_modulename.Index('.');
         if (pos != kNPOS)
            out_modulename.Remove(pos, out_modulename.Length());

         if (!out_modulename.Length()) {
            out_modulename = "USER";
            return false;
         }
      }
   }

   return true;
}

//______________________________________________________________________________
void THtml::TFileDefinition::ExpandSearchPath(TString& path) const
{
   // Create all permutations of path and THtml's input path:
   // path being PP/ and THtml's input being .:include/:src/ gives
   // .:./PP/:include:include/PP/:src/:src/PP
   THtml* owner = GetOwner();
   if (!owner) return;

   TString pathext;
   TString inputdir = owner->GetInputPath();
   TString tok;
   Ssiz_t start = 0;
   while (inputdir.Tokenize(tok, start, THtml::GetDirDelimiter())) {
      if (pathext.Length())
         pathext += GetDirDelimiter();
      if (tok.EndsWith("\\"))
         tok.Remove(tok.Length() - 1);
      pathext += tok;
      if (path.BeginsWith(tok))
         pathext += GetDirDelimiter() + path;
      else
         pathext += GetDirDelimiter() + tok + "/" + path;
   }
   path = pathext;

}

//______________________________________________________________________________
void THtml::TFileDefinition::SplitClassIntoDirFile(const TString& clname, TString& dir,
                                                   TString& filename) const
{
   // Given a class name with a scope, split the class name into directory part
   // and file name: A::B::C becomes module B, filename C.
   TString token;
   Ssiz_t from = 0;
   filename = "";
   dir = "";
   while (clname.Tokenize(token, from, "::") ) {
      dir = filename;
      filename = token;
   }

   // convert from Scope, class to module, filename.h
   dir.ToLower();
}


//______________________________________________________________________________
bool THtml::TFileDefinition::GetDeclFileName(const TClass* cl, TString& out_filename,
                                             TString& out_fsys, TFileSysEntry** fse) const
{
   // Determine cl's declaration file name. Usually it's just
   // cl->GetDeclFileName(), but sometimes conversions need to be done
   // like include/ to abc/cde/inc/. If no declaration file name is
   // available, look for b/inc/C.h for class A::B::C. out_fsys will contain
   // the file system's (i.e. local machine's) full path name to the file.
   // The function returns false if the class's header file cannot be found.
   //
   // If your software cannot be mapped into this scheme then derive your
   // own class from TFileDefinition and pass it to THtml::SetFileDefinition().

   return GetFileName(cl, true, out_filename, out_fsys, fse);
}

//______________________________________________________________________________
bool THtml::TFileDefinition::GetImplFileName(const TClass* cl, TString& out_filename,
                                             TString& out_fsys, TFileSysEntry** fse) const
{
   // Determine cl's implementation file name. Usually it's just
   // cl->GetImplFileName(), but sometimes conversions need to be done.
   // If no implementation file name is available look for b/src/C.cxx for
   // class A::B::C. out_fsys will contain the file system's (i.e. local
   // machine's) full path name to the file.
   // The function returns false if the class's source file cannot be found.
   //
   // If your software cannot be mapped into this scheme then derive your
   // own class from TFileDefinition and pass it to THtml::SetFileDefinition().

   return GetFileName(cl, false, out_filename, out_fsys, fse);
}


//______________________________________________________________________________
void THtml::TFileDefinition::NormalizePath(TString& filename) const
{
   // Remove "/./" and collapse "/subdir/../" to "/"
   static const char* delim[] = {"/", "\\\\"};
   for (int i = 0; i < 2; ++i) {
      const char* d = delim[i];
      filename = filename.ReplaceAll(TString::Format("%c.%c", d[0], d[0]), TString(d[0]));
      TPRegexp reg(TString::Format("%s[^%s]+%s\\.\\.%s", d, d, d, d));
      while (reg.Substitute(filename, TString(d[0]), "", 0, 1)) {}
   }
   if (filename.BeginsWith("./") || filename.BeginsWith(".\\"))
      filename.Remove(0,2);
}


//______________________________________________________________________________
TString THtml::TFileDefinition::MatchFileSysName(TString& filename, TFileSysEntry** fse) const
{
   // Find filename in the list of system files; return the system file name
   // and change filename to the file name as included.
   // filename must be normalized (no "/./" etc) before calling.

   TList* bucket = GetOwner()->GetLocalFiles()->GetEntries().GetListForObject(gSystem->BaseName(filename));
   TString filesysname;
   if (bucket) {
      TIter iFS(bucket);
      TFileSysEntry* fsentry = 0;
      while ((fsentry = (TFileSysEntry*) iFS())) {
         if (!filename.EndsWith(fsentry->GetName()))
            continue;
         fsentry->GetFullName(filesysname, kTRUE); // get the short version
         if (!filename.EndsWith(filesysname)) {
            filesysname = "";
            continue;
         }
         filename = filesysname;
         fsentry->GetFullName(filesysname, kFALSE); // get the long version
         if (fse) *fse = fsentry;
         break;
      }
   }
   return filesysname;
}


//______________________________________________________________________________
bool THtml::TFileDefinition::GetFileName(const TClass* cl, bool decl,
                                         TString& out_filename, TString& out_fsys,
                                         TFileSysEntry** fse) const
{
   // Common implementation for GetDeclFileName(), GetImplFileName()

   out_fsys = "";

   if (!cl) {
      out_filename = "";
      return false;
   }

   TString possibleFileName;
   TString possiblePath;
   TString filesysname;

   TString clfile = decl ? cl->GetDeclFileName() : cl->GetImplFileName();
   NormalizePath(clfile);

   out_filename = clfile;
   if (clfile.Length()) {
      // check that clfile doesn't start with one of the include paths;
      // that's not what we want (include/TObject.h), we want the actual file
      // if it exists (core/base/inc/TObject.h)

      // special case for TMath namespace:
      if (clfile == "include/TMathBase.h") {
         clfile = "math/mathcore/inc/TMath.h";
         out_filename = clfile;
      }

      TString inclDir;
      TString inclPath(GetOwner()->GetPathInfo().fIncludePath);
      Ssiz_t pos = 0;
      Ssiz_t longestMatch = kNPOS;
      while (inclPath.Tokenize(inclDir, pos, GetOwner()->GetDirDelimiter())) {
         if (clfile.BeginsWith(inclDir) && (longestMatch == kNPOS || inclDir.Length() > longestMatch))
            longestMatch = inclDir.Length();
      }
      if (longestMatch != kNPOS) {
         clfile.Remove(0, longestMatch);
         if (clfile.BeginsWith("/") || clfile.BeginsWith("\\"))
            clfile.Remove(0, 1);
         TString asincl(clfile);
         GetOwner()->GetPathDefinition().GetFileNameFromInclude(asincl, clfile);
         out_filename = clfile;
      } else {
         // header file without a -Iinclude-dir prefix
         filesysname = MatchFileSysName(out_filename, fse);
         if (filesysname[0]) {
            clfile = out_filename;
         }
      }
   } else {
      // check for a file named like the class:
      filesysname = cl->GetName();
      int templateLevel = 0;
      Ssiz_t end = filesysname.Length();
      Ssiz_t start = end - 1;
      for (; start >= 0 && (templateLevel || filesysname[start] != ':'); --start) {
         if (filesysname[start] == '>')
            ++templateLevel;
         else if (filesysname[start] == '<') {
            --templateLevel;
            if (!templateLevel)
               end = start;
         }
      }
      filesysname = filesysname(start + 1, end - start - 1);
      if (decl)
         filesysname += ".h";
      else
         filesysname += ".cxx";
      out_filename = filesysname;
      filesysname = MatchFileSysName(out_filename, fse);
      if (filesysname[0]) {
         clfile = out_filename;
      }
   }

   if (!decl && !clfile.Length()) {
      // determine possible impl file name from the decl file name,
      // replacing ".whatever" by ".cxx", and looking for it in the known
      // file names
      TString declSysFileName;
      if (GetFileName(cl, true, filesysname, declSysFileName)) {
         filesysname = gSystem->BaseName(filesysname);
         Ssiz_t posExt = filesysname.Last('.');
         if (posExt != kNPOS)
            filesysname.Remove(posExt);
         filesysname += ".cxx";
         out_filename = filesysname;
         filesysname = MatchFileSysName(out_filename, fse);
         if (filesysname[0]) {
            clfile = out_filename;
         }
      }
   }

   if (clfile.Length() && !decl) {
      // Do not return the source file for these packages, even though we can find them.
      // THtml needs to have the class description in the source file if it finds the
      // source file, and these classes have their class descriptions in the header files.
      // THtml needs to be improved to collect all of a class' documentation before writing
      // it out, so it can take the class doc from the header even though a source exists.
      static const char* vetoClasses[] = {"math/mathcore/", "math/mathmore/", "math/genvector/",
                                          "math/minuit2/", "math/smatrix/"};
      for (unsigned int i = 0; i < sizeof(vetoClasses) / sizeof(char*); ++i) {
         if (clfile.Contains(vetoClasses[i])) {
            // of course there are exceptions from the exceptions:
            // TComplex and TRandom, TRandom1,...
            if (strcmp(cl->GetName(), "TComplex")
                && strcmp(cl->GetName(), "TMath")
                && strncmp(cl->GetName(), "TRandom", 7)) {
               out_filename = "";
               return false;
            } else break;
         }
      }
   }


   if (!clfile.Length()) {
      // determine possible decl file name from class + scope name:
      // A::B::C::myclass will result in possible file name myclass.h
      // in directory C/inc/
      out_filename = cl->GetName();
      if (!out_filename.Contains("::")) {
         out_filename = "";
         return false;
      }
      SplitClassIntoDirFile(out_filename, possiblePath, possibleFileName);

      // convert from Scope, class to module, filename.h
      if (possibleFileName.Length()) {
         if (decl)
            possibleFileName += ".h";
         else
            possibleFileName += ".cxx";
      }
      if (possiblePath.Length())
         possiblePath += "/";
      if (decl)
         possiblePath += "inc/";
      else
         possiblePath += "src/";
      out_filename = possiblePath + "/" + possibleFileName;
   } else {
      possiblePath = gSystem->DirName(clfile);
      possibleFileName = gSystem->BaseName(clfile);
   }

   if (possiblePath.Length())
      ExpandSearchPath(possiblePath);
   else possiblePath=".";

   out_fsys = gSystem->FindFile(possiblePath, possibleFileName, kReadPermission);
   if (out_fsys.Length()) {
      NormalizePath(out_fsys);
      return true;
   }
   out_filename = "";
   return false;
}

//______________________________________________________________________________
bool THtml::TPathDefinition::GetMacroPath(const TString& module, TString& out_dir) const
{
   // Determine the path to look for macros (see TDocMacroDirective) for
   // classes from a given module. If the path was sucessfully determined return true.
   // For ROOT, this directory is the "doc/macros" subdirectory of the module
   // directory; the path returned is GetDocDir(module) + "/macros".
   //
   // If your software cannot be mapped into this scheme then derive your
   // own class from TPathDefinition and pass it to THtml::SetPathDefinition().

   TString moduledoc;
   if (!GetDocDir(module, moduledoc))
      return false;
   if (moduledoc.EndsWith("\\"))
      moduledoc.Remove(moduledoc.Length() - 1);

   TString macropath(GetOwner()->GetMacroPath());
   TString macrodirpart;
   out_dir = "";
   Ssiz_t pos = 0;
   while (macropath.Tokenize(macrodirpart, pos, ":")) {
      out_dir += moduledoc + "/" + macrodirpart + ":";
   }
   return true;
}


//______________________________________________________________________________
bool THtml::TPathDefinition::GetDocDir(const TString& module, TString& doc_dir) const
{
   // Determine the module's documentation directory. If module is empty,
   // set doc_dir to the product's documentation directory.
   // If the path was sucessfuly determined return true.
   // For ROOT, this directory is the subdir "doc/" in the
   // module's path; the directory returned is module + "/doc".
   //
   // If your software cannot be mapped into this scheme then derive your
   // own class from TPathDefinition and pass it to THtml::SetPathDefinition().

   if (module.Length())
      doc_dir = module + "/";
   doc_dir += GetOwner()->GetPathInfo().fDocPath;
   return true;
}


//______________________________________________________________________________
bool THtml::TPathDefinition::GetIncludeAs(TClass* cl, TString& out_dir) const
{
   // Determine the path and filename used in an include statement for the
   // header file of the given class. E.g. the class ROOT::Math::Boost is
   // meant to be included as "Math/Genvector/Boost.h" - which is what
   // out_dir is set to. GetIncludeAs() returns whether the include
   // statement's path was successfully determined.
   //
   // Any leading directory part that is part of fIncludePath (see SetIncludePath)
   // will be removed. For ROOT, leading "include/" is removed; everything after
   // is the include path. Only classes from TMVA are different; they are included
   // as TMVA/ClassName.h.
   //
   // If your software cannot be mapped into this scheme then derive your
   // own class from TPathDefinition and pass it to THtml::SetPathDefinition().

   out_dir = "";
   if (!cl || !GetOwner()) return false;

   const char* clname = cl->GetName();
   TString hdr;
   if (!GetOwner()->GetDeclFileName(cl, kFALSE, hdr))
      return false;

   out_dir = hdr;
   bool includePathMatches = false;
   TString tok;
   Ssiz_t pos = 0;
   while (!includePathMatches && GetOwner()->GetPathInfo().fIncludePath.Tokenize(tok, pos, THtml::GetDirDelimiter()))
      if (out_dir.BeginsWith(tok)) {
         out_dir = hdr(tok.Length(), hdr.Length());
         if (out_dir[0] == '/' || out_dir[0] == '\\')
            out_dir.Remove(0, 1);
         includePathMatches = true;
      }

   if (!includePathMatches) {
      // We probably have a file super/module/inc/optional/filename.h.
      // That gets translated into optional/filename.h.
      // Assume that only one occurrence of "/inc/" exists in hdr.
      // If /inc/ is not part of the include file name then
      // just return the full path.
      // If we have matched any include path then this ROOT-only
      // algorithm is skipped!
      Ssiz_t posInc = hdr.Index("/inc/");
      if (posInc == kNPOS) return true;
      hdr.Remove(0, posInc + 5);
      out_dir = hdr;

      // TMVA and RooStats special treatment:
      // TMVA::Whatever claims to be in in math/tmva/inc/Whatever.h
      // but it needs to get included as TMVA/Whatever.h
      if (strstr(clname, "TMVA::"))
         out_dir.Prepend("TMVA/");
      if (strstr(clname, "RooStats::"))
         out_dir.Prepend("RooStats/");
   }

   return (out_dir.Length());
}


//______________________________________________________________________________
bool THtml::TPathDefinition::GetFileNameFromInclude(const char* included, TString& out_fsname) const
{
   // Set out_fsname to the full pathname corresponding to a file
   // included as "included". Return false if this file cannot be determined
   // or found. For ROOT, out_fsname corresponds to included prepended with
   // "include"; only THtml prefers to work on the original files, e.g.
   // core/base/inc/TObject.h instead of include/TObject.h, so the
   // default implementation searches the TFileSysDB for an entry with
   // basename(included) and with matching directory part, setting out_fsname
   // to the TFileSysEntry's path.

   if (!included) return false;

   out_fsname = included;

   if (!strncmp(included, "TMVA/", 5)) {
      out_fsname.Remove(0, 4);
      out_fsname.Prepend("tmva/inc");
      return true;
   }
   // special treatment for nested histfactory
   if (!strncmp(included, "RooStats/HistFactory/", 21)) {
      out_fsname.Remove(0, 20);
      out_fsname.Prepend("roofit/histfactory/inc");
      return true;
   }
   // special treatment for roostats
   if (!strncmp(included, "RooStats/", 9)) {
      out_fsname.Remove(0, 8);
      out_fsname.Prepend("roofit/roostats/inc");
      return true;
   }

   TString incBase(gSystem->BaseName(included));
   TList* bucket = GetOwner()->GetLocalFiles()->GetEntries().GetListForObject(incBase);
   if (!bucket) return false;

   TString alldir(gSystem->DirName(included));
   TObjArray* arrSubDirs = alldir.Tokenize("/");
   TIter iEntry(bucket);
{'fraction_non_alphanumeric': 0.12590387960847604, 'fraction_numerical': 0.00358407914317453, 'mean_word_length': 2.7556675062972293, 'pattern_counts': {'":': 20, '<': 39, '<?xml version=': 0, '>': 111, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '464620', 'n_tokens_mistral': 14127, 'n_tokens_neox': 13765, 'n_words': 4046}