<commit_before>//------------------------------------------------------------------------------
// CLING - the C++ LLVM-based InterpreterG :)
// version: $Id$
// author:  <NAME> <<EMAIL>>
//------------------------------------------------------------------------------

#include "cling/Interpreter/ClangInternalState.h"

#include "clang/AST/ASTContext.h"
#include "clang/AST/RecursiveASTVisitor.h"
#include "clang/Basic/SourceManager.h"

#include "llvm/ADT/SmallString.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/Signals.h"

#include <cstdio>
#include <string>
#include <time.h>

using namespace clang;

namespace cling {
  ClangInternalState::ClangInternalState(ASTContext& C, llvm::Module& M, 
                                         const std::string& name)
    : m_ASTContext(C), m_Module(M), m_DiffCommand("diff -u "), m_Name(name) {
    store();
  }

  ClangInternalState::~ClangInternalState() {
    // cleanup the temporary files:
    remove(m_LookupTablesFile.c_str());
    remove(m_IncludedFilesFile.c_str());
    remove(m_ASTFile.c_str());
    remove(m_LLVMModuleFile.c_str());
  }

  void ClangInternalState::store() {
    m_LookupTablesOS.reset(createOutputFile("lookup", &m_LookupTablesFile));
    m_IncludedFilesOS.reset(createOutputFile("included", &m_IncludedFilesFile));
    m_ASTOS.reset(createOutputFile("ast", &m_ASTFile));
    m_LLVMModuleOS.reset(createOutputFile("module", &m_LLVMModuleFile));
    
    printLookupTables(*m_LookupTablesOS.get(), m_ASTContext);
    printIncludedFiles(*m_IncludedFilesOS.get(), 
                       m_ASTContext.getSourceManager());
    printAST(*m_ASTOS.get(), m_ASTContext);
    printLLVMModule(*m_LLVMModuleOS.get(), m_Module);
  }
  namespace {
    std::string getCurrentTimeAsString() {
      time_t rawtime;
      struct tm * timeinfo;
      char buffer [80];

      time (&rawtime);
      timeinfo = localtime (&rawtime);

      strftime (buffer, 80, "%I_%M_%S", timeinfo);
      return buffer;
    }
  }

  // Copied with modifications from CompilerInstance.cpp
  llvm::raw_fd_ostream* 
  ClangInternalState::createOutputFile(llvm::StringRef OutFile,
                                       std::string *TempPathName/*=0*/,
                                       bool RemoveFileOnSignal/*=true*/) {
    llvm::OwningPtr<llvm::raw_fd_ostream> OS;
    std::string OSFile;
    llvm::SmallString<256> OutputPath;
    llvm::sys::path::system_temp_directory(/*erasedOnReboot*/false, OutputPath);

    // Only create the temporary if the parent directory exists (or create
    // missing directories is true) and we can actually write to OutPath,
    // otherwise we want to fail early.
    llvm::SmallString<256> TempPath(OutputPath);
    llvm::sys::fs::make_absolute(TempPath);
    assert(llvm::sys::fs::is_directory(TempPath.str()) && "Must be a folder.");
    // Create a temporary file.
    llvm::sys::path::append(TempPath, OutFile);
    TempPath += "-" + getCurrentTimeAsString();
    TempPath += "-%%%%%%%%";
    int fd;
    if (llvm::sys::fs::createUniqueFile(TempPath.str(), fd, TempPath)
        == llvm::errc::success) {
      OS.reset(new llvm::raw_fd_ostream(fd, /*shouldClose=*/true));
      OSFile = TempPath.str();
    }

    // Make sure the out stream file gets removed if we crash.
    if (RemoveFileOnSignal)
      llvm::sys::RemoveFileOnSignal(OSFile);

    if (TempPathName)
      *TempPathName = OSFile;

    return OS.take();
  }

  void ClangInternalState::compare(ClangInternalState& other) {
    std::string differences = "";
    if (differentContent(m_LookupTablesFile, other.m_LookupTablesFile, 
                         differences)) {
      llvm::errs() << "Differences in the lookup tables\n";
      llvm::errs() << differences << "\n";
      differences = "";
    }

    if (differentContent(m_IncludedFilesFile, other.m_IncludedFilesFile, 
                         differences)) {
      llvm::errs() << "Differences in the included files\n";
      llvm::errs() << differences << "\n";
      differences = "";
    }

    if (differentContent(m_ASTFile, other.m_ASTFile, differences)) {
      llvm::errs() << "Differences in the AST \n";
      llvm::errs() << differences << "\n";
      differences = "";
    }

    if (differentContent(m_LLVMModuleFile, other.m_LLVMModuleFile, differences)){
      llvm::errs() << "Differences in the llvm Module \n";
      llvm::errs() << differences << "\n";
      differences = "";
    }
  }

  bool ClangInternalState::differentContent(const std::string& file1, 
                                            const std::string& file2, 
                                            std::string& differences) const {
    FILE* pipe = popen((m_DiffCommand + file1 + " " + file2).c_str() , "r");
    assert(pipe && "Error creating the pipe");
    assert(differences.empty() && "Must be empty");

    char buffer[128];
    while(!feof(pipe)) {
      if(fgets(buffer, 128, pipe) != NULL)
        differences += buffer;
    }
    pclose(pipe);
    return !differences.empty();
  }


  class DumpLookupTables : public RecursiveASTVisitor<DumpLookupTables> {
  private:
    //llvm::raw_ostream& m_OS;
  public:
    //DumpLookupTables(llvm::raw_ostream& OS) : m_OS(OS) { }
    DumpLookupTables(llvm::raw_ostream&) { }
    bool VisitDeclContext(DeclContext* DC) {
      //DC->dumpLookups(m_OS);
      return true;
    }
  };

  void ClangInternalState::printLookupTables(llvm::raw_ostream& Out, 
                                             ASTContext& C) {
    DumpLookupTables dumper(Out);
    dumper.TraverseDecl(C.getTranslationUnitDecl());
  }

  void ClangInternalState::printIncludedFiles(llvm::raw_ostream& Out, 
                                              SourceManager& SM) {
    for (clang::SourceManager::fileinfo_iterator I = SM.fileinfo_begin(),
           E = SM.fileinfo_end(); I != E; ++I) {
      const clang::SrcMgr::ContentCache &C = *I->second;
      const clang::FileEntry *FE = C.OrigEntry;
      // Our error recovery purges the cache of the FileEntry, but keeps
      // the FileEntry's pointer so that if it was used by smb (like the
      // SourceManager) it wouldn't be dangling. In that case we shouldn't
      // print the FileName, because semantically it is not there.
      if (!FE->getSize() && !FE->getModificationTime())
        continue;
      std::string fileName(FE->getName());
      if (!(fileName.compare(0, 5, "/usr/") == 0 &&
            fileName.find("/bits/") != std::string::npos)) {
        Out << fileName << '\n';
      }
    }
  }

  void ClangInternalState::printAST(llvm::raw_ostream& Out, ASTContext& C) {
    TranslationUnitDecl* TU = C.getTranslationUnitDecl();
    unsigned Indentation = 0;
    bool PrintInstantiation = false;
    std::string ErrMsg;
    clang::PrintingPolicy policy = C.getPrintingPolicy();
    TU->print(Out, policy, Indentation, PrintInstantiation);
    // TODO: For future when we relpace the bump allocation with slab.
    //
    //Out << "Allocated memory: " << C.getAllocatedMemory();
    //Out << "Side table allocated memory: " << C.getSideTableAllocatedMemory();
    Out.flush();
  }

  void ClangInternalState::printLLVMModule(llvm::raw_ostream& Out, 
                                           llvm::Module& M) {
    M.print(Out, /*AssemblyAnnotationWriter*/ 0);
  }
} // end namespace cling
<commit_msg>Dump the lookup tables, too.<commit_after>//------------------------------------------------------------------------------
// CLING - the C++ LLVM-based InterpreterG :)
// version: $Id$
// author:  <NAME> <<EMAIL>>
//------------------------------------------------------------------------------

#include "cling/Interpreter/ClangInternalState.h"

#include "clang/AST/ASTContext.h"
#include "clang/AST/RecursiveASTVisitor.h"
#include "clang/Basic/SourceManager.h"

#include "llvm/ADT/SmallString.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/Signals.h"

#include <cstdio>
#include <string>
#include <time.h>

using namespace clang;

namespace cling {
  ClangInternalState::ClangInternalState(ASTContext& C, llvm::Module& M, 
                                         const std::string& name)
    : m_ASTContext(C), m_Module(M), m_DiffCommand("diff -u "), m_Name(name) {
    store();
  }

  ClangInternalState::~ClangInternalState() {
    // cleanup the temporary files:
    remove(m_LookupTablesFile.c_str());
    remove(m_IncludedFilesFile.c_str());
    remove(m_ASTFile.c_str());
    remove(m_LLVMModuleFile.c_str());
  }

  void ClangInternalState::store() {
    m_LookupTablesOS.reset(createOutputFile("lookup", &m_LookupTablesFile));
    m_IncludedFilesOS.reset(createOutputFile("included", &m_IncludedFilesFile));
    m_ASTOS.reset(createOutputFile("ast", &m_ASTFile));
    m_LLVMModuleOS.reset(createOutputFile("module", &m_LLVMModuleFile));
    
    printLookupTables(*m_LookupTablesOS.get(), m_ASTContext);
    printIncludedFiles(*m_IncludedFilesOS.get(), 
                       m_ASTContext.getSourceManager());
    printAST(*m_ASTOS.get(), m_ASTContext);
    printLLVMModule(*m_LLVMModuleOS.get(), m_Module);
  }
  namespace {
    std::string getCurrentTimeAsString() {
      time_t rawtime;
      struct tm * timeinfo;
      char buffer [80];

      time (&rawtime);
      timeinfo = localtime (&rawtime);

      strftime (buffer, 80, "%I_%M_%S", timeinfo);
      return buffer;
    }
  }

  // Copied with modifications from CompilerInstance.cpp
  llvm::raw_fd_ostream* 
  ClangInternalState::createOutputFile(llvm::StringRef OutFile,
                                       std::string *TempPathName/*=0*/,
                                       bool RemoveFileOnSignal/*=true*/) {
    llvm::OwningPtr<llvm::raw_fd_ostream> OS;
    std::string OSFile;
    llvm::SmallString<256> OutputPath;
    llvm::sys::path::system_temp_directory(/*erasedOnReboot*/false, OutputPath);

    // Only create the temporary if the parent directory exists (or create
    // missing directories is true) and we can actually write to OutPath,
    // otherwise we want to fail early.
    llvm::SmallString<256> TempPath(OutputPath);
    llvm::sys::fs::make_absolute(TempPath);
    assert(llvm::sys::fs::is_directory(TempPath.str()) && "Must be a folder.");
    // Create a temporary file.
    llvm::sys::path::append(TempPath, OutFile);
    TempPath += "-" + getCurrentTimeAsString();
    TempPath += "-%%%%%%%%";
    int fd;
    if (llvm::sys::fs::createUniqueFile(TempPath.str(), fd, TempPath)
        == llvm::errc::success) {
      OS.reset(new llvm::raw_fd_ostream(fd, /*shouldClose=*/true));
      OSFile = TempPath.str();
    }

    // Make sure the out stream file gets removed if we crash.
    if (RemoveFileOnSignal)
      llvm::sys::RemoveFileOnSignal(OSFile);

    if (TempPathName)
      *TempPathName = OSFile;

    return OS.take();
  }

  void ClangInternalState::compare(ClangInternalState& other) {
    std::string differences = "";
    if (differentContent(m_LookupTablesFile, other.m_LookupTablesFile, 
                         differences)) {
      llvm::errs() << "Differences in the lookup tables\n";
      llvm::errs() << differences << "\n";
      differences = "";
    }

    if (differentContent(m_IncludedFilesFile, other.m_IncludedFilesFile, 
                         differences)) {
      llvm::errs() << "Differences in the included files\n";
      llvm::errs() << differences << "\n";
      differences = "";
    }

    if (differentContent(m_ASTFile, other.m_ASTFile, differences)) {
      llvm::errs() << "Differences in the AST \n";
      llvm::errs() << differences << "\n";
      differences = "";
    }

    if (differentContent(m_LLVMModuleFile, other.m_LLVMModuleFile, differences)){
      llvm::errs() << "Differences in the llvm Module \n";
      llvm::errs() << differences << "\n";
      differences = "";
    }
  }

  bool ClangInternalState::differentContent(const std::string& file1, 
                                            const std::string& file2, 
                                            std::string& differences) const {
    FILE* pipe = popen((m_DiffCommand + file1 + " " + file2).c_str() , "r");
    assert(pipe && "Error creating the pipe");
    assert(differences.empty() && "Must be empty");

    char buffer[128];
    while(!feof(pipe)) {
      if(fgets(buffer, 128, pipe) != NULL)
        differences += buffer;
    }
    pclose(pipe);
    return !differences.empty();
  }


  class DumpLookupTables : public RecursiveASTVisitor<DumpLookupTables> {
  private:
    llvm::raw_ostream& m_OS;
  public:
    DumpLookupTables(llvm::raw_ostream& OS) : m_OS(OS) { }
    bool VisitDecl(Decl* D) {
      if (DeclContext* DC = dyn_cast<DeclContext>(D))
        VisitDeclContext(DC);
      return true;
    }

    bool VisitDeclContext(DeclContext* DC) {
      DC->dumpLookups(m_OS);
      return true;
    }
  };

  void ClangInternalState::printLookupTables(llvm::raw_ostream& Out, 
                                             ASTContext& C) {
    DumpLookupTables dumper(Out);
    dumper.TraverseDecl(C.getTranslationUnitDecl());
  }

  void ClangInternalState::printIncludedFiles(llvm::raw_ostream& Out, 
                                              SourceManager& SM) {
    for (clang::SourceManager::fileinfo_iterator I = SM.fileinfo_begin(),
           E = SM.fileinfo_end(); I != E; ++I) {
      const clang::SrcMgr::ContentCache &C = *I->second;
      const clang::FileEntry *FE = C.OrigEntry;
      // Our error recovery purges the cache of the FileEntry, but keeps
      // the FileEntry's pointer so that if it was used by smb (like the
      // SourceManager) it wouldn't be dangling. In that case we shouldn't
      // print the FileName, because semantically it is not there.
      if (!FE->getSize() && !FE->getModificationTime())
        continue;
      std::string fileName(FE->getName());
      if (!(fileName.compare(0, 5, "/usr/") == 0 &&
            fileName.find("/bits/") != std::string::npos)) {
        Out << fileName << '\n';
      }
    }
  }

  void ClangInternalState::printAST(llvm::raw_ostream& Out, ASTContext& C) {
    TranslationUnitDecl* TU = C.getTranslationUnitDecl();
    unsigned Indentation = 0;
    bool PrintInstantiation = false;
    std::string ErrMsg;
    clang::PrintingPolicy policy = C.getPrintingPolicy();
    TU->print(Out, policy, Indentation, PrintInstantiation);
    // TODO: For future when we relpace the bump allocation with slab.
    //
    //Out << "Allocated memory: " << C.getAllocatedMemory();
    //Out << "Side table allocated memory: " << C.getSideTableAllocatedMemory();
    Out.flush();
  }

  void ClangInternalState::printLLVMModule(llvm::raw_ostream& Out, 
                                           llvm::Module& M) {
    M.print(Out, /*AssemblyAnnotationWriter*/ 0);
  }
} // end namespace cling

{'fraction_non_alphanumeric': 0.15764139590854392, 'fraction_numerical': 0.003476400588313946, 'mean_word_length': 2.6981458590852907, 'pattern_counts': {'":': 0, '<': 96, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2350466', 'n_tokens_mistral': 4576, 'n_tokens_neox': 4452, 'n_words': 941}