<commit_before>//------------------------------------------------------------------------------
// CLING - the C++ LLVM-based InterpreterG :)
// author:  <NAME> <<EMAIL>>
//
// This file is dual-licensed: you can choose to license it under the University
// of Illinois Open Source License or the GNU Lesser General Public License. See
// LICENSE.TXT for details.
//------------------------------------------------------------------------------

#include "IncrementalParser.h"

#include "AutoSynthesizer.h"
#include "BackendPass.h"
#include "CheckEmptyTransactionTransformer.h"
#include "DeclCollector.h"
#include "DeclExtractor.h"
#include "DynamicLookup.h"
#include "IncrementalExecutor.h"
#include "NullDerefProtectionTransformer.h"
#include "ValueExtractionSynthesizer.h"
#include "TransactionPool.h"
#include "TransactionUnloader.h"
#include "ValuePrinterSynthesizer.h"
#include "cling/Interpreter/CIFactory.h"
#include "cling/Interpreter/Interpreter.h"
#include "cling/Interpreter/InterpreterCallbacks.h"
#include "cling/Interpreter/Transaction.h"

#include "clang/AST/Attr.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/Decl.h"
#include "clang/AST/DeclGroup.h"
#include "clang/AST/RecursiveASTVisitor.h"
#include "clang/Basic/FileManager.h"
#include "clang/CodeGen/ModuleBuilder.h"
#include "clang/Parse/Parser.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Sema/Sema.h"
#include "clang/Sema/SemaDiagnostic.h"
#include "clang/Serialization/ASTWriter.h"

#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/CrashRecoveryContext.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/raw_os_ostream.h"

#include <iostream>
#include <stdio.h>
#include <sstream>

// Include the necessary headers to interface with the Windows registry and
// environment.
#ifdef _MSC_VER
  #define WIN32_LEAN_AND_MEAN
  #define NOGDI
  #define NOMINMAX
  #include <Windows.h>
  #include <sstream>
  #define popen _popen
  #define pclose _pclose
  #pragma comment(lib, "Advapi32.lib")
#endif

using namespace clang;

namespace {
  ///\brief Check the compile-time C++ ABI version vs the run-time ABI version,
  /// a mismatch could cause havoc. Reports if ABI versions differ.
  static void CheckABICompatibility(clang::CompilerInstance* CI) {
#ifdef __GLIBCXX__
# define CLING_CXXABIV __GLIBCXX__
# define CLING_CXXABIS "__GLIBCXX__"
#elif _LIBCPP_VERSION
# define CLING_CXXABIV _LIBCPP_VERSION
# define CLING_CXXABIS "_LIBCPP_VERSION"
#elif defined (_MSC_VER)
    // For MSVC we do not use CLING_CXXABI*
#else
# define CLING_CXXABIV -1 // intentionally invalid macro name
# define CLING_CXXABIS "-1" // intentionally invalid macro name
    llvm::errs()
      << "Warning in cling::CIFactory::createCI():\n  "
      "C++ ABI check not implemented for this standard library\n";
    return;
#endif
#ifdef _MSC_VER
    HKEY regVS;
    int VSVersion = (_MSC_VER / 100) - 6;
    std::stringstream subKey;
    subKey << "VisualStudio.DTE." << VSVersion << ".0";
    if (RegOpenKeyEx(HKEY_CLASSES_ROOT, subKey.str().c_str(), 0, KEY_READ, &regVS) == ERROR_SUCCESS) {
      RegCloseKey(regVS);
    }
    else {
      llvm::errs()
        << "Warning in cling::CIFactory::createCI():\n  "
        "Possible C++ standard library mismatch, compiled with Visual Studio v"
        << VSVersion << ".0,\n"
        "but this version of Visual Studio was not found in your system's registry.\n";
    }
#else

  struct EarlyReturnWarn {
    bool shouldWarn = true;
    ~EarlyReturnWarn() {
      if (shouldWarn) {
        llvm::errs()
          << "Warning in cling::IncrementalParser::CheckABICompatibility():\n  "
             "Possible C++ standard library mismatch, compiled with "
             CLING_CXXABIS " v" << CLING_CXXABIV
          << " but extraction of runtime standard library version failed.\n";
      }
    }
  } warnAtReturn;
  clang::Preprocessor& PP = CI->getPreprocessor();
  clang::IdentifierInfo* II = PP.getIdentifierInfo(CLING_CXXABIS);
  if (!II)
    return;
  const clang::DefMacroDirective* MD
    = llvm::dyn_cast<clang::DefMacroDirective>(PP.getMacroDirective(II));
  if (!MD)
    return;
  const clang::MacroInfo* MI = MD->getMacroInfo();
  if (!MI || MI->getNumTokens() != 1)
    return;
  const clang::Token& Tok = *MI->tokens_begin();
  if (!Tok.isLiteral())
    return;
  if (!Tok.getLength() || !Tok.getLiteralData())
    return;

  std::string cxxabivStr;
  {
     llvm::raw_string_ostream cxxabivStrStrm(cxxabivStr);
     cxxabivStrStrm << CLING_CXXABIV;
  }
  llvm::StringRef tokStr(Tok.getLiteralData(), Tok.getLength());

  warnAtReturn.shouldWarn = false;
  if (!tokStr.equals(cxxabivStr)) {
    llvm::errs()
      << "Warning in cling::IncrementalParser::CheckABICompatibility():\n  "
        "C++ ABI mismatch, compiled with "
        CLING_CXXABIS " v" << CLING_CXXABIV
      << " running with v" << tokStr << "\n";
  }
#endif
#undef CLING_CXXABIV
#undef CLING_CXXABIS
  }
} // unnamed namespace

namespace cling {
  IncrementalParser::IncrementalParser(Interpreter* interp,
                                       int argc, const char* const *argv,
                                       const char* llvmdir):
    m_Interpreter(interp), m_Consumer(0), m_ModuleNo(0) {

    CompilerInstance* CI = CIFactory::createCI("", argc, argv, llvmdir);
    assert(CI && "CompilerInstance is (null)!");

    m_Consumer = dyn_cast<DeclCollector>(&CI->getSema().getASTConsumer());
    assert(m_Consumer && "Expected ChainedConsumer!");

    m_CI.reset(CI);

    if (CI->getFrontendOpts().ProgramAction != clang::frontend::ParseSyntaxOnly){
      m_CodeGen.reset(CreateLLVMCodeGen(CI->getDiagnostics(), "cling-module-0",
                                        CI->getCodeGenOpts(),
                                        CI->getTargetOpts(),
                                        *m_Interpreter->getLLVMContext()
                                        ));
    }

    initializeVirtualFile();

    // Add transformers to the IncrementalParser, which owns them
    Sema* TheSema = &CI->getSema();
    // Register the AST Transformers
    m_ASTTransformers.push_back(new AutoSynthesizer(TheSema));
    m_ASTTransformers.push_back(new EvaluateTSynthesizer(TheSema));
    m_ASTTransformers.push_back(new ValuePrinterSynthesizer(TheSema, 0));
    m_ASTTransformers.push_back(new DeclExtractor(TheSema));
    m_ASTTransformers.push_back(new ValueExtractionSynthesizer(TheSema));
    m_ASTTransformers.push_back(new NullDerefProtectionTransformer(TheSema));
    m_ASTTransformers.push_back(new CheckEmptyTransactionTransformer(TheSema));


#ifdef _LIBCPP_VERSION
    // libc++ relies on force_inline attributes, else symbols will be missing.
    // But its passes (CallGraph and Inliner) - being module passes - have a
    // quadratically increasing runtime: for each transaction they need to
    // iterate over all previous transactions' functions.
    // Until this is solved (for instance by feeding only the new functions
    // to the CallGraph) we penalize only the use of libc++.
    if (m_CodeGen) {
      llvm::Module* TheModule = m_CodeGen->GetModule();
      // IR passes make sense if we do CodeGen.
      m_IRTransformers.push_back(new BackendPass(TheSema, TheModule,
                                                 CI->getDiagnostics(),
                                                 CI->getTargetOpts(),
                                                 CI->getLangOpts(),
                                                 CI->getCodeGenOpts()));
    }
#endif
  }

  void
  IncrementalParser::Initialize(llvm::SmallVectorImpl<Transaction*> &result) {
    m_TransactionPool.reset(new TransactionPool(getCI()->getSema()));
    if (hasCodeGenerator())
      getCodeGenerator()->Initialize(getCI()->getASTContext());

    CompilationOptions CO;
    CO.DeclarationExtraction = 0;
    CO.ValuePrinting = CompilationOptions::VPDisabled;
    CO.CodeGeneration = hasCodeGenerator();

    // pull in PCHs
    const std::string& PCHFileName
      = m_CI->getInvocation ().getPreprocessorOpts().ImplicitPCHInclude;
    if (!PCHFileName.empty()) {
      Transaction* CurT = beginTransaction(CO);
      m_CI->createPCHExternalASTSource(PCHFileName,
                                       true /*DisablePCHValidation*/,
                                       true /*AllowPCHWithCompilerErrors*/,
                                       0 /*DeserializationListener*/,
                                       true /*OwnsDeserializationListener*/);
      if (Transaction* EndedT = endTransaction(CurT))
        result.push_back(EndedT);
    }

    Transaction* CurT = beginTransaction(CO);
    Sema* TheSema = &m_CI->getSema();
    m_Parser.reset(new Parser(m_CI->getPreprocessor(), *TheSema,
                              false /*skipFuncBodies*/));
    m_CI->getPreprocessor().EnterMainSourceFile();
    // Initialize the parser after we have entered the main source file.
    m_Parser->Initialize();
    // Perform initialization that occurs after the parser has been initialized
    // but before it parses anything. Initializes the consumers too.
    // No - already done by m_Parser->Initialize().
    // TheSema->Initialize();

    ExternalASTSource *External = TheSema->getASTContext().getExternalSource();
    if (External)
      External->StartTranslationUnit(m_Consumer);

    // <new> is needed by the ValuePrinter so it's a good thing to include it.
    // We need to include it to determine the version number of the standard
    // library implementation.
    ParseInternal("#include <new>");
    CheckABICompatibility(m_CI.get());

    // DO NOT commit the transactions here: static initialization in these
    // transactions requires gCling through local_cxa_atexit(), but that has not
    // been defined yet!
    if (Transaction* EndedT = endTransaction(CurT))
      result.push_back(EndedT);
  }

  const Transaction* IncrementalParser::getCurrentTransaction() const {
    return m_Consumer->getTransaction();
  }

  SourceLocation IncrementalParser::getLastMemoryBufferEndLoc() const {
    const SourceManager& SM = getCI()->getSourceManager();
    SourceLocation Result = SM.getLocForStartOfFile(m_VirtualFileID);
    return Result.getLocWithOffset(m_MemoryBuffers.size() + 1);
  }

  IncrementalParser::~IncrementalParser() {
    const Transaction* T = getFirstTransaction();
    const Transaction* nextT = 0;
    while (T) {
      assert((T->getState() == Transaction::kCommitted
              || T->getState() == Transaction::kRolledBackWithErrors
              || T->getState() == Transaction::kNumStates // reset from the pool
              || T->getState() == Transaction::kRolledBack)
             && "Not committed?");
      nextT = T->getNext();
      delete T;
      T = nextT;
    }

    for (size_t i = 0; i < m_ASTTransformers.size(); ++i)
      delete m_ASTTransformers[i];

    for (size_t i = 0; i < m_IRTransformers.size(); ++i)
      delete m_IRTransformers[i];
  }

  Transaction* IncrementalParser::beginTransaction(const CompilationOptions&
                                                   Opts) {
    Transaction* OldCurT = m_Consumer->getTransaction();
    Transaction* NewCurT = m_TransactionPool->takeTransaction();
    NewCurT->setCompilationOpts(Opts);
    // If we are in the middle of transaction and we see another begin
    // transaction - it must be nested transaction.
    if (OldCurT && OldCurT != NewCurT
        && (OldCurT->getState() == Transaction::kCollecting
            || OldCurT->getState() == Transaction::kCompleted)) {
      OldCurT->addNestedTransaction(NewCurT); // takes the ownership
    }

    m_Consumer->setTransaction(NewCurT);
    return NewCurT;
  }

  Transaction* IncrementalParser::endTransaction(Transaction* T) {
    assert(T && "Null transaction!?");
    assert(T->getState() == Transaction::kCollecting);

#ifndef NDEBUG
    if (T->hasNestedTransactions()) {
      for(Transaction::const_nested_iterator I = T->nested_begin(),
            E = T->nested_end(); I != E; ++I)
        assert((*I)->isCompleted() && "Nested transaction not completed!?");
    }
#endif

    T->setState(Transaction::kCompleted);
    // Empty transaction send it back to the pool.
    if (T->empty()) {
      assert((!m_Consumer->getTransaction()
              || (m_Consumer->getTransaction() == T))
             && "Cannot release different T");
      // If a nested transaction the active one should be its parent
      // from now on. FIXME: Merge conditional with commitTransaction
      if (T->isNestedTransaction())
        m_Consumer->setTransaction(T->getParent());
      else
        m_Consumer->setTransaction((Transaction*)0);

      m_TransactionPool->releaseTransaction(T);
      return 0;
    }

    transformTransactionAST(T);
    if (T->empty()) {
      m_TransactionPool->releaseTransaction(T);
      return 0;
    }

    const DiagnosticsEngine& Diags = getCI()->getSema().getDiagnostics();

    //TODO: Make the enum orable.
    if (Diags.getNumWarnings() > 0)
      T->setIssuedDiags(Transaction::kWarnings);

    if (Diags.hasErrorOccurred() || Diags.hasFatalErrorOccurred())
      T->setIssuedDiags(Transaction::kErrors);

    if (!T->isNestedTransaction() && T != getLastTransaction()) {
      if (getLastTransaction())
        m_Transactions.back()->setNext(T);
      m_Transactions.push_back(T);
    }
    return T;
  }

  void IncrementalParser::commitTransaction(Transaction* T) {
    //Transaction* CurT = m_Consumer->getTransaction();
    assert(T->isCompleted() && "Transaction not ended!?");
    assert(T->getState() != Transaction::kCommitted
           && "Committing an already committed transaction.");
    assert(!T->empty() && "Transactions must not be empty;");

    // If committing a nested transaction the active one should be its parent
    // from now on.
    if (T->isNestedTransaction())
      m_Consumer->setTransaction(T->getParent());

    // Check for errors...
    if (T->getIssuedDiags() == Transaction::kErrors) {
      rollbackTransaction(T);
      return;
    }

    if (T->hasNestedTransactions()) {
      for (Transaction::const_nested_iterator I = T->nested_begin(),
            E = T->nested_end(); I != E; ++I)
        if ((*I)->getState() != Transaction::kCommitted)
          commitTransaction(*I);
    }

    // If there was an error coming from the transformers.
    if (T->getIssuedDiags() == Transaction::kErrors) {
      rollbackTransaction(T);
      return;
    }

    // Here we expect a template instantiation. We need to open the transaction
    // that we are currently work with.
    {
      Transaction* prevConsumerT = m_Consumer->getTransaction();
      m_Consumer->setTransaction(T);
      Transaction* nestedT = beginTransaction(CompilationOptions());
      // Pull all template instantiations in that came from the consumers.
      getCI()->getSema().PerformPendingInstantiations();
      if (Transaction* T = endTransaction(nestedT))
        commitTransaction(T);
      m_Consumer->setTransaction(prevConsumerT);
    }
    m_Consumer->HandleTranslationUnit(getCI()->getASTContext());


    // The static initializers might run anything and can thus cause more
    // decls that need to end up in a transaction. But this one is done
    // with CodeGen...
    if (T->getCompilationOpts().CodeGeneration && hasCodeGenerator()) {
      Transaction* prevConsumerT = m_Consumer->getTransaction();
      m_Consumer->setTransaction(T);
      codeGenTransaction(T);
      transformTransactionIR(T);
      T->setState(Transaction::kCommitted);
      if (!T->getParent()) {
        if (m_Interpreter->runStaticInitializersOnce(*T)
            >= Interpreter::kExeFirstError) {
          // Roll back on error in a transformer
          assert(0 && "Error on inits.");
          //rollbackTransaction(nestedT);
          return;
        }
      }
      m_Consumer->setTransaction(prevConsumerT);
    }
    T->setState(Transaction::kCommitted);

    if (InterpreterCallbacks* callbacks = m_Interpreter->getCallbacks())
      callbacks->TransactionCommitted(*T);

  }

  void IncrementalParser::markWholeTransactionAsUsed(Transaction* T) const {
    ASTContext& C = m_CI->getASTContext();
    for (Transaction::const_iterator I = T->decls_begin(), E = T->decls_end();
         I != E; ++I) {
      // Copy DCI; it might get relocated below.
      Transaction::DelayCallInfo DCI = *I;
      // FIXME: implement for multiple decls in a DGR.
      assert(DCI.m_DGR.isSingleDecl());
      Decl* D = DCI.m_DGR.getSingleDecl();
      if (!D->hasAttr<clang::UsedAttr>())
        D->addAttr(::new (D->getASTContext())
                   clang::UsedAttr(D->getSourceRange(), D->getASTContext(),
                                   0/*AttributeSpellingListIndex*/));
    }<commit_msg>Static init triggers relocation which can fail; roll back.<commit_after>//------------------------------------------------------------------------------
// CLING - the C++ LLVM-based InterpreterG :)
// author:  <NAME> <<EMAIL>>
//
// This file is dual-licensed: you can choose to license it under the University
// of Illinois Open Source License or the GNU Lesser General Public License. See
// LICENSE.TXT for details.
//------------------------------------------------------------------------------

#include "IncrementalParser.h"

#include "AutoSynthesizer.h"
#include "BackendPass.h"
#include "CheckEmptyTransactionTransformer.h"
#include "DeclCollector.h"
#include "DeclExtractor.h"
#include "DynamicLookup.h"
#include "IncrementalExecutor.h"
#include "NullDerefProtectionTransformer.h"
#include "ValueExtractionSynthesizer.h"
#include "TransactionPool.h"
#include "TransactionUnloader.h"
#include "ValuePrinterSynthesizer.h"
#include "cling/Interpreter/CIFactory.h"
#include "cling/Interpreter/Interpreter.h"
#include "cling/Interpreter/InterpreterCallbacks.h"
#include "cling/Interpreter/Transaction.h"

#include "clang/AST/Attr.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/Decl.h"
#include "clang/AST/DeclGroup.h"
#include "clang/AST/RecursiveASTVisitor.h"
#include "clang/Basic/FileManager.h"
#include "clang/CodeGen/ModuleBuilder.h"
#include "clang/Parse/Parser.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Sema/Sema.h"
#include "clang/Sema/SemaDiagnostic.h"
#include "clang/Serialization/ASTWriter.h"

#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/CrashRecoveryContext.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/raw_os_ostream.h"

#include <iostream>
#include <stdio.h>
#include <sstream>

// Include the necessary headers to interface with the Windows registry and
// environment.
#ifdef _MSC_VER
  #define WIN32_LEAN_AND_MEAN
  #define NOGDI
  #define NOMINMAX
  #include <Windows.h>
  #include <sstream>
  #define popen _popen
  #define pclose _pclose
  #pragma comment(lib, "Advapi32.lib")
#endif

using namespace clang;

namespace {
  ///\brief Check the compile-time C++ ABI version vs the run-time ABI version,
  /// a mismatch could cause havoc. Reports if ABI versions differ.
  static void CheckABICompatibility(clang::CompilerInstance* CI) {
#ifdef __GLIBCXX__
# define CLING_CXXABIV __GLIBCXX__
# define CLING_CXXABIS "__GLIBCXX__"
#elif _LIBCPP_VERSION
# define CLING_CXXABIV _LIBCPP_VERSION
# define CLING_CXXABIS "_LIBCPP_VERSION"
#elif defined (_MSC_VER)
    // For MSVC we do not use CLING_CXXABI*
#else
# define CLING_CXXABIV -1 // intentionally invalid macro name
# define CLING_CXXABIS "-1" // intentionally invalid macro name
    llvm::errs()
      << "Warning in cling::CIFactory::createCI():\n  "
      "C++ ABI check not implemented for this standard library\n";
    return;
#endif
#ifdef _MSC_VER
    HKEY regVS;
    int VSVersion = (_MSC_VER / 100) - 6;
    std::stringstream subKey;
    subKey << "VisualStudio.DTE." << VSVersion << ".0";
    if (RegOpenKeyEx(HKEY_CLASSES_ROOT, subKey.str().c_str(), 0, KEY_READ, &regVS) == ERROR_SUCCESS) {
      RegCloseKey(regVS);
    }
    else {
      llvm::errs()
        << "Warning in cling::CIFactory::createCI():\n  "
        "Possible C++ standard library mismatch, compiled with Visual Studio v"
        << VSVersion << ".0,\n"
        "but this version of Visual Studio was not found in your system's registry.\n";
    }
#else

  struct EarlyReturnWarn {
    bool shouldWarn = true;
    ~EarlyReturnWarn() {
      if (shouldWarn) {
        llvm::errs()
          << "Warning in cling::IncrementalParser::CheckABICompatibility():\n  "
             "Possible C++ standard library mismatch, compiled with "
             CLING_CXXABIS " v" << CLING_CXXABIV
          << " but extraction of runtime standard library version failed.\n";
      }
    }
  } warnAtReturn;
  clang::Preprocessor& PP = CI->getPreprocessor();
  clang::IdentifierInfo* II = PP.getIdentifierInfo(CLING_CXXABIS);
  if (!II)
    return;
  const clang::DefMacroDirective* MD
    = llvm::dyn_cast<clang::DefMacroDirective>(PP.getMacroDirective(II));
  if (!MD)
    return;
  const clang::MacroInfo* MI = MD->getMacroInfo();
  if (!MI || MI->getNumTokens() != 1)
    return;
  const clang::Token& Tok = *MI->tokens_begin();
  if (!Tok.isLiteral())
    return;
  if (!Tok.getLength() || !Tok.getLiteralData())
    return;

  std::string cxxabivStr;
  {
     llvm::raw_string_ostream cxxabivStrStrm(cxxabivStr);
     cxxabivStrStrm << CLING_CXXABIV;
  }
  llvm::StringRef tokStr(Tok.getLiteralData(), Tok.getLength());

  warnAtReturn.shouldWarn = false;
  if (!tokStr.equals(cxxabivStr)) {
    llvm::errs()
      << "Warning in cling::IncrementalParser::CheckABICompatibility():\n  "
        "C++ ABI mismatch, compiled with "
        CLING_CXXABIS " v" << CLING_CXXABIV
      << " running with v" << tokStr << "\n";
  }
#endif
#undef CLING_CXXABIV
#undef CLING_CXXABIS
  }
} // unnamed namespace

namespace cling {
  IncrementalParser::IncrementalParser(Interpreter* interp,
                                       int argc, const char* const *argv,
                                       const char* llvmdir):
    m_Interpreter(interp), m_Consumer(0), m_ModuleNo(0) {

    CompilerInstance* CI = CIFactory::createCI("", argc, argv, llvmdir);
    assert(CI && "CompilerInstance is (null)!");

    m_Consumer = dyn_cast<DeclCollector>(&CI->getSema().getASTConsumer());
    assert(m_Consumer && "Expected ChainedConsumer!");

    m_CI.reset(CI);

    if (CI->getFrontendOpts().ProgramAction != clang::frontend::ParseSyntaxOnly){
      m_CodeGen.reset(CreateLLVMCodeGen(CI->getDiagnostics(), "cling-module-0",
                                        CI->getCodeGenOpts(),
                                        CI->getTargetOpts(),
                                        *m_Interpreter->getLLVMContext()
                                        ));
    }

    initializeVirtualFile();

    // Add transformers to the IncrementalParser, which owns them
    Sema* TheSema = &CI->getSema();
    // Register the AST Transformers
    m_ASTTransformers.push_back(new AutoSynthesizer(TheSema));
    m_ASTTransformers.push_back(new EvaluateTSynthesizer(TheSema));
    m_ASTTransformers.push_back(new ValuePrinterSynthesizer(TheSema, 0));
    m_ASTTransformers.push_back(new DeclExtractor(TheSema));
    m_ASTTransformers.push_back(new ValueExtractionSynthesizer(TheSema));
    m_ASTTransformers.push_back(new NullDerefProtectionTransformer(TheSema));
    m_ASTTransformers.push_back(new CheckEmptyTransactionTransformer(TheSema));


#ifdef _LIBCPP_VERSION
    // libc++ relies on force_inline attributes, else symbols will be missing.
    // But its passes (CallGraph and Inliner) - being module passes - have a
    // quadratically increasing runtime: for each transaction they need to
    // iterate over all previous transactions' functions.
    // Until this is solved (for instance by feeding only the new functions
    // to the CallGraph) we penalize only the use of libc++.
    if (m_CodeGen) {
      llvm::Module* TheModule = m_CodeGen->GetModule();
      // IR passes make sense if we do CodeGen.
      m_IRTransformers.push_back(new BackendPass(TheSema, TheModule,
                                                 CI->getDiagnostics(),
                                                 CI->getTargetOpts(),
                                                 CI->getLangOpts(),
                                                 CI->getCodeGenOpts()));
    }
#endif
  }

  void
  IncrementalParser::Initialize(llvm::SmallVectorImpl<Transaction*> &result) {
    m_TransactionPool.reset(new TransactionPool(getCI()->getSema()));
    if (hasCodeGenerator())
      getCodeGenerator()->Initialize(getCI()->getASTContext());

    CompilationOptions CO;
    CO.DeclarationExtraction = 0;
    CO.ValuePrinting = CompilationOptions::VPDisabled;
    CO.CodeGeneration = hasCodeGenerator();

    // pull in PCHs
    const std::string& PCHFileName
      = m_CI->getInvocation ().getPreprocessorOpts().ImplicitPCHInclude;
    if (!PCHFileName.empty()) {
      Transaction* CurT = beginTransaction(CO);
      m_CI->createPCHExternalASTSource(PCHFileName,
                                       true /*DisablePCHValidation*/,
                                       true /*AllowPCHWithCompilerErrors*/,
                                       0 /*DeserializationListener*/,
                                       true /*OwnsDeserializationListener*/);
      if (Transaction* EndedT = endTransaction(CurT))
        result.push_back(EndedT);
    }

    Transaction* CurT = beginTransaction(CO);
    Sema* TheSema = &m_CI->getSema();
    m_Parser.reset(new Parser(m_CI->getPreprocessor(), *TheSema,
                              false /*skipFuncBodies*/));
    m_CI->getPreprocessor().EnterMainSourceFile();
    // Initialize the parser after we have entered the main source file.
    m_Parser->Initialize();
    // Perform initialization that occurs after the parser has been initialized
    // but before it parses anything. Initializes the consumers too.
    // No - already done by m_Parser->Initialize().
    // TheSema->Initialize();

    ExternalASTSource *External = TheSema->getASTContext().getExternalSource();
    if (External)
      External->StartTranslationUnit(m_Consumer);

    // <new> is needed by the ValuePrinter so it's a good thing to include it.
    // We need to include it to determine the version number of the standard
    // library implementation.
    ParseInternal("#include <new>");
    CheckABICompatibility(m_CI.get());

    // DO NOT commit the transactions here: static initialization in these
    // transactions requires gCling through local_cxa_atexit(), but that has not
    // been defined yet!
    if (Transaction* EndedT = endTransaction(CurT))
      result.push_back(EndedT);
  }

  const Transaction* IncrementalParser::getCurrentTransaction() const {
    return m_Consumer->getTransaction();
  }

  SourceLocation IncrementalParser::getLastMemoryBufferEndLoc() const {
    const SourceManager& SM = getCI()->getSourceManager();
    SourceLocation Result = SM.getLocForStartOfFile(m_VirtualFileID);
    return Result.getLocWithOffset(m_MemoryBuffers.size() + 1);
  }

  IncrementalParser::~IncrementalParser() {
    const Transaction* T = getFirstTransaction();
    const Transaction* nextT = 0;
    while (T) {
      assert((T->getState() == Transaction::kCommitted
              || T->getState() == Transaction::kRolledBackWithErrors
              || T->getState() == Transaction::kNumStates // reset from the pool
              || T->getState() == Transaction::kRolledBack)
             && "Not committed?");
      nextT = T->getNext();
      delete T;
      T = nextT;
    }

    for (size_t i = 0; i < m_ASTTransformers.size(); ++i)
      delete m_ASTTransformers[i];

    for (size_t i = 0; i < m_IRTransformers.size(); ++i)
      delete m_IRTransformers[i];
  }

  Transaction* IncrementalParser::beginTransaction(const CompilationOptions&
                                                   Opts) {
    Transaction* OldCurT = m_Consumer->getTransaction();
    Transaction* NewCurT = m_TransactionPool->takeTransaction();
    NewCurT->setCompilationOpts(Opts);
    // If we are in the middle of transaction and we see another begin
    // transaction - it must be nested transaction.
    if (OldCurT && OldCurT != NewCurT
        && (OldCurT->getState() == Transaction::kCollecting
            || OldCurT->getState() == Transaction::kCompleted)) {
      OldCurT->addNestedTransaction(NewCurT); // takes the ownership
    }

    m_Consumer->setTransaction(NewCurT);
    return NewCurT;
  }

  Transaction* IncrementalParser::endTransaction(Transaction* T) {
    assert(T && "Null transaction!?");
    assert(T->getState() == Transaction::kCollecting);

#ifndef NDEBUG
    if (T->hasNestedTransactions()) {
      for(Transaction::const_nested_iterator I = T->nested_begin(),
            E = T->nested_end(); I != E; ++I)
        assert((*I)->isCompleted() && "Nested transaction not completed!?");
    }
#endif

    T->setState(Transaction::kCompleted);
    // Empty transaction send it back to the pool.
    if (T->empty()) {
      assert((!m_Consumer->getTransaction()
              || (m_Consumer->getTransaction() == T))
             && "Cannot release different T");
      // If a nested transaction the active one should be its parent
      // from now on. FIXME: Merge conditional with commitTransaction
      if (T->isNestedTransaction())
        m_Consumer->setTransaction(T->getParent());
      else
        m_Consumer->setTransaction((Transaction*)0);

      m_TransactionPool->releaseTransaction(T);
      return 0;
    }

    transformTransactionAST(T);
    if (T->empty()) {
      m_TransactionPool->releaseTransaction(T);
      return 0;
    }

    const DiagnosticsEngine& Diags = getCI()->getSema().getDiagnostics();

    //TODO: Make the enum orable.
    if (Diags.getNumWarnings() > 0)
      T->setIssuedDiags(Transaction::kWarnings);

    if (Diags.hasErrorOccurred() || Diags.hasFatalErrorOccurred())
      T->setIssuedDiags(Transaction::kErrors);

    if (!T->isNestedTransaction() && T != getLastTransaction()) {
      if (getLastTransaction())
        m_Transactions.back()->setNext(T);
      m_Transactions.push_back(T);
    }
    return T;
  }

  void IncrementalParser::commitTransaction(Transaction* T) {
    //Transaction* CurT = m_Consumer->getTransaction();
    assert(T->isCompleted() && "Transaction not ended!?");
    assert(T->getState() != Transaction::kCommitted
           && "Committing an already committed transaction.");
    assert(!T->empty() && "Transactions must not be empty;");

    // If committing a nested transaction the active one should be its parent
    // from now on.
    if (T->isNestedTransaction())
      m_Consumer->setTransaction(T->getParent());

    // Check for errors...
    if (T->getIssuedDiags() == Transaction::kErrors) {
      rollbackTransaction(T);
      return;
    }

    if (T->hasNestedTransactions()) {
      for (Transaction::const_nested_iterator I = T->nested_begin(),
            E = T->nested_end(); I != E; ++I)
        if ((*I)->getState() != Transaction::kCommitted)
          commitTransaction(*I);
    }

    // If there was an error coming from the transformers.
    if (T->getIssuedDiags() == Transaction::kErrors) {
      rollbackTransaction(T);
      return;
    }

    // Here we expect a template instantiation. We need to open the transaction
    // that we are currently work with.
    {
      Transaction* prevConsumerT = m_Consumer->getTransaction();
      m_Consumer->setTransaction(T);
      Transaction* nestedT = beginTransaction(CompilationOptions());
      // Pull all template instantiations in that came from the consumers.
      getCI()->getSema().PerformPendingInstantiations();
      if (Transaction* T = endTransaction(nestedT))
        commitTransaction(T);
      m_Consumer->setTransaction(prevConsumerT);
    }
    m_Consumer->HandleTranslationUnit(getCI()->getASTContext());


    // The static initializers might run anything and can thus cause more
    // decls that need to end up in a transaction. But this one is done
    // with CodeGen...
    if (T->getCompilationOpts().CodeGeneration && hasCodeGenerator()) {
      Transaction* prevConsumerT = m_Consumer->getTransaction();
      m_Consumer->setTransaction(T);
      codeGenTransaction(T);
      transformTransactionIR(T);
      T->setState(Transaction::kCommitted);
      if (!T->getParent()) {
        if (m_Interpreter->runStaticInitializersOnce(*T)
            >= Interpreter::kExeFirstError) {
          // Roll back on error in initializers
          //assert(0 && "Error on inits.");
          rollbackTransaction(T);
          T->setState(Transaction::kRolledBackWithErrors);
          return;
        }
      }
      m_Consumer->setTransaction(prevConsumerT);
    }
    T->setState(Transaction::kCommitted);

    if (InterpreterCallbacks* callbacks = m_Interpreter->getCallbacks())
      callbacks->TransactionCommitted(*T);

  }

  void IncrementalParser::markWholeTransactionAsUsed(Transaction* T) const {
    ASTContext& C = m_CI->getASTContext();
    for (Transaction::const_iterator I = T->decls_begin(), E = T->decls_end();
         I != E; ++I) {
      // Copy DCI; it might get relocated below.
      Transaction::DelayCallInfo DCI = *I;
      // FIXME: implement for multiple decls in a DGR.
      assert(DCI.m_DGR.isSingleDecl());
      Decl* D = DCI.m_DGR.getSingleDecl();
      if (!D->hasAttr<clang::UsedAttr>())
        D->addAttr(::new (D->getASTContext())
                   clang::UsedAttr(D->getSourceRange(), D->getASTContext(),
                                   0/*AttributeSpellingListIndex*/));
    }
{'fraction_non_alphanumeric': 0.1309145925420511, 'fraction_numerical': 0.0017799400753507966, 'mean_word_length': 2.935785172212493, 'pattern_counts': {'":': 0, '<': 99, '<?xml version=': 0, '>': 238, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 52, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7541789', 'n_tokens_mistral': 10045, 'n_tokens_neox': 9807, 'n_words': 2302}