<commit_before>//===- CIndexHigh.cpp - Higher level API functions ------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#include "IndexingContext.h"
#include "CXCursor.h"
#include "CXSourceLocation.h"
#include "CXTranslationUnit.h"
#include "CXString.h"
#include "CIndexer.h"

#include "clang/Frontend/ASTUnit.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/Utils.h"
#include "clang/Sema/SemaConsumer.h"
#include "clang/AST/ASTConsumer.h"
#include "clang/AST/DeclVisitor.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/PPCallbacks.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/CrashRecoveryContext.h"

using namespace clang;
using namespace cxstring;
using namespace cxtu;
using namespace cxindex;

namespace {

//===----------------------------------------------------------------------===//
// IndexPPCallbacks
//===----------------------------------------------------------------------===//

class IndexPPCallbacks : public PPCallbacks {
  Preprocessor &PP;
  IndexingContext &IndexCtx;
  bool IsMainFileEntered;

public:
  IndexPPCallbacks(Preprocessor &PP, IndexingContext &indexCtx)
    : PP(PP), IndexCtx(indexCtx), IsMainFileEntered(false) { }

  virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
                          SrcMgr::CharacteristicKind FileType, FileID PrevFID) {
    if (IsMainFileEntered)
      return;

    SourceManager &SM = PP.getSourceManager();
    SourceLocation MainFileLoc = SM.getLocForStartOfFile(SM.getMainFileID());

    if (Loc == MainFileLoc && Reason == PPCallbacks::EnterFile) {
      IsMainFileEntered = true;
      IndexCtx.enteredMainFile(SM.getFileEntryForID(SM.getMainFileID()));
    }
  }

  virtual void InclusionDirective(SourceLocation HashLoc,
                                  const Token &IncludeTok,
                                  StringRef FileName,
                                  bool IsAngled,
                                  const FileEntry *File,
                                  SourceLocation EndLoc,
                                  StringRef SearchPath,
                                  StringRef RelativePath) {
    bool isImport = (IncludeTok.is(tok::identifier) &&
            IncludeTok.getIdentifierInfo()->getPPKeywordID() == tok::pp_import);
    IndexCtx.ppIncludedFile(HashLoc, FileName, File, isImport, IsAngled);
  }

  /// MacroDefined - This hook is called whenever a macro definition is seen.
  virtual void MacroDefined(const Token &Id, const MacroInfo *MI) {
  }

  /// MacroUndefined - This hook is called whenever a macro #undef is seen.
  /// MI is released immediately following this callback.
  virtual void MacroUndefined(const Token &MacroNameTok, const MacroInfo *MI) {
  }

  /// MacroExpands - This is called by when a macro invocation is found.
  virtual void MacroExpands(const Token &MacroNameTok, const MacroInfo* MI,
                            SourceRange Range) {
  }
  
  /// SourceRangeSkipped - This hook is called when a source range is skipped.
  /// \param Range The SourceRange that was skipped. The range begins at the
  /// #if/#else directive and ends after the #endif/#else directive.
  virtual void SourceRangeSkipped(SourceRange Range) {
  }
};

//===----------------------------------------------------------------------===//
// IndexingConsumer
//===----------------------------------------------------------------------===//

class IndexingConsumer : public ASTConsumer {
  IndexingContext &IndexCtx;

public:
  explicit IndexingConsumer(IndexingContext &indexCtx)
    : IndexCtx(indexCtx) { }

  // ASTConsumer Implementation

  virtual void Initialize(ASTContext &Context) {
    IndexCtx.setASTContext(Context);
    IndexCtx.startedTranslationUnit();
  }

  virtual void HandleTranslationUnit(ASTContext &Ctx) {
  }

  virtual bool HandleTopLevelDecl(DeclGroupRef DG) {
    IndexCtx.indexDeclGroupRef(DG);
    return !IndexCtx.shouldAbort();
  }

  /// \brief Handle the specified top-level declaration that occurred inside
  /// and ObjC container.
  virtual void HandleTopLevelDeclInObjCContainer(DeclGroupRef D) {
    // They will be handled after the interface is seen first.
    IndexCtx.addTUDeclInObjCContainer(D);
  }

  /// \brief This is called by the AST reader when deserializing things.
  /// The default implementation forwards to HandleTopLevelDecl but we don't
  /// care about them when indexing, so have an empty definition.
  virtual void HandleInterestingDecl(DeclGroupRef D) {}
};

//===----------------------------------------------------------------------===//
// CaptureDiagnosticConsumer
//===----------------------------------------------------------------------===//

class CaptureDiagnosticConsumer : public DiagnosticConsumer {
  SmallVector<StoredDiagnostic, 4> Errors;
public:

  virtual void HandleDiagnostic(DiagnosticsEngine::Level level,
                                const Diagnostic &Info) {
    if (level >= DiagnosticsEngine::Error)
      Errors.push_back(StoredDiagnostic(level, Info));
  }

  DiagnosticConsumer *clone(DiagnosticsEngine &Diags) const {
    return new IgnoringDiagConsumer();
  }
};

//===----------------------------------------------------------------------===//
// IndexingFrontendAction
//===----------------------------------------------------------------------===//

class IndexingFrontendAction : public ASTFrontendAction {
  IndexingContext IndexCtx;

public:
  IndexingFrontendAction(CXClientData clientData,
                         IndexerCallbacks &indexCallbacks,
                         unsigned indexOptions,
                         CXTranslationUnit cxTU)
    : IndexCtx(clientData, indexCallbacks, indexOptions, cxTU) { }

  virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
                                         StringRef InFile) {
    IndexCtx.setASTContext(CI.getASTContext());
    Preprocessor &PP = CI.getPreprocessor();
    PP.addPPCallbacks(new IndexPPCallbacks(PP, IndexCtx));
    return new IndexingConsumer(IndexCtx);
  }

  virtual TranslationUnitKind getTranslationUnitKind() { return TU_Prefix; }
  virtual bool hasCodeCompletionSupport() const { return false; }
};

//===----------------------------------------------------------------------===//
// clang_indexSourceFileUnit Implementation
//===----------------------------------------------------------------------===//

struct IndexSourceFileInfo {
  CXIndexAction idxAction;
  CXClientData client_data;
  IndexerCallbacks *index_callbacks;
  unsigned index_callbacks_size;
  unsigned index_options;
  const char *source_filename;
  const char *const *command_line_args;
  int num_command_line_args;
  struct CXUnsavedFile *unsaved_files;
  unsigned num_unsaved_files;
  CXTranslationUnit *out_TU;
  unsigned TU_options;
  int result;
};

struct MemBufferOwner {
  SmallVector<const llvm::MemoryBuffer *, 8> Buffers;
  
  ~MemBufferOwner() {
    for (SmallVectorImpl<const llvm::MemoryBuffer *>::iterator
           I = Buffers.begin(), E = Buffers.end(); I != E; ++I)
      delete *I;
  }
};

} // anonymous namespace

static void clang_indexSourceFile_Impl(void *UserData) {
  IndexSourceFileInfo *ITUI =
    static_cast<IndexSourceFileInfo*>(UserData);
  CXIndex CIdx = (CXIndex)ITUI->idxAction;
  CXClientData client_data = ITUI->client_data;
  IndexerCallbacks *client_index_callbacks = ITUI->index_callbacks;
  unsigned index_callbacks_size = ITUI->index_callbacks_size;
  unsigned index_options = ITUI->index_options;
  const char *source_filename = ITUI->source_filename;
  const char * const *command_line_args = ITUI->command_line_args;
  int num_command_line_args = ITUI->num_command_line_args;
  struct CXUnsavedFile *unsaved_files = ITUI->unsaved_files;
  unsigned num_unsaved_files = ITUI->num_unsaved_files;
  CXTranslationUnit *out_TU  = ITUI->out_TU;
  unsigned TU_options = ITUI->TU_options;
  ITUI->result = 1; // init as error.
  
  if (out_TU)
    *out_TU = 0;
  bool requestedToGetTU = (out_TU != 0); 

  if (!CIdx)
    return;
  if (!client_index_callbacks || index_callbacks_size == 0)
    return;

  IndexerCallbacks CB;
  memset(&CB, 0, sizeof(CB));
  unsigned ClientCBSize = index_callbacks_size < sizeof(CB)
                                  ? index_callbacks_size : sizeof(CB);
  memcpy(&CB, client_index_callbacks, ClientCBSize);

  CIndexer *CXXIdx = static_cast<CIndexer *>(CIdx);

  CaptureDiagnosticConsumer *CaptureDiag = new CaptureDiagnosticConsumer();

  // Configure the diagnostics.
  DiagnosticOptions DiagOpts;
  llvm::IntrusiveRefCntPtr<DiagnosticsEngine>
    Diags(CompilerInstance::createDiagnostics(DiagOpts, num_command_line_args, 
                                                command_line_args,
                                                CaptureDiag,
                                                /*ShouldOwnClient=*/true));

  // Recover resources if we crash before exiting this function.
  llvm::CrashRecoveryContextCleanupRegistrar<DiagnosticsEngine,
    llvm::CrashRecoveryContextReleaseRefCleanup<DiagnosticsEngine> >
    DiagCleanup(Diags.getPtr());
  
  llvm::OwningPtr<std::vector<const char *> > 
    Args(new std::vector<const char*>());

  // Recover resources if we crash before exiting this method.
  llvm::CrashRecoveryContextCleanupRegistrar<std::vector<const char*> >
    ArgsCleanup(Args.get());
  
  Args->insert(Args->end(), command_line_args,
               command_line_args + num_command_line_args);

  // The 'source_filename' argument is optional.  If the caller does not<commit_msg>[libclang] Fix leak in clang_indexSourceFile_Impl<commit_after>//===- CIndexHigh.cpp - Higher level API functions ------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#include "IndexingContext.h"
#include "CXCursor.h"
#include "CXSourceLocation.h"
#include "CXTranslationUnit.h"
#include "CXString.h"
#include "CIndexer.h"

#include "clang/Frontend/ASTUnit.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/Utils.h"
#include "clang/Sema/SemaConsumer.h"
#include "clang/AST/ASTConsumer.h"
#include "clang/AST/DeclVisitor.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/PPCallbacks.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/CrashRecoveryContext.h"

using namespace clang;
using namespace cxstring;
using namespace cxtu;
using namespace cxindex;

namespace {

//===----------------------------------------------------------------------===//
// IndexPPCallbacks
//===----------------------------------------------------------------------===//

class IndexPPCallbacks : public PPCallbacks {
  Preprocessor &PP;
  IndexingContext &IndexCtx;
  bool IsMainFileEntered;

public:
  IndexPPCallbacks(Preprocessor &PP, IndexingContext &indexCtx)
    : PP(PP), IndexCtx(indexCtx), IsMainFileEntered(false) { }

  virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
                          SrcMgr::CharacteristicKind FileType, FileID PrevFID) {
    if (IsMainFileEntered)
      return;

    SourceManager &SM = PP.getSourceManager();
    SourceLocation MainFileLoc = SM.getLocForStartOfFile(SM.getMainFileID());

    if (Loc == MainFileLoc && Reason == PPCallbacks::EnterFile) {
      IsMainFileEntered = true;
      IndexCtx.enteredMainFile(SM.getFileEntryForID(SM.getMainFileID()));
    }
  }

  virtual void InclusionDirective(SourceLocation HashLoc,
                                  const Token &IncludeTok,
                                  StringRef FileName,
                                  bool IsAngled,
                                  const FileEntry *File,
                                  SourceLocation EndLoc,
                                  StringRef SearchPath,
                                  StringRef RelativePath) {
    bool isImport = (IncludeTok.is(tok::identifier) &&
            IncludeTok.getIdentifierInfo()->getPPKeywordID() == tok::pp_import);
    IndexCtx.ppIncludedFile(HashLoc, FileName, File, isImport, IsAngled);
  }

  /// MacroDefined - This hook is called whenever a macro definition is seen.
  virtual void MacroDefined(const Token &Id, const MacroInfo *MI) {
  }

  /// MacroUndefined - This hook is called whenever a macro #undef is seen.
  /// MI is released immediately following this callback.
  virtual void MacroUndefined(const Token &MacroNameTok, const MacroInfo *MI) {
  }

  /// MacroExpands - This is called by when a macro invocation is found.
  virtual void MacroExpands(const Token &MacroNameTok, const MacroInfo* MI,
                            SourceRange Range) {
  }
  
  /// SourceRangeSkipped - This hook is called when a source range is skipped.
  /// \param Range The SourceRange that was skipped. The range begins at the
  /// #if/#else directive and ends after the #endif/#else directive.
  virtual void SourceRangeSkipped(SourceRange Range) {
  }
};

//===----------------------------------------------------------------------===//
// IndexingConsumer
//===----------------------------------------------------------------------===//

class IndexingConsumer : public ASTConsumer {
  IndexingContext &IndexCtx;

public:
  explicit IndexingConsumer(IndexingContext &indexCtx)
    : IndexCtx(indexCtx) { }

  // ASTConsumer Implementation

  virtual void Initialize(ASTContext &Context) {
    IndexCtx.setASTContext(Context);
    IndexCtx.startedTranslationUnit();
  }

  virtual void HandleTranslationUnit(ASTContext &Ctx) {
  }

  virtual bool HandleTopLevelDecl(DeclGroupRef DG) {
    IndexCtx.indexDeclGroupRef(DG);
    return !IndexCtx.shouldAbort();
  }

  /// \brief Handle the specified top-level declaration that occurred inside
  /// and ObjC container.
  virtual void HandleTopLevelDeclInObjCContainer(DeclGroupRef D) {
    // They will be handled after the interface is seen first.
    IndexCtx.addTUDeclInObjCContainer(D);
  }

  /// \brief This is called by the AST reader when deserializing things.
  /// The default implementation forwards to HandleTopLevelDecl but we don't
  /// care about them when indexing, so have an empty definition.
  virtual void HandleInterestingDecl(DeclGroupRef D) {}
};

//===----------------------------------------------------------------------===//
// CaptureDiagnosticConsumer
//===----------------------------------------------------------------------===//

class CaptureDiagnosticConsumer : public DiagnosticConsumer {
  SmallVector<StoredDiagnostic, 4> Errors;
public:

  virtual void HandleDiagnostic(DiagnosticsEngine::Level level,
                                const Diagnostic &Info) {
    if (level >= DiagnosticsEngine::Error)
      Errors.push_back(StoredDiagnostic(level, Info));
  }

  DiagnosticConsumer *clone(DiagnosticsEngine &Diags) const {
    return new IgnoringDiagConsumer();
  }
};

//===----------------------------------------------------------------------===//
// IndexingFrontendAction
//===----------------------------------------------------------------------===//

class IndexingFrontendAction : public ASTFrontendAction {
  IndexingContext IndexCtx;

public:
  IndexingFrontendAction(CXClientData clientData,
                         IndexerCallbacks &indexCallbacks,
                         unsigned indexOptions,
                         CXTranslationUnit cxTU)
    : IndexCtx(clientData, indexCallbacks, indexOptions, cxTU) { }

  virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
                                         StringRef InFile) {
    IndexCtx.setASTContext(CI.getASTContext());
    Preprocessor &PP = CI.getPreprocessor();
    PP.addPPCallbacks(new IndexPPCallbacks(PP, IndexCtx));
    return new IndexingConsumer(IndexCtx);
  }

  virtual TranslationUnitKind getTranslationUnitKind() { return TU_Prefix; }
  virtual bool hasCodeCompletionSupport() const { return false; }
};

//===----------------------------------------------------------------------===//
// clang_indexSourceFileUnit Implementation
//===----------------------------------------------------------------------===//

struct IndexSourceFileInfo {
  CXIndexAction idxAction;
  CXClientData client_data;
  IndexerCallbacks *index_callbacks;
  unsigned index_callbacks_size;
  unsigned index_options;
  const char *source_filename;
  const char *const *command_line_args;
  int num_command_line_args;
  struct CXUnsavedFile *unsaved_files;
  unsigned num_unsaved_files;
  CXTranslationUnit *out_TU;
  unsigned TU_options;
  int result;
};

struct MemBufferOwner {
  SmallVector<const llvm::MemoryBuffer *, 8> Buffers;
  
  ~MemBufferOwner() {
    for (SmallVectorImpl<const llvm::MemoryBuffer *>::iterator
           I = Buffers.begin(), E = Buffers.end(); I != E; ++I)
      delete *I;
  }
};

} // anonymous namespace

static void clang_indexSourceFile_Impl(void *UserData) {
  IndexSourceFileInfo *ITUI =
    static_cast<IndexSourceFileInfo*>(UserData);
  CXIndex CIdx = (CXIndex)ITUI->idxAction;
  CXClientData client_data = ITUI->client_data;
  IndexerCallbacks *client_index_callbacks = ITUI->index_callbacks;
  unsigned index_callbacks_size = ITUI->index_callbacks_size;
  unsigned index_options = ITUI->index_options;
  const char *source_filename = ITUI->source_filename;
  const char * const *command_line_args = ITUI->command_line_args;
  int num_command_line_args = ITUI->num_command_line_args;
  struct CXUnsavedFile *unsaved_files = ITUI->unsaved_files;
  unsigned num_unsaved_files = ITUI->num_unsaved_files;
  CXTranslationUnit *out_TU  = ITUI->out_TU;
  unsigned TU_options = ITUI->TU_options;
  ITUI->result = 1; // init as error.
  
  if (out_TU)
    *out_TU = 0;
  bool requestedToGetTU = (out_TU != 0); 

  if (!CIdx)
    return;
  if (!client_index_callbacks || index_callbacks_size == 0)
    return;

  IndexerCallbacks CB;
  memset(&CB, 0, sizeof(CB));
  unsigned ClientCBSize = index_callbacks_size < sizeof(CB)
                                  ? index_callbacks_size : sizeof(CB);
  memcpy(&CB, client_index_callbacks, ClientCBSize);

  CIndexer *CXXIdx = static_cast<CIndexer *>(CIdx);

  CaptureDiagnosticConsumer *CaptureDiag = new CaptureDiagnosticConsumer();

  // Configure the diagnostics.
  DiagnosticOptions DiagOpts;
  llvm::IntrusiveRefCntPtr<DiagnosticsEngine>
    Diags(CompilerInstance::createDiagnostics(DiagOpts, num_command_line_args, 
                                                command_line_args,
                                                CaptureDiag,
                                                /*ShouldOwnClient=*/true,
                                                /*ShouldCloneClient=*/false));

  // Recover resources if we crash before exiting this function.
  llvm::CrashRecoveryContextCleanupRegistrar<DiagnosticsEngine,
    llvm::CrashRecoveryContextReleaseRefCleanup<DiagnosticsEngine> >
    DiagCleanup(Diags.getPtr());
  
  llvm::OwningPtr<std::vector<const char *> > 
    Args(new std::vector<const char*>());

  // Recover resources if we crash before exiting this method.
  llvm::CrashRecoveryContextCleanupRegistrar<std::vector<const char*> >
    ArgsCleanup(Args.get());
  
  Args->insert(Args->end(), command_line_args,
               command_line_args + num_command_line_args);

  // The 'source_filename' argument is optional.  If the caller does not
{'fraction_non_alphanumeric': 0.17345025089969082, 'fraction_numerical': 0.0007096152871407573, 'mean_word_length': 2.3668941979522184, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1525528', 'n_tokens_mistral': 5434, 'n_tokens_neox': 5288, 'n_words': 1238}