<commit_before>//===--- InclusionRewriter.cpp - Rewrite includes into their expansions ---===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This code rewrites include invocations into their expansions.  This gives you
// a file with all included files merged into it.
//
//===----------------------------------------------------------------------===//

#include "clang/Rewrite/Frontend/Rewriters.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Frontend/PreprocessorOutputOptions.h"
#include "clang/Lex/HeaderSearch.h"
#include "clang/Lex/Pragma.h"
#include "clang/Lex/Preprocessor.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/Support/raw_ostream.h"

using namespace clang;
using namespace llvm;

namespace {

class InclusionRewriter : public PPCallbacks {
  /// Information about which #includes were actually performed,
  /// created by preprocessor callbacks.
  struct IncludedFile {
    FileID Id;
    SrcMgr::CharacteristicKind FileType;
    IncludedFile(FileID Id, SrcMgr::CharacteristicKind FileType)
        : Id(Id), FileType(FileType) {}
  };
  Preprocessor &PP; ///< Used to find inclusion directives.
  SourceManager &SM; ///< Used to read and manage source files.
  raw_ostream &OS; ///< The destination stream for rewritten contents.
  StringRef MainEOL; ///< The line ending marker to use.
  const llvm::MemoryBuffer *PredefinesBuffer; ///< The preprocessor predefines.
  bool ShowLineMarkers; ///< Show #line markers.
  bool UseLineDirectives; ///< Use of line directives or line markers.
  /// Tracks where inclusions that change the file are found.
  std::map<unsigned, IncludedFile> FileIncludes;
  /// Tracks where inclusions that import modules are found.
  std::map<unsigned, const Module *> ModuleIncludes;
  /// Used transitively for building up the FileIncludes mapping over the
  /// various \c PPCallbacks callbacks.
  SourceLocation LastInclusionLocation;
public:
  InclusionRewriter(Preprocessor &PP, raw_ostream &OS, bool ShowLineMarkers,
                    bool UseLineDirectives);
  bool Process(FileID FileId, SrcMgr::CharacteristicKind FileType);
  void setPredefinesBuffer(const llvm::MemoryBuffer *Buf) {
    PredefinesBuffer = Buf;
  }
  void detectMainFileEOL();
private:
  void FileChanged(SourceLocation Loc, FileChangeReason Reason,
                   SrcMgr::CharacteristicKind FileType,
                   FileID PrevFID) override;
  void FileSkipped(const FileEntry &SkippedFile, const Token &FilenameTok,
                   SrcMgr::CharacteristicKind FileType) override;
  void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
                          StringRef FileName, bool IsAngled,
                          CharSourceRange FilenameRange, const FileEntry *File,
                          StringRef SearchPath, StringRef RelativePath,
                          const Module *Imported) override;
  void WriteLineInfo(const char *Filename, int Line,
                     SrcMgr::CharacteristicKind FileType,
                     StringRef Extra = StringRef());
  void WriteImplicitModuleImport(const Module *Mod);
  void OutputContentUpTo(const MemoryBuffer &FromFile,
                         unsigned &WriteFrom, unsigned WriteTo,
                         StringRef EOL, int &lines,
                         bool EnsureNewline);
  void CommentOutDirective(Lexer &DirectivesLex, const Token &StartToken,
                           const MemoryBuffer &FromFile, StringRef EOL,
                           unsigned &NextToWrite, int &Lines);
  bool HandleHasInclude(FileID FileId, Lexer &RawLex,
                        const DirectoryLookup *Lookup, Token &Tok,
                        bool &FileExists);
  const IncludedFile *FindIncludeAtLocation(SourceLocation Loc) const;
  const Module *FindModuleAtLocation(SourceLocation Loc) const;
  StringRef NextIdentifierName(Lexer &RawLex, Token &RawToken);
};

}  // end anonymous namespace

/// Initializes an InclusionRewriter with a \p PP source and \p OS destination.
InclusionRewriter::InclusionRewriter(Preprocessor &PP, raw_ostream &OS,
                                     bool ShowLineMarkers,
                                     bool UseLineDirectives)
    : PP(PP), SM(PP.getSourceManager()), OS(OS), MainEOL("\n"),
      PredefinesBuffer(nullptr), ShowLineMarkers(ShowLineMarkers),
      UseLineDirectives(UseLineDirectives),
      LastInclusionLocation(SourceLocation()) {}

/// Write appropriate line information as either #line directives or GNU line
/// markers depending on what mode we're in, including the \p Filename and
/// \p Line we are located at, using the specified \p EOL line separator, and
/// any \p Extra context specifiers in GNU line directives.
void InclusionRewriter::WriteLineInfo(const char *Filename, int Line,
                                      SrcMgr::CharacteristicKind FileType,
                                      StringRef Extra) {
  if (!ShowLineMarkers)
    return;
  if (UseLineDirectives) {
    OS << "#line" << ' ' << Line << ' ' << '"';
    OS.write_escaped(Filename);
    OS << '"';
  } else {
    // Use GNU linemarkers as described here:
    // http://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html
    OS << '#' << ' ' << Line << ' ' << '"';
    OS.write_escaped(Filename);
    OS << '"';
    if (!Extra.empty())
      OS << Extra;
    if (FileType == SrcMgr::C_System)
      // "`3' This indicates that the following text comes from a system header
      // file, so certain warnings should be suppressed."
      OS << " 3";
    else if (FileType == SrcMgr::C_ExternCSystem)
      // as above for `3', plus "`4' This indicates that the following text
      // should be treated as being wrapped in an implicit extern "C" block."
      OS << " 3 4";
  }
  OS << MainEOL;
}

void InclusionRewriter::WriteImplicitModuleImport(const Module *Mod) {
  OS << "@import " << Mod->getFullModuleName() << ";"
     << " /* clang -frewrite-includes: implicit import */" << MainEOL;
}

/// FileChanged - Whenever the preprocessor enters or exits a #include file
/// it invokes this handler.
void InclusionRewriter::FileChanged(SourceLocation Loc,
                                    FileChangeReason Reason,
                                    SrcMgr::CharacteristicKind NewFileType,
                                    FileID) {
  if (Reason != EnterFile)
    return;
  if (LastInclusionLocation.isInvalid())
    // we didn't reach this file (eg: the main file) via an inclusion directive
    return;
  FileID Id = FullSourceLoc(Loc, SM).getFileID();
  auto P = FileIncludes.insert(std::make_pair(
      LastInclusionLocation.getRawEncoding(), IncludedFile(Id, NewFileType)));
  assert(P.second && "Unexpected revisitation of the same include directive");
  LastInclusionLocation = SourceLocation();
}

/// Called whenever an inclusion is skipped due to canonical header protection
/// macros.
void InclusionRewriter::FileSkipped(const FileEntry &/*SkippedFile*/,
                                    const Token &/*FilenameTok*/,
                                    SrcMgr::CharacteristicKind /*FileType*/) {
  assert(!LastInclusionLocation.isInvalid() &&
         "A file, that wasn't found via an inclusion directive, was skipped");
  LastInclusionLocation = SourceLocation();
}

/// This should be called whenever the preprocessor encounters include
/// directives. It does not say whether the file has been included, but it
/// provides more information about the directive (hash location instead
/// of location inside the included file). It is assumed that the matching
/// FileChanged() or FileSkipped() is called after this.
void InclusionRewriter::InclusionDirective(SourceLocation HashLoc,
                                           const Token &/*IncludeTok*/,
                                           StringRef /*FileName*/,
                                           bool /*IsAngled*/,
                                           CharSourceRange /*FilenameRange*/,
                                           const FileEntry * /*File*/,
                                           StringRef /*SearchPath*/,
                                           StringRef /*RelativePath*/,
                                           const Module *Imported) {
  assert(LastInclusionLocation.isInvalid() &&
         "Another inclusion directive was found before the previous one "
         "was processed");
  if (Imported) {
    auto P = ModuleIncludes.insert(
        std::make_pair(HashLoc.getRawEncoding(), Imported));
    assert(P.second && "Unexpected revisitation of the same include directive");
  } else
    LastInclusionLocation = HashLoc;
}

/// Simple lookup for a SourceLocation (specifically one denoting the hash in
/// an inclusion directive) in the map of inclusion information, FileChanges.
const InclusionRewriter::IncludedFile *<commit_msg>Fix a couple of unused variables in no-asserts<commit_after>//===--- InclusionRewriter.cpp - Rewrite includes into their expansions ---===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This code rewrites include invocations into their expansions.  This gives you
// a file with all included files merged into it.
//
//===----------------------------------------------------------------------===//

#include "clang/Rewrite/Frontend/Rewriters.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Frontend/PreprocessorOutputOptions.h"
#include "clang/Lex/HeaderSearch.h"
#include "clang/Lex/Pragma.h"
#include "clang/Lex/Preprocessor.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/Support/raw_ostream.h"

using namespace clang;
using namespace llvm;

namespace {

class InclusionRewriter : public PPCallbacks {
  /// Information about which #includes were actually performed,
  /// created by preprocessor callbacks.
  struct IncludedFile {
    FileID Id;
    SrcMgr::CharacteristicKind FileType;
    IncludedFile(FileID Id, SrcMgr::CharacteristicKind FileType)
        : Id(Id), FileType(FileType) {}
  };
  Preprocessor &PP; ///< Used to find inclusion directives.
  SourceManager &SM; ///< Used to read and manage source files.
  raw_ostream &OS; ///< The destination stream for rewritten contents.
  StringRef MainEOL; ///< The line ending marker to use.
  const llvm::MemoryBuffer *PredefinesBuffer; ///< The preprocessor predefines.
  bool ShowLineMarkers; ///< Show #line markers.
  bool UseLineDirectives; ///< Use of line directives or line markers.
  /// Tracks where inclusions that change the file are found.
  std::map<unsigned, IncludedFile> FileIncludes;
  /// Tracks where inclusions that import modules are found.
  std::map<unsigned, const Module *> ModuleIncludes;
  /// Used transitively for building up the FileIncludes mapping over the
  /// various \c PPCallbacks callbacks.
  SourceLocation LastInclusionLocation;
public:
  InclusionRewriter(Preprocessor &PP, raw_ostream &OS, bool ShowLineMarkers,
                    bool UseLineDirectives);
  bool Process(FileID FileId, SrcMgr::CharacteristicKind FileType);
  void setPredefinesBuffer(const llvm::MemoryBuffer *Buf) {
    PredefinesBuffer = Buf;
  }
  void detectMainFileEOL();
private:
  void FileChanged(SourceLocation Loc, FileChangeReason Reason,
                   SrcMgr::CharacteristicKind FileType,
                   FileID PrevFID) override;
  void FileSkipped(const FileEntry &SkippedFile, const Token &FilenameTok,
                   SrcMgr::CharacteristicKind FileType) override;
  void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
                          StringRef FileName, bool IsAngled,
                          CharSourceRange FilenameRange, const FileEntry *File,
                          StringRef SearchPath, StringRef RelativePath,
                          const Module *Imported) override;
  void WriteLineInfo(const char *Filename, int Line,
                     SrcMgr::CharacteristicKind FileType,
                     StringRef Extra = StringRef());
  void WriteImplicitModuleImport(const Module *Mod);
  void OutputContentUpTo(const MemoryBuffer &FromFile,
                         unsigned &WriteFrom, unsigned WriteTo,
                         StringRef EOL, int &lines,
                         bool EnsureNewline);
  void CommentOutDirective(Lexer &DirectivesLex, const Token &StartToken,
                           const MemoryBuffer &FromFile, StringRef EOL,
                           unsigned &NextToWrite, int &Lines);
  bool HandleHasInclude(FileID FileId, Lexer &RawLex,
                        const DirectoryLookup *Lookup, Token &Tok,
                        bool &FileExists);
  const IncludedFile *FindIncludeAtLocation(SourceLocation Loc) const;
  const Module *FindModuleAtLocation(SourceLocation Loc) const;
  StringRef NextIdentifierName(Lexer &RawLex, Token &RawToken);
};

}  // end anonymous namespace

/// Initializes an InclusionRewriter with a \p PP source and \p OS destination.
InclusionRewriter::InclusionRewriter(Preprocessor &PP, raw_ostream &OS,
                                     bool ShowLineMarkers,
                                     bool UseLineDirectives)
    : PP(PP), SM(PP.getSourceManager()), OS(OS), MainEOL("\n"),
      PredefinesBuffer(nullptr), ShowLineMarkers(ShowLineMarkers),
      UseLineDirectives(UseLineDirectives),
      LastInclusionLocation(SourceLocation()) {}

/// Write appropriate line information as either #line directives or GNU line
/// markers depending on what mode we're in, including the \p Filename and
/// \p Line we are located at, using the specified \p EOL line separator, and
/// any \p Extra context specifiers in GNU line directives.
void InclusionRewriter::WriteLineInfo(const char *Filename, int Line,
                                      SrcMgr::CharacteristicKind FileType,
                                      StringRef Extra) {
  if (!ShowLineMarkers)
    return;
  if (UseLineDirectives) {
    OS << "#line" << ' ' << Line << ' ' << '"';
    OS.write_escaped(Filename);
    OS << '"';
  } else {
    // Use GNU linemarkers as described here:
    // http://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html
    OS << '#' << ' ' << Line << ' ' << '"';
    OS.write_escaped(Filename);
    OS << '"';
    if (!Extra.empty())
      OS << Extra;
    if (FileType == SrcMgr::C_System)
      // "`3' This indicates that the following text comes from a system header
      // file, so certain warnings should be suppressed."
      OS << " 3";
    else if (FileType == SrcMgr::C_ExternCSystem)
      // as above for `3', plus "`4' This indicates that the following text
      // should be treated as being wrapped in an implicit extern "C" block."
      OS << " 3 4";
  }
  OS << MainEOL;
}

void InclusionRewriter::WriteImplicitModuleImport(const Module *Mod) {
  OS << "@import " << Mod->getFullModuleName() << ";"
     << " /* clang -frewrite-includes: implicit import */" << MainEOL;
}

/// FileChanged - Whenever the preprocessor enters or exits a #include file
/// it invokes this handler.
void InclusionRewriter::FileChanged(SourceLocation Loc,
                                    FileChangeReason Reason,
                                    SrcMgr::CharacteristicKind NewFileType,
                                    FileID) {
  if (Reason != EnterFile)
    return;
  if (LastInclusionLocation.isInvalid())
    // we didn't reach this file (eg: the main file) via an inclusion directive
    return;
  FileID Id = FullSourceLoc(Loc, SM).getFileID();
  auto P = FileIncludes.insert(std::make_pair(
      LastInclusionLocation.getRawEncoding(), IncludedFile(Id, NewFileType)));
  (void)P;
  assert(P.second && "Unexpected revisitation of the same include directive");
  LastInclusionLocation = SourceLocation();
}

/// Called whenever an inclusion is skipped due to canonical header protection
/// macros.
void InclusionRewriter::FileSkipped(const FileEntry &/*SkippedFile*/,
                                    const Token &/*FilenameTok*/,
                                    SrcMgr::CharacteristicKind /*FileType*/) {
  assert(!LastInclusionLocation.isInvalid() &&
         "A file, that wasn't found via an inclusion directive, was skipped");
  LastInclusionLocation = SourceLocation();
}

/// This should be called whenever the preprocessor encounters include
/// directives. It does not say whether the file has been included, but it
/// provides more information about the directive (hash location instead
/// of location inside the included file). It is assumed that the matching
/// FileChanged() or FileSkipped() is called after this.
void InclusionRewriter::InclusionDirective(SourceLocation HashLoc,
                                           const Token &/*IncludeTok*/,
                                           StringRef /*FileName*/,
                                           bool /*IsAngled*/,
                                           CharSourceRange /*FilenameRange*/,
                                           const FileEntry * /*File*/,
                                           StringRef /*SearchPath*/,
                                           StringRef /*RelativePath*/,
                                           const Module *Imported) {
  assert(LastInclusionLocation.isInvalid() &&
         "Another inclusion directive was found before the previous one "
         "was processed");
  if (Imported) {
    auto P = ModuleIncludes.insert(
        std::make_pair(HashLoc.getRawEncoding(), Imported));
    (void)P;
    assert(P.second && "Unexpected revisitation of the same include directive");
  } else
    LastInclusionLocation = HashLoc;
}

/// Simple lookup for a SourceLocation (specifically one denoting the hash in
/// an inclusion directive) in the map of inclusion information, FileChanges.
const InclusionRewriter::IncludedFile *
{'fraction_non_alphanumeric': 0.11103778254413463, 'fraction_numerical': 0.0006599571027883187, 'mean_word_length': 2.385589275740086, 'pattern_counts': {'":': 0, '<': 105, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 54, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '662407', 'n_tokens_mistral': 4697, 'n_tokens_neox': 4502, 'n_words': 1469}