<commit_before>//===--- PrintPreprocessedOutput.cpp - Implement the -E mode --------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This code simply runs the preprocessor on the input file and prints out the
// result.  This is the traditional behavior of the -E option.
//
//===----------------------------------------------------------------------===//

#include "clang.h"
#include "clang/Lex/PPCallbacks.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/Pragma.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/Diagnostic.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/System/Path.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Config/config.h"
#include "llvm/Support/raw_ostream.h"
#include <cstdio>
using namespace clang;

//===----------------------------------------------------------------------===//
// Preprocessed token printer
//===----------------------------------------------------------------------===//

static llvm::cl::opt<bool>
DisableLineMarkers("P", llvm::cl::desc("Disable linemarker output in -E mode"));
static llvm::cl::opt<bool>
EnableCommentOutput("C", llvm::cl::desc("Enable comment output in -E mode"));
static llvm::cl::opt<bool>
EnableMacroCommentOutput("CC",
                         llvm::cl::desc("Enable comment output in -E mode, "
                                        "even from macro expansions"));

namespace {
class PrintPPOutputPPCallbacks : public PPCallbacks {
  Preprocessor &PP;
public:
  llvm::raw_ostream &OS;
private:
  unsigned CurLine;
  bool EmittedTokensOnThisLine;
  DirectoryLookup::DirType FileType;
  llvm::SmallString<512> CurFilename;
public:
  PrintPPOutputPPCallbacks(Preprocessor &pp, llvm::raw_ostream &os)
     : PP(pp), OS(os) {
    CurLine = 0;
    CurFilename += "<uninit>";
    EmittedTokensOnThisLine = false;
    FileType = DirectoryLookup::NormalHeaderDir;
  }
  
  void SetEmittedTokensOnThisLine() { EmittedTokensOnThisLine = true; }
  bool hasEmittedTokensOnThisLine() const { return EmittedTokensOnThisLine; }
  
  virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
                           DirectoryLookup::DirType FileType);
  virtual void Ident(SourceLocation Loc, const std::string &str);
  

  bool HandleFirstTokOnLine(Token &Tok);
  bool MoveToLine(SourceLocation Loc);
  bool AvoidConcat(const Token &PrevTok, const Token &Tok);
};
}  // end anonymous namespace

/// UToStr - Do itoa on the specified number, in-place in the specified buffer.
/// endptr points to the end of the buffer.
static char *UToStr(unsigned N, char *EndPtr) {
  // Null terminate the buffer.
  *--EndPtr = '\0';
  if (N == 0)          // Zero is a special case.
    *--EndPtr = '0';
  while (N) {
    *--EndPtr = '0' + char(N % 10);
    N /= 10;
  }
  return EndPtr;
}


/// MoveToLine - Move the output to the source line specified by the location
/// object.  We can do this by emitting some number of \n's, or be emitting a
/// #line directive.  This returns false if already at the specified line, true
/// if some newlines were emitted.
bool PrintPPOutputPPCallbacks::MoveToLine(SourceLocation Loc) {
  if (DisableLineMarkers) {
    unsigned LineNo = PP.getSourceManager().getLogicalLineNumber(Loc);
    if (LineNo == CurLine) return false;
    
    CurLine = LineNo;
    
    if (!EmittedTokensOnThisLine)
      return true;
    
    OS << '\n';
    EmittedTokensOnThisLine = false;
    return true;
  }
  
  unsigned LineNo = PP.getSourceManager().getLogicalLineNumber(Loc);
  
  // If this line is "close enough" to the original line, just print newlines,
  // otherwise print a #line directive.
  if (LineNo-CurLine < 8) {
    if (LineNo-CurLine == 1)
      OS << '\n';
    else if (LineNo == CurLine)
      return false;    // Phys line moved, but logical line didn't.
    else {
      const char *NewLines = "\n\n\n\n\n\n\n\n";
      OS.write(NewLines, LineNo-CurLine);
    }
    CurLine = LineNo;
  } else {
    if (EmittedTokensOnThisLine) {
      OS << '\n';
      EmittedTokensOnThisLine = false;
    }
    
    CurLine = LineNo;
    
    OS << '#' << ' ';
    char NumberBuffer[20];
    const char *NumStr = UToStr(LineNo, NumberBuffer+20);
    OS.write(NumStr, (NumberBuffer+20)-NumStr-1);
    OS << ' ';
    OS << '"';
    OS.write(&CurFilename[0], CurFilename.size());
    OS << '"';
    
    if (FileType == DirectoryLookup::SystemHeaderDir)
      OS.write(" 3", 2);
    else if (FileType == DirectoryLookup::ExternCSystemHeaderDir)
      OS.write(" 3 4", 4);
    OS << '\n';
  } 
  return true;
}


/// FileChanged - Whenever the preprocessor enters or exits a #include file
/// it invokes this handler.  Update our conception of the current source
/// position.
void PrintPPOutputPPCallbacks::FileChanged(SourceLocation Loc,
                                           FileChangeReason Reason,
                                           DirectoryLookup::DirType FileType) {
  // Unless we are exiting a #include, make sure to skip ahead to the line the
  // #include directive was at.
  SourceManager &SourceMgr = PP.getSourceManager();
  if (Reason == PPCallbacks::EnterFile) {
    MoveToLine(SourceMgr.getIncludeLoc(Loc));
  } else if (Reason == PPCallbacks::SystemHeaderPragma) {
    MoveToLine(Loc);
    
    // TODO GCC emits the # directive for this directive on the line AFTER the
    // directive and emits a bunch of spaces that aren't needed.  Emulate this
    // strange behavior.
  }
  
  Loc = SourceMgr.getLogicalLoc(Loc);
  CurLine = SourceMgr.getLineNumber(Loc);
  
  if (DisableLineMarkers) return;

  CurFilename.clear();
  CurFilename += SourceMgr.getSourceName(Loc);
  Lexer::Stringify(CurFilename);
  FileType = FileType;
  
  if (EmittedTokensOnThisLine) {
    OS << '\n';
    EmittedTokensOnThisLine = false;
  }
  
  OS << '#' << ' ';
  
  char NumberBuffer[20];
  const char *NumStr = UToStr(CurLine, NumberBuffer+20);
  OS.write(NumStr, (NumberBuffer+20)-NumStr-1);
  OS << ' ' << '"';
  OS.write(&CurFilename[0], CurFilename.size());
  OS << '"';
  
  switch (Reason) {
  case PPCallbacks::EnterFile:
    OS.write(" 1", 2);
    break;
  case PPCallbacks::ExitFile:
    OS.write(" 2", 2);
    break;
  case PPCallbacks::SystemHeaderPragma: break;
  case PPCallbacks::RenameFile: break;
  }
  
  if (FileType == DirectoryLookup::SystemHeaderDir)
    OS.write(" 3", 2);
  else if (FileType == DirectoryLookup::ExternCSystemHeaderDir)
    OS.write(" 3 4", 4);
  
  OS << '\n';
}

/// HandleIdent - Handle #ident directives when read by the preprocessor.
///
void PrintPPOutputPPCallbacks::Ident(SourceLocation Loc, const std::string &S) {
  MoveToLine(Loc);
  
  OS.write("#ident ", strlen("#ident "));
  OS.write(&S[0], S.size());
  EmittedTokensOnThisLine = true;<commit_msg>Use raw_ostream to output an unsigned.<commit_after>//===--- PrintPreprocessedOutput.cpp - Implement the -E mode --------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This code simply runs the preprocessor on the input file and prints out the
// result.  This is the traditional behavior of the -E option.
//
//===----------------------------------------------------------------------===//

#include "clang.h"
#include "clang/Lex/PPCallbacks.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/Pragma.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/Diagnostic.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/System/Path.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Config/config.h"
#include "llvm/Support/raw_ostream.h"
#include <cstdio>
using namespace clang;

//===----------------------------------------------------------------------===//
// Preprocessed token printer
//===----------------------------------------------------------------------===//

static llvm::cl::opt<bool>
DisableLineMarkers("P", llvm::cl::desc("Disable linemarker output in -E mode"));
static llvm::cl::opt<bool>
EnableCommentOutput("C", llvm::cl::desc("Enable comment output in -E mode"));
static llvm::cl::opt<bool>
EnableMacroCommentOutput("CC",
                         llvm::cl::desc("Enable comment output in -E mode, "
                                        "even from macro expansions"));

namespace {
class PrintPPOutputPPCallbacks : public PPCallbacks {
  Preprocessor &PP;
public:
  llvm::raw_ostream &OS;
private:
  unsigned CurLine;
  bool EmittedTokensOnThisLine;
  DirectoryLookup::DirType FileType;
  llvm::SmallString<512> CurFilename;
public:
  PrintPPOutputPPCallbacks(Preprocessor &pp, llvm::raw_ostream &os)
     : PP(pp), OS(os) {
    CurLine = 0;
    CurFilename += "<uninit>";
    EmittedTokensOnThisLine = false;
    FileType = DirectoryLookup::NormalHeaderDir;
  }
  
  void SetEmittedTokensOnThisLine() { EmittedTokensOnThisLine = true; }
  bool hasEmittedTokensOnThisLine() const { return EmittedTokensOnThisLine; }
  
  virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
                           DirectoryLookup::DirType FileType);
  virtual void Ident(SourceLocation Loc, const std::string &str);
  

  bool HandleFirstTokOnLine(Token &Tok);
  bool MoveToLine(SourceLocation Loc);
  bool AvoidConcat(const Token &PrevTok, const Token &Tok);
};
}  // end anonymous namespace

/// MoveToLine - Move the output to the source line specified by the location
/// object.  We can do this by emitting some number of \n's, or be emitting a
/// #line directive.  This returns false if already at the specified line, true
/// if some newlines were emitted.
bool PrintPPOutputPPCallbacks::MoveToLine(SourceLocation Loc) {
  if (DisableLineMarkers) {
    unsigned LineNo = PP.getSourceManager().getLogicalLineNumber(Loc);
    if (LineNo == CurLine) return false;
    
    CurLine = LineNo;
    
    if (!EmittedTokensOnThisLine)
      return true;
    
    OS << '\n';
    EmittedTokensOnThisLine = false;
    return true;
  }
  
  unsigned LineNo = PP.getSourceManager().getLogicalLineNumber(Loc);
  
  // If this line is "close enough" to the original line, just print newlines,
  // otherwise print a #line directive.
  if (LineNo-CurLine < 8) {
    if (LineNo-CurLine == 1)
      OS << '\n';
    else if (LineNo == CurLine)
      return false;    // Phys line moved, but logical line didn't.
    else {
      const char *NewLines = "\n\n\n\n\n\n\n\n";
      OS.write(NewLines, LineNo-CurLine);
    }
    CurLine = LineNo;
  } else {
    if (EmittedTokensOnThisLine) {
      OS << '\n';
      EmittedTokensOnThisLine = false;
    }
    
    CurLine = LineNo;
    
    OS << '#' << ' ' << LineNo << ' ' << '"';
    OS.write(&CurFilename[0], CurFilename.size());
    OS << '"';
    
    if (FileType == DirectoryLookup::SystemHeaderDir)
      OS.write(" 3", 2);
    else if (FileType == DirectoryLookup::ExternCSystemHeaderDir)
      OS.write(" 3 4", 4);
    OS << '\n';
  } 
  return true;
}


/// FileChanged - Whenever the preprocessor enters or exits a #include file
/// it invokes this handler.  Update our conception of the current source
/// position.
void PrintPPOutputPPCallbacks::FileChanged(SourceLocation Loc,
                                           FileChangeReason Reason,
                                           DirectoryLookup::DirType FileType) {
  // Unless we are exiting a #include, make sure to skip ahead to the line the
  // #include directive was at.
  SourceManager &SourceMgr = PP.getSourceManager();
  if (Reason == PPCallbacks::EnterFile) {
    MoveToLine(SourceMgr.getIncludeLoc(Loc));
  } else if (Reason == PPCallbacks::SystemHeaderPragma) {
    MoveToLine(Loc);
    
    // TODO GCC emits the # directive for this directive on the line AFTER the
    // directive and emits a bunch of spaces that aren't needed.  Emulate this
    // strange behavior.
  }
  
  Loc = SourceMgr.getLogicalLoc(Loc);
  CurLine = SourceMgr.getLineNumber(Loc);
  
  if (DisableLineMarkers) return;

  CurFilename.clear();
  CurFilename += SourceMgr.getSourceName(Loc);
  Lexer::Stringify(CurFilename);
  FileType = FileType;
  
  if (EmittedTokensOnThisLine) {
    OS << '\n';
    EmittedTokensOnThisLine = false;
  }
  
  OS << '#' << ' ' << CurLine << ' ' << '"';
  OS.write(&CurFilename[0], CurFilename.size());
  OS << '"';
  
  switch (Reason) {
  case PPCallbacks::EnterFile:
    OS.write(" 1", 2);
    break;
  case PPCallbacks::ExitFile:
    OS.write(" 2", 2);
    break;
  case PPCallbacks::SystemHeaderPragma: break;
  case PPCallbacks::RenameFile: break;
  }
  
  if (FileType == DirectoryLookup::SystemHeaderDir)
    OS.write(" 3", 2);
  else if (FileType == DirectoryLookup::ExternCSystemHeaderDir)
    OS.write(" 3 4", 4);
  
  OS << '\n';
}

/// HandleIdent - Handle #ident directives when read by the preprocessor.
///
void PrintPPOutputPPCallbacks::Ident(SourceLocation Loc, const std::string &S) {
  MoveToLine(Loc);
  
  OS.write("#ident ", strlen("#ident "));
  OS.write(&S[0], S.size());
  EmittedTokensOnThisLine = true;
{'fraction_non_alphanumeric': 0.17126204886796684, 'fraction_numerical': 0.005081073003063588, 'mean_word_length': 2.6369565217391306, 'pattern_counts': {'":': 0, '<': 85, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1988576', 'n_tokens_mistral': 4095, 'n_tokens_neox': 4012, 'n_words': 1091}