<commit_before>//===--- PCHReader.cpp - Precompiled Headers Reader -------------*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//  This file defines the PCHReader class, which reads a precompiled header.
//
//===----------------------------------------------------------------------===//

#include "clang/Frontend/PCHReader.h"
#include "clang/Frontend/FrontendDiagnostic.h"
#include "../Sema/Sema.h" // FIXME: move Sema headers elsewhere
#include "clang/AST/ASTConsumer.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/Expr.h"
#include "clang/AST/Type.h"
#include "clang/Lex/MacroInfo.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/HeaderSearch.h"
#include "clang/Basic/OnDiskHashTable.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/SourceManagerInternals.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/TargetInfo.h"
#include "llvm/Bitcode/BitstreamReader.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/MemoryBuffer.h"
#include <algorithm>
#include <iterator>
#include <cstdio>
#include <sys/stat.h>
using namespace clang;

//===----------------------------------------------------------------------===//
// PCH reader validator implementation
//===----------------------------------------------------------------------===//

PCHReaderListener::~PCHReaderListener() {}

bool
PCHValidator::ReadLanguageOptions(const LangOptions &LangOpts) {
  const LangOptions &PPLangOpts = PP.getLangOptions();
#define PARSE_LANGOPT_BENIGN(Option)
#define PARSE_LANGOPT_IMPORTANT(Option, DiagID)                    \
  if (PPLangOpts.Option != LangOpts.Option) {                      \
    Reader.Diag(DiagID) << LangOpts.Option << PPLangOpts.Option;   \
    return true;                                                   \
  }

  PARSE_LANGOPT_BENIGN(Trigraphs);
  PARSE_LANGOPT_BENIGN(BCPLComment);
  PARSE_LANGOPT_BENIGN(DollarIdents);
  PARSE_LANGOPT_BENIGN(AsmPreprocessor);
  PARSE_LANGOPT_IMPORTANT(GNUMode, diag::warn_pch_gnu_extensions);
  PARSE_LANGOPT_BENIGN(ImplicitInt);
  PARSE_LANGOPT_BENIGN(Digraphs);
  PARSE_LANGOPT_BENIGN(HexFloats);
  PARSE_LANGOPT_IMPORTANT(C99, diag::warn_pch_c99);
  PARSE_LANGOPT_IMPORTANT(Microsoft, diag::warn_pch_microsoft_extensions);
  PARSE_LANGOPT_IMPORTANT(CPlusPlus, diag::warn_pch_cplusplus);
  PARSE_LANGOPT_IMPORTANT(CPlusPlus0x, diag::warn_pch_cplusplus0x);
  PARSE_LANGOPT_BENIGN(CXXOperatorName);
  PARSE_LANGOPT_IMPORTANT(ObjC1, diag::warn_pch_objective_c);
  PARSE_LANGOPT_IMPORTANT(ObjC2, diag::warn_pch_objective_c2);
  PARSE_LANGOPT_IMPORTANT(ObjCNonFragileABI, diag::warn_pch_nonfragile_abi);
  PARSE_LANGOPT_BENIGN(PascalStrings);
  PARSE_LANGOPT_BENIGN(WritableStrings);
  PARSE_LANGOPT_IMPORTANT(LaxVectorConversions, 
                          diag::warn_pch_lax_vector_conversions);
  PARSE_LANGOPT_IMPORTANT(Exceptions, diag::warn_pch_exceptions);
  PARSE_LANGOPT_IMPORTANT(NeXTRuntime, diag::warn_pch_objc_runtime);
  PARSE_LANGOPT_IMPORTANT(Freestanding, diag::warn_pch_freestanding);
  PARSE_LANGOPT_IMPORTANT(NoBuiltin, diag::warn_pch_builtins);
  PARSE_LANGOPT_IMPORTANT(ThreadsafeStatics, 
                          diag::warn_pch_thread_safe_statics);
  PARSE_LANGOPT_IMPORTANT(Blocks, diag::warn_pch_blocks);
  PARSE_LANGOPT_BENIGN(EmitAllDecls);
  PARSE_LANGOPT_IMPORTANT(MathErrno, diag::warn_pch_math_errno);
  PARSE_LANGOPT_IMPORTANT(OverflowChecking, diag::warn_pch_overflow_checking);
  PARSE_LANGOPT_IMPORTANT(HeinousExtensions, 
                          diag::warn_pch_heinous_extensions);
  // FIXME: Most of the options below are benign if the macro wasn't
  // used. Unfortunately, this means that a PCH compiled without
  // optimization can't be used with optimization turned on, even
  // though the only thing that changes is whether __OPTIMIZE__ was
  // defined... but if __OPTIMIZE__ never showed up in the header, it
  // doesn't matter. We could consider making this some special kind
  // of check.
  PARSE_LANGOPT_IMPORTANT(Optimize, diag::warn_pch_optimize);
  PARSE_LANGOPT_IMPORTANT(OptimizeSize, diag::warn_pch_optimize_size);
  PARSE_LANGOPT_IMPORTANT(Static, diag::warn_pch_static);
  PARSE_LANGOPT_IMPORTANT(PICLevel, diag::warn_pch_pic_level);
  PARSE_LANGOPT_IMPORTANT(GNUInline, diag::warn_pch_gnu_inline);
  PARSE_LANGOPT_IMPORTANT(NoInline, diag::warn_pch_no_inline);
  PARSE_LANGOPT_IMPORTANT(AccessControl, diag::warn_pch_access_control);
  PARSE_LANGOPT_IMPORTANT(CharIsSigned, diag::warn_pch_char_signed);
  if ((PPLangOpts.getGCMode() != 0) != (LangOpts.getGCMode() != 0)) {
    Reader.Diag(diag::warn_pch_gc_mode) 
      << LangOpts.getGCMode() << PPLangOpts.getGCMode();
    return true;
  }
  PARSE_LANGOPT_BENIGN(getVisibilityMode());
  PARSE_LANGOPT_BENIGN(InstantiationDepth);
#undef PARSE_LANGOPT_IRRELEVANT
#undef PARSE_LANGOPT_BENIGN

  return false;
}

bool PCHValidator::ReadTargetTriple(const std::string &Triple) {
  if (Triple != PP.getTargetInfo().getTargetTriple()) {
    Reader.Diag(diag::warn_pch_target_triple)
      << Triple << PP.getTargetInfo().getTargetTriple();
    return true;
  }
  return false;
}

/// \brief Split the given string into a vector of lines, eliminating
/// any empty lines in the process.
///
/// \param Str the string to split.
/// \param Len the length of Str.
/// \param KeepEmptyLines true if empty lines should be included
/// \returns a vector of lines, with the line endings removed
static std::vector<std::string> splitLines(const char *Str, unsigned Len,
                                           bool KeepEmptyLines = false) {
  std::vector<std::string> Lines;
  for (unsigned LineStart = 0; LineStart < Len; ++LineStart) {
    unsigned LineEnd = LineStart;
    while (LineEnd < Len && Str[LineEnd] != '\n')
      ++LineEnd;
    if (LineStart != LineEnd || KeepEmptyLines)
      Lines.push_back(std::string(&Str[LineStart], &Str[LineEnd]));
    LineStart = LineEnd;
  }
  return Lines;
}

/// \brief Determine whether the string Haystack starts with the
/// substring Needle.
static bool startsWith(const std::string &Haystack, const char *Needle) {
  for (unsigned I = 0, N = Haystack.size(); Needle[I] != 0; ++I) {
    if (I == N)
      return false;
    if (Haystack[I] != Needle[I])
      return false;
  }

  return true;
}

/// \brief Determine whether the string Haystack starts with the
/// substring Needle.
static inline bool startsWith(const std::string &Haystack,
                              const std::string &Needle) {
  return startsWith(Haystack, Needle.c_str());
}

bool PCHValidator::ReadPredefinesBuffer(const char *PCHPredef, 
                                        unsigned PCHPredefLen,
                                        FileID PCHBufferID,
                                        std::string &SuggestedPredefines) {
  const char *Predef = PP.getPredefines().c_str();
  unsigned PredefLen = PP.getPredefines().size();

  // If the two predefines buffers compare equal, we're done!
  if (PredefLen == PCHPredefLen && 
      strncmp(Predef, PCHPredef, PCHPredefLen) == 0)
    return false;

  SourceManager &SourceMgr = PP.getSourceManager();
  
  // The predefines buffers are different. Determine what the
  // differences are, and whether they require us to reject the PCH
  // file.
  std::vector<std::string> CmdLineLines = splitLines(Predef, PredefLen);
  std::vector<std::string> PCHLines = splitLines(PCHPredef, PCHPredefLen);

  // Sort both sets of predefined buffer lines, since 
  std::sort(CmdLineLines.begin(), CmdLineLines.end());
  std::sort(PCHLines.begin(), PCHLines.end());

  // Determine which predefines that where used to build the PCH file
  // are missing from the command line.
  std::vector<std::string> MissingPredefines;
  std::set_difference(PCHLines.begin(), PCHLines.end(),
                      CmdLineLines.begin(), CmdLineLines.end(),
                      std::back_inserter(MissingPredefines));

  bool MissingDefines = false;
  bool ConflictingDefines = false;
  for (unsigned I = 0, N = MissingPredefines.size(); I != N; ++I) {
    const std::string &Missing = MissingPredefines[I];
    if (!startsWith(Missing, "#define ") != 0) {
      Reader.Diag(diag::warn_pch_compiler_options_mismatch);
      return true;
    }
    
    // This is a macro definition. Determine the name of the macro
    // we're defining.
    std::string::size_type StartOfMacroName = strlen("#define ");
    std::string::size_type EndOfMacroName 
      = Missing.find_first_of("( \n\r", StartOfMacroName);
    assert(EndOfMacroName != std::string::npos &&
           "Couldn't find the end of the macro name");
    std::string MacroName = Missing.substr(StartOfMacroName,
                                           EndOfMacroName - StartOfMacroName);

    // Determine whether this macro was given a different definition
    // on the command line.
    std::string MacroDefStart = "#define " + MacroName;
    std::string::size_type MacroDefLen = MacroDefStart.size();
    std::vector<std::string>::iterator ConflictPos
      = std::lower_bound(CmdLineLines.begin(), CmdLineLines.end(),
                         MacroDefStart);
    for (; ConflictPos != CmdLineLines.end(); ++ConflictPos) {
      if (!startsWith(*ConflictPos, MacroDefStart)) {
        // Different macro; we're done.
        ConflictPos = CmdLineLines.end();
        break; 
      }
      
      assert(ConflictPos->size() > MacroDefLen && 
             "Invalid #define in predefines buffer?");
      if ((*ConflictPos)[MacroDefLen] != ' ' && 
          (*ConflictPos)[MacroDefLen] != '(')
        continue; // Longer macro name; keep trying.
      
      // We found a conflicting macro definition.
      break;
    }
    
    if (ConflictPos != CmdLineLines.end()) {
      Reader.Diag(diag::warn_cmdline_conflicting_macro_def)
          << MacroName;

      // Show the definition of this macro within the PCH file.
      const char *MissingDef = strstr(PCHPredef, Missing.c_str());
      unsigned Offset = MissingDef - PCHPredef;
      SourceLocation PCHMissingLoc
        = SourceMgr.getLocForStartOfFile(PCHBufferID)
            .getFileLocWithOffset(Offset);
      Reader.Diag(PCHMissingLoc, diag::note_pch_macro_defined_as)
        << MacroName;

      ConflictingDefines = true;
      continue;
    }
    
    // If the macro doesn't conflict, then we'll just pick up the
    // macro definition from the PCH file. Warn the user that they
    // made a mistake.
    if (ConflictingDefines)
      continue; // Don't complain if there are already conflicting defs
    
    if (!MissingDefines) {
      Reader.Diag(diag::warn_cmdline_missing_macro_defs);
      MissingDefines = true;
    }

    // Show the definition of this macro within the PCH file.
    const char *MissingDef = strstr(PCHPredef, Missing.c_str());
    unsigned Offset = MissingDef - PCHPredef;
    SourceLocation PCHMissingLoc
      = SourceMgr.getLocForStartOfFile(PCHBufferID)
      .getFileLocWithOffset(Offset);
    Reader.Diag(PCHMissingLoc, diag::note_using_macro_def_from_pch);
  }
  
  if (ConflictingDefines)
    return true;
  
  // Determine what predefines were introduced based on command-line
  // parameters that were not present when building the PCH
  // file. Extra #defines are okay, so long as the identifiers being
  // defined were not used within the precompiled header.
  std::vector<std::string> ExtraPredefines;
  std::set_difference(CmdLineLines.begin(), CmdLineLines.end(),
                      PCHLines.begin(), PCHLines.end(),
                      std::back_inserter(ExtraPredefines));  
  for (unsigned I = 0, N = ExtraPredefines.size(); I != N; ++I) {
    const std::string &Extra = ExtraPredefines[I];
    if (!startsWith(Extra, "#define ") != 0) {
      Reader.Diag(diag::warn_pch_compiler_options_mismatch);
      return true;
    }

    // This is an extra macro definition. Determine the name of the
    // macro we're defining.
    std::string::size_type StartOfMacroName = strlen("#define ");
    std::string::size_type EndOfMacroName 
      = Extra.find_first_of("( \n\r", StartOfMacroName);
    assert(EndOfMacroName != std::string::npos &&
           "Couldn't find the end of the macro name");
    std::string MacroName = Extra.substr(StartOfMacroName,
                                         EndOfMacroName - StartOfMacroName);

    // Check whether this name was used somewhere in the PCH file. If
    // so, defining it as a macro could change behavior, so we reject
    // the PCH file.
    if (IdentifierInfo *II = Reader.get(MacroName.c_str(),
                                 MacroName.c_str() + MacroName.size())) {
      Reader.Diag(diag::warn_macro_name_used_in_pch)
        << II;
      return true;
    }

    // Add this definition to the suggested predefines buffer.
    SuggestedPredefines += Extra;
    SuggestedPredefines += '\n';
  }

  // If we get here, it's because the predefines buffer had compatible
  // contents. Accept the PCH file.
  return false;
}

void PCHValidator::ReadHeaderFileInfo(const HeaderFileInfo &HFI) {
  PP.getHeaderSearchInfo().setHeaderFileInfoForUID(HFI, NumHeaderInfos++);
}

void PCHValidator::ReadCounter(unsigned Value) {
  PP.setCounterValue(Value);
}



//===----------------------------------------------------------------------===//
// PCH reader implementation
//===----------------------------------------------------------------------===//

PCHReader::PCHReader(Preprocessor &PP, ASTContext *Context) 
  : Listener(new PCHValidator(PP, *this)), SourceMgr(PP.getSourceManager()),
    FileMgr(PP.getFileManager()), Diags(PP.getDiagnostics()),
    SemaObj(0), PP(&PP), Context(Context), Consumer(0),
    IdentifierTableData(0), IdentifierLookupTable(0),
    IdentifierOffsets(0),
    MethodPoolLookupTable(0), MethodPoolLookupTableData(0),
    TotalSelectorsInMethodPool(0), SelectorOffsets(0),
    TotalNumSelectors(0), NumStatHits(0), NumStatMisses(0), 
    NumSLocEntriesRead(0), NumStatementsRead(0), 
    NumMacrosRead(0), NumMethodPoolSelectorsRead(0), NumMethodPoolMisses(0),
    NumLexicalDeclContextsRead(0), NumVisibleDeclContextsRead(0) { }

PCHReader::PCHReader(SourceManager &SourceMgr, FileManager &FileMgr,
                     Diagnostic &Diags) 
  : SourceMgr(SourceMgr), FileMgr(FileMgr), Diags(Diags),
    SemaObj(0), PP(PP), Context(Context), Consumer(0),
    IdentifierTableData(0), IdentifierLookupTable(0),
    IdentifierOffsets(0),
    MethodPoolLookupTable(0), MethodPoolLookupTableData(0),
    TotalSelectorsInMethodPool(0), SelectorOffsets(0),
    TotalNumSelectors(0), NumStatHits(0), NumStatMisses(0), 
    NumSLocEntriesRead(0), NumStatementsRead(0), 
    NumMacrosRead(0), NumMethodPoolSelectorsRead(0), NumMethodPoolMisses(0),
    NumLexicalDeclContextsRead(0), NumVisibleDeclContextsRead(0) { }

PCHReader::~PCHReader() {}

Expr *PCHReader::ReadDeclExpr() {
  return dyn_cast_or_null<Expr>(ReadStmt(DeclsCursor));
}
<commit_msg>Fix bug in the new PCHReader constructor. PP and Context should be initialized to 0.<commit_after>//===--- PCHReader.cpp - Precompiled Headers Reader -------------*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//  This file defines the PCHReader class, which reads a precompiled header.
//
//===----------------------------------------------------------------------===//

#include "clang/Frontend/PCHReader.h"
#include "clang/Frontend/FrontendDiagnostic.h"
#include "../Sema/Sema.h" // FIXME: move Sema headers elsewhere
#include "clang/AST/ASTConsumer.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/Expr.h"
#include "clang/AST/Type.h"
#include "clang/Lex/MacroInfo.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/HeaderSearch.h"
#include "clang/Basic/OnDiskHashTable.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/SourceManagerInternals.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/TargetInfo.h"
#include "llvm/Bitcode/BitstreamReader.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/MemoryBuffer.h"
#include <algorithm>
#include <iterator>
#include <cstdio>
#include <sys/stat.h>
using namespace clang;

//===----------------------------------------------------------------------===//
// PCH reader validator implementation
//===----------------------------------------------------------------------===//

PCHReaderListener::~PCHReaderListener() {}

bool
PCHValidator::ReadLanguageOptions(const LangOptions &LangOpts) {
  const LangOptions &PPLangOpts = PP.getLangOptions();
#define PARSE_LANGOPT_BENIGN(Option)
#define PARSE_LANGOPT_IMPORTANT(Option, DiagID)                    \
  if (PPLangOpts.Option != LangOpts.Option) {                      \
    Reader.Diag(DiagID) << LangOpts.Option << PPLangOpts.Option;   \
    return true;                                                   \
  }

  PARSE_LANGOPT_BENIGN(Trigraphs);
  PARSE_LANGOPT_BENIGN(BCPLComment);
  PARSE_LANGOPT_BENIGN(DollarIdents);
  PARSE_LANGOPT_BENIGN(AsmPreprocessor);
  PARSE_LANGOPT_IMPORTANT(GNUMode, diag::warn_pch_gnu_extensions);
  PARSE_LANGOPT_BENIGN(ImplicitInt);
  PARSE_LANGOPT_BENIGN(Digraphs);
  PARSE_LANGOPT_BENIGN(HexFloats);
  PARSE_LANGOPT_IMPORTANT(C99, diag::warn_pch_c99);
  PARSE_LANGOPT_IMPORTANT(Microsoft, diag::warn_pch_microsoft_extensions);
  PARSE_LANGOPT_IMPORTANT(CPlusPlus, diag::warn_pch_cplusplus);
  PARSE_LANGOPT_IMPORTANT(CPlusPlus0x, diag::warn_pch_cplusplus0x);
  PARSE_LANGOPT_BENIGN(CXXOperatorName);
  PARSE_LANGOPT_IMPORTANT(ObjC1, diag::warn_pch_objective_c);
  PARSE_LANGOPT_IMPORTANT(ObjC2, diag::warn_pch_objective_c2);
  PARSE_LANGOPT_IMPORTANT(ObjCNonFragileABI, diag::warn_pch_nonfragile_abi);
  PARSE_LANGOPT_BENIGN(PascalStrings);
  PARSE_LANGOPT_BENIGN(WritableStrings);
  PARSE_LANGOPT_IMPORTANT(LaxVectorConversions, 
                          diag::warn_pch_lax_vector_conversions);
  PARSE_LANGOPT_IMPORTANT(Exceptions, diag::warn_pch_exceptions);
  PARSE_LANGOPT_IMPORTANT(NeXTRuntime, diag::warn_pch_objc_runtime);
  PARSE_LANGOPT_IMPORTANT(Freestanding, diag::warn_pch_freestanding);
  PARSE_LANGOPT_IMPORTANT(NoBuiltin, diag::warn_pch_builtins);
  PARSE_LANGOPT_IMPORTANT(ThreadsafeStatics, 
                          diag::warn_pch_thread_safe_statics);
  PARSE_LANGOPT_IMPORTANT(Blocks, diag::warn_pch_blocks);
  PARSE_LANGOPT_BENIGN(EmitAllDecls);
  PARSE_LANGOPT_IMPORTANT(MathErrno, diag::warn_pch_math_errno);
  PARSE_LANGOPT_IMPORTANT(OverflowChecking, diag::warn_pch_overflow_checking);
  PARSE_LANGOPT_IMPORTANT(HeinousExtensions, 
                          diag::warn_pch_heinous_extensions);
  // FIXME: Most of the options below are benign if the macro wasn't
  // used. Unfortunately, this means that a PCH compiled without
  // optimization can't be used with optimization turned on, even
  // though the only thing that changes is whether __OPTIMIZE__ was
  // defined... but if __OPTIMIZE__ never showed up in the header, it
  // doesn't matter. We could consider making this some special kind
  // of check.
  PARSE_LANGOPT_IMPORTANT(Optimize, diag::warn_pch_optimize);
  PARSE_LANGOPT_IMPORTANT(OptimizeSize, diag::warn_pch_optimize_size);
  PARSE_LANGOPT_IMPORTANT(Static, diag::warn_pch_static);
  PARSE_LANGOPT_IMPORTANT(PICLevel, diag::warn_pch_pic_level);
  PARSE_LANGOPT_IMPORTANT(GNUInline, diag::warn_pch_gnu_inline);
  PARSE_LANGOPT_IMPORTANT(NoInline, diag::warn_pch_no_inline);
  PARSE_LANGOPT_IMPORTANT(AccessControl, diag::warn_pch_access_control);
  PARSE_LANGOPT_IMPORTANT(CharIsSigned, diag::warn_pch_char_signed);
  if ((PPLangOpts.getGCMode() != 0) != (LangOpts.getGCMode() != 0)) {
    Reader.Diag(diag::warn_pch_gc_mode) 
      << LangOpts.getGCMode() << PPLangOpts.getGCMode();
    return true;
  }
  PARSE_LANGOPT_BENIGN(getVisibilityMode());
  PARSE_LANGOPT_BENIGN(InstantiationDepth);
#undef PARSE_LANGOPT_IRRELEVANT
#undef PARSE_LANGOPT_BENIGN

  return false;
}

bool PCHValidator::ReadTargetTriple(const std::string &Triple) {
  if (Triple != PP.getTargetInfo().getTargetTriple()) {
    Reader.Diag(diag::warn_pch_target_triple)
      << Triple << PP.getTargetInfo().getTargetTriple();
    return true;
  }
  return false;
}

/// \brief Split the given string into a vector of lines, eliminating
/// any empty lines in the process.
///
/// \param Str the string to split.
/// \param Len the length of Str.
/// \param KeepEmptyLines true if empty lines should be included
/// \returns a vector of lines, with the line endings removed
static std::vector<std::string> splitLines(const char *Str, unsigned Len,
                                           bool KeepEmptyLines = false) {
  std::vector<std::string> Lines;
  for (unsigned LineStart = 0; LineStart < Len; ++LineStart) {
    unsigned LineEnd = LineStart;
    while (LineEnd < Len && Str[LineEnd] != '\n')
      ++LineEnd;
    if (LineStart != LineEnd || KeepEmptyLines)
      Lines.push_back(std::string(&Str[LineStart], &Str[LineEnd]));
    LineStart = LineEnd;
  }
  return Lines;
}

/// \brief Determine whether the string Haystack starts with the
/// substring Needle.
static bool startsWith(const std::string &Haystack, const char *Needle) {
  for (unsigned I = 0, N = Haystack.size(); Needle[I] != 0; ++I) {
    if (I == N)
      return false;
    if (Haystack[I] != Needle[I])
      return false;
  }

  return true;
}

/// \brief Determine whether the string Haystack starts with the
/// substring Needle.
static inline bool startsWith(const std::string &Haystack,
                              const std::string &Needle) {
  return startsWith(Haystack, Needle.c_str());
}

bool PCHValidator::ReadPredefinesBuffer(const char *PCHPredef, 
                                        unsigned PCHPredefLen,
                                        FileID PCHBufferID,
                                        std::string &SuggestedPredefines) {
  const char *Predef = PP.getPredefines().c_str();
  unsigned PredefLen = PP.getPredefines().size();

  // If the two predefines buffers compare equal, we're done!
  if (PredefLen == PCHPredefLen && 
      strncmp(Predef, PCHPredef, PCHPredefLen) == 0)
    return false;

  SourceManager &SourceMgr = PP.getSourceManager();
  
  // The predefines buffers are different. Determine what the
  // differences are, and whether they require us to reject the PCH
  // file.
  std::vector<std::string> CmdLineLines = splitLines(Predef, PredefLen);
  std::vector<std::string> PCHLines = splitLines(PCHPredef, PCHPredefLen);

  // Sort both sets of predefined buffer lines, since 
  std::sort(CmdLineLines.begin(), CmdLineLines.end());
  std::sort(PCHLines.begin(), PCHLines.end());

  // Determine which predefines that where used to build the PCH file
  // are missing from the command line.
  std::vector<std::string> MissingPredefines;
  std::set_difference(PCHLines.begin(), PCHLines.end(),
                      CmdLineLines.begin(), CmdLineLines.end(),
                      std::back_inserter(MissingPredefines));

  bool MissingDefines = false;
  bool ConflictingDefines = false;
  for (unsigned I = 0, N = MissingPredefines.size(); I != N; ++I) {
    const std::string &Missing = MissingPredefines[I];
    if (!startsWith(Missing, "#define ") != 0) {
      Reader.Diag(diag::warn_pch_compiler_options_mismatch);
      return true;
    }
    
    // This is a macro definition. Determine the name of the macro
    // we're defining.
    std::string::size_type StartOfMacroName = strlen("#define ");
    std::string::size_type EndOfMacroName 
      = Missing.find_first_of("( \n\r", StartOfMacroName);
    assert(EndOfMacroName != std::string::npos &&
           "Couldn't find the end of the macro name");
    std::string MacroName = Missing.substr(StartOfMacroName,
                                           EndOfMacroName - StartOfMacroName);

    // Determine whether this macro was given a different definition
    // on the command line.
    std::string MacroDefStart = "#define " + MacroName;
    std::string::size_type MacroDefLen = MacroDefStart.size();
    std::vector<std::string>::iterator ConflictPos
      = std::lower_bound(CmdLineLines.begin(), CmdLineLines.end(),
                         MacroDefStart);
    for (; ConflictPos != CmdLineLines.end(); ++ConflictPos) {
      if (!startsWith(*ConflictPos, MacroDefStart)) {
        // Different macro; we're done.
        ConflictPos = CmdLineLines.end();
        break; 
      }
      
      assert(ConflictPos->size() > MacroDefLen && 
             "Invalid #define in predefines buffer?");
      if ((*ConflictPos)[MacroDefLen] != ' ' && 
          (*ConflictPos)[MacroDefLen] != '(')
        continue; // Longer macro name; keep trying.
      
      // We found a conflicting macro definition.
      break;
    }
    
    if (ConflictPos != CmdLineLines.end()) {
      Reader.Diag(diag::warn_cmdline_conflicting_macro_def)
          << MacroName;

      // Show the definition of this macro within the PCH file.
      const char *MissingDef = strstr(PCHPredef, Missing.c_str());
      unsigned Offset = MissingDef - PCHPredef;
      SourceLocation PCHMissingLoc
        = SourceMgr.getLocForStartOfFile(PCHBufferID)
            .getFileLocWithOffset(Offset);
      Reader.Diag(PCHMissingLoc, diag::note_pch_macro_defined_as)
        << MacroName;

      ConflictingDefines = true;
      continue;
    }
    
    // If the macro doesn't conflict, then we'll just pick up the
    // macro definition from the PCH file. Warn the user that they
    // made a mistake.
    if (ConflictingDefines)
      continue; // Don't complain if there are already conflicting defs
    
    if (!MissingDefines) {
      Reader.Diag(diag::warn_cmdline_missing_macro_defs);
      MissingDefines = true;
    }

    // Show the definition of this macro within the PCH file.
    const char *MissingDef = strstr(PCHPredef, Missing.c_str());
    unsigned Offset = MissingDef - PCHPredef;
    SourceLocation PCHMissingLoc
      = SourceMgr.getLocForStartOfFile(PCHBufferID)
      .getFileLocWithOffset(Offset);
    Reader.Diag(PCHMissingLoc, diag::note_using_macro_def_from_pch);
  }
  
  if (ConflictingDefines)
    return true;
  
  // Determine what predefines were introduced based on command-line
  // parameters that were not present when building the PCH
  // file. Extra #defines are okay, so long as the identifiers being
  // defined were not used within the precompiled header.
  std::vector<std::string> ExtraPredefines;
  std::set_difference(CmdLineLines.begin(), CmdLineLines.end(),
                      PCHLines.begin(), PCHLines.end(),
                      std::back_inserter(ExtraPredefines));  
  for (unsigned I = 0, N = ExtraPredefines.size(); I != N; ++I) {
    const std::string &Extra = ExtraPredefines[I];
    if (!startsWith(Extra, "#define ") != 0) {
      Reader.Diag(diag::warn_pch_compiler_options_mismatch);
      return true;
    }

    // This is an extra macro definition. Determine the name of the
    // macro we're defining.
    std::string::size_type StartOfMacroName = strlen("#define ");
    std::string::size_type EndOfMacroName 
      = Extra.find_first_of("( \n\r", StartOfMacroName);
    assert(EndOfMacroName != std::string::npos &&
           "Couldn't find the end of the macro name");
    std::string MacroName = Extra.substr(StartOfMacroName,
                                         EndOfMacroName - StartOfMacroName);

    // Check whether this name was used somewhere in the PCH file. If
    // so, defining it as a macro could change behavior, so we reject
    // the PCH file.
    if (IdentifierInfo *II = Reader.get(MacroName.c_str(),
                                 MacroName.c_str() + MacroName.size())) {
      Reader.Diag(diag::warn_macro_name_used_in_pch)
        << II;
      return true;
    }

    // Add this definition to the suggested predefines buffer.
    SuggestedPredefines += Extra;
    SuggestedPredefines += '\n';
  }

  // If we get here, it's because the predefines buffer had compatible
  // contents. Accept the PCH file.
  return false;
}

void PCHValidator::ReadHeaderFileInfo(const HeaderFileInfo &HFI) {
  PP.getHeaderSearchInfo().setHeaderFileInfoForUID(HFI, NumHeaderInfos++);
}

void PCHValidator::ReadCounter(unsigned Value) {
  PP.setCounterValue(Value);
}



//===----------------------------------------------------------------------===//
// PCH reader implementation
//===----------------------------------------------------------------------===//

PCHReader::PCHReader(Preprocessor &PP, ASTContext *Context) 
  : Listener(new PCHValidator(PP, *this)), SourceMgr(PP.getSourceManager()),
    FileMgr(PP.getFileManager()), Diags(PP.getDiagnostics()),
    SemaObj(0), PP(&PP), Context(Context), Consumer(0),
    IdentifierTableData(0), IdentifierLookupTable(0),
    IdentifierOffsets(0),
    MethodPoolLookupTable(0), MethodPoolLookupTableData(0),
    TotalSelectorsInMethodPool(0), SelectorOffsets(0),
    TotalNumSelectors(0), NumStatHits(0), NumStatMisses(0), 
    NumSLocEntriesRead(0), NumStatementsRead(0), 
    NumMacrosRead(0), NumMethodPoolSelectorsRead(0), NumMethodPoolMisses(0),
    NumLexicalDeclContextsRead(0), NumVisibleDeclContextsRead(0) { }

PCHReader::PCHReader(SourceManager &SourceMgr, FileManager &FileMgr,
                     Diagnostic &Diags) 
  : SourceMgr(SourceMgr), FileMgr(FileMgr), Diags(Diags),
    SemaObj(0), PP(0), Context(0), Consumer(0),
    IdentifierTableData(0), IdentifierLookupTable(0),
    IdentifierOffsets(0),
    MethodPoolLookupTable(0), MethodPoolLookupTableData(0),
    TotalSelectorsInMethodPool(0), SelectorOffsets(0),
    TotalNumSelectors(0), NumStatHits(0), NumStatMisses(0), 
    NumSLocEntriesRead(0), NumStatementsRead(0), 
    NumMacrosRead(0), NumMethodPoolSelectorsRead(0), NumMethodPoolMisses(0),
    NumLexicalDeclContextsRead(0), NumVisibleDeclContextsRead(0) { }

PCHReader::~PCHReader() {}

Expr *PCHReader::ReadDeclExpr() {
  return dyn_cast_or_null<Expr>(ReadStmt(DeclsCursor));
}

{'fraction_non_alphanumeric': 0.14068591819203066, 'fraction_numerical': 0.00386572391462367, 'mean_word_length': 3.193847859221283, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6508817', 'n_tokens_mistral': 9697, 'n_tokens_neox': 9417, 'n_words': 2010}