<commit_before>//===--- PathDiagnostic.cpp - Path-Specific Diagnostic Handling -*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//  This file defines the PathDiagnostic-related interfaces.
//
//===----------------------------------------------------------------------===//

#include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
#include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
#include "clang/Basic/SourceManager.h"
#include "clang/AST/Expr.h"
#include "clang/AST/Decl.h"
#include "clang/AST/DeclCXX.h"
#include "clang/AST/DeclObjC.h"
#include "clang/AST/ParentMap.h"
#include "clang/AST/StmtCXX.h"
#include "llvm/ADT/SmallString.h"

using namespace clang;
using namespace ento;

bool PathDiagnosticMacroPiece::containsEvent() const {
  for (PathPieces::const_iterator I = subPieces.begin(), E = subPieces.end();
       I!=E; ++I) {
    if (isa<PathDiagnosticEventPiece>(*I))
      return true;
    if (PathDiagnosticMacroPiece *MP = dyn_cast<PathDiagnosticMacroPiece>(*I))
      if (MP->containsEvent())
        return true;
  }
  return false;
}

static StringRef StripTrailingDots(StringRef s) {
  for (StringRef::size_type i = s.size(); i != 0; --i)
    if (s[i - 1] != '.')
      return s.substr(0, i);
  return "";
}

PathDiagnosticPiece::PathDiagnosticPiece(StringRef s,
                                         Kind k, DisplayHint hint)
  : str(StripTrailingDots(s)), kind(k), Hint(hint) {}

PathDiagnosticPiece::PathDiagnosticPiece(Kind k, DisplayHint hint)
  : kind(k), Hint(hint) {}

PathDiagnosticPiece::~PathDiagnosticPiece() {}
PathDiagnosticEventPiece::~PathDiagnosticEventPiece() {}
PathDiagnosticCallPiece::~PathDiagnosticCallPiece() {}
PathDiagnosticControlFlowPiece::~PathDiagnosticControlFlowPiece() {}
PathDiagnosticMacroPiece::~PathDiagnosticMacroPiece() {}


PathPieces::~PathPieces() {}

void PathPieces::flattenTo(PathPieces &Primary, PathPieces &Current,
                           bool ShouldFlattenMacros) const {
  for (PathPieces::const_iterator I = begin(), E = end(); I != E; ++I) {
    PathDiagnosticPiece *Piece = I->getPtr();

    switch (Piece->getKind()) {
    case PathDiagnosticPiece::Call: {
      PathDiagnosticCallPiece *Call = cast<PathDiagnosticCallPiece>(Piece);
      IntrusiveRefCntPtr<PathDiagnosticEventPiece> CallEnter =
        Call->getCallEnterEvent();
      if (CallEnter)
        Current.push_back(CallEnter);
      Call->path.flattenTo(Primary, Primary, ShouldFlattenMacros);
      IntrusiveRefCntPtr<PathDiagnosticEventPiece> callExit =
        Call->getCallExitEvent();
      if (callExit)
        Current.push_back(callExit);
      break;
    }
    case PathDiagnosticPiece::Macro: {
      PathDiagnosticMacroPiece *Macro = cast<PathDiagnosticMacroPiece>(Piece);
      if (ShouldFlattenMacros) {
        Macro->subPieces.flattenTo(Primary, Primary, ShouldFlattenMacros);
      } else {
        Current.push_back(Piece);
        PathPieces NewPath;
        Macro->subPieces.flattenTo(Primary, NewPath, ShouldFlattenMacros);
        // FIXME: This probably shouldn't mutate the original path piece.
        Macro->subPieces = NewPath;
      }
      break;
    }
    case PathDiagnosticPiece::Event:
    case PathDiagnosticPiece::ControlFlow:
      Current.push_back(Piece);
      break;
    }
  }
}


PathDiagnostic::~PathDiagnostic() {}

PathDiagnostic::PathDiagnostic(const Decl *declWithIssue,
                               StringRef bugtype, StringRef verboseDesc,
                               StringRef shortDesc, StringRef category)
  : DeclWithIssue(declWithIssue),
    BugType(StripTrailingDots(bugtype)),
    VerboseDesc(StripTrailingDots(verboseDesc)),
    ShortDesc(StripTrailingDots(shortDesc)),
    Category(StripTrailingDots(category)),
    path(pathImpl) {}

void PathDiagnosticConsumer::anchor() { }

PathDiagnosticConsumer::~PathDiagnosticConsumer() {
  // Delete the contents of the FoldingSet if it isn't empty already.
  for (llvm::FoldingSet<PathDiagnostic>::iterator it =
       Diags.begin(), et = Diags.end() ; it != et ; ++it) {
    delete &*it;
  }
}

void PathDiagnosticConsumer::HandlePathDiagnostic(PathDiagnostic *D) {
  llvm::OwningPtr<PathDiagnostic> OwningD(D);
  
  if (!D || D->path.empty())
    return;
  
  // We need to flatten the locations (convert Stmt* to locations) because
  // the referenced statements may be freed by the time the diagnostics
  // are emitted.
  D->flattenLocations();

  // If the PathDiagnosticConsumer does not support diagnostics that
  // cross file boundaries, prune out such diagnostics now.
  if (!supportsCrossFileDiagnostics()) {
    // Verify that the entire path is from the same FileID.
    FileID FID;
    const SourceManager &SMgr = (*D->path.begin())->getLocation().getManager();
    llvm::SmallVector<const PathPieces *, 5> WorkList;
    WorkList.push_back(&D->path);

    while (!WorkList.empty()) {
      const PathPieces &path = *WorkList.back();
      WorkList.pop_back();

      for (PathPieces::const_iterator I = path.begin(), E = path.end();
           I != E; ++I) {
        const PathDiagnosticPiece *piece = I->getPtr();
        FullSourceLoc L = piece->getLocation().asLocation().getExpansionLoc();
      
        if (FID.isInvalid()) {
          FID = SMgr.getFileID(L);
        } else if (SMgr.getFileID(L) != FID)
          return; // FIXME: Emit a warning?
      
        // Check the source ranges.
        ArrayRef<SourceRange> Ranges = piece->getRanges();
        for (ArrayRef<SourceRange>::iterator I = Ranges.begin(),
                                             E = Ranges.end(); I != E; ++I) {
          SourceLocation L = SMgr.getExpansionLoc(I->getBegin());
          if (!L.isFileID() || SMgr.getFileID(L) != FID)
            return; // FIXME: Emit a warning?
          L = SMgr.getExpansionLoc(I->getEnd());
          if (!L.isFileID() || SMgr.getFileID(L) != FID)
            return; // FIXME: Emit a warning?
        }
        
        if (const PathDiagnosticCallPiece *call =
            dyn_cast<PathDiagnosticCallPiece>(piece)) {
          WorkList.push_back(&call->path);
        }
        else if (const PathDiagnosticMacroPiece *macro =
                 dyn_cast<PathDiagnosticMacroPiece>(piece)) {
          WorkList.push_back(&macro->subPieces);
        }
      }
    }
    
    if (FID.isInvalid())
      return; // FIXME: Emit a warning?
  }  

  // Profile the node to see if we already have something matching it
  llvm::FoldingSetNodeID profile;
  D->Profile(profile);
  void *InsertPos = 0;

  if (PathDiagnostic *orig = Diags.FindNodeOrInsertPos(profile, InsertPos)) {
    // Keep the PathDiagnostic with the shorter path.
    // Note, the enclosing routine is called in deterministic order, so the
    // results will be consistent between runs (no reason to break ties if the
    // size is the same).
    const unsigned orig_size = orig->full_size();
    const unsigned new_size = D->full_size();
    if (orig_size <= new_size)
      return;

    Diags.RemoveNode(orig);
    delete orig;
  }
  
  Diags.InsertNode(OwningD.take());
}


namespace {
  
static llvm::Optional<bool> comparePiece(PathDiagnosticPiece &X,
                                         PathDiagnosticPiece &Y) {
  PathDiagnosticPiece::Kind XK = X.getKind(), YK = Y.getKind();
  if (XK != YK) {
    return XK < YK;
  }
  FullSourceLoc XL = X.getLocation().asLocation();
  FullSourceLoc YL = Y.getLocation().asLocation();
  if (XL < YL)
    return true;
  if (YL < XL)
    return false;
  const std::string &XS = X.getString();
  const std::string &YS = Y.getString();
  if (XS != YS)
    return XS < YS;
  return llvm::Optional<bool>();
}
  
static bool comparePathPieces(const PathPieces &X, const PathPieces &Y) {
  if (X.size() < Y.size())
    return true;
  if (X.size() > Y.size())
    return false;
  // Compare individual parts of the path.
  assert(X.size() == Y.size());
  for (unsigned i = 0, n = X.size(); i < n; ++i) {
    llvm::Optional<bool> B = comparePiece(*X[i], *Y[i]);
    if (B.hasValue())
      return B.getValue();
  }
  return false;
}
  
struct CompareDiagnostics {
  // Compare if 'X' is "<" than 'Y'.
  bool operator()(const PathDiagnostic *X, const PathDiagnostic *Y) const {
    // First compare by location
    const FullSourceLoc &XLoc = X->getLocation().asLocation();
    const FullSourceLoc &YLoc = Y->getLocation().asLocation();
    if (XLoc < YLoc)
      return true;
    if (YLoc < XLoc)
      return false;
    
    // Next, compare by bug type.
    StringRef XBugType = X->getBugType();
    StringRef YBugType = Y->getBugType();
    if (XBugType < YBugType)
      return true;
    if (XBugType != YBugType)
      return false;
    
    // Next, compare by bug description.
    StringRef XDesc = X->getVerboseDescription();
    StringRef YDesc = Y->getVerboseDescription();
    if (XDesc < YDesc)
      return true;
    if (XDesc != YDesc)
      return false;
    
    // Fall back to comparing path pieces.
    return comparePathPieces(X->path, Y->path);
  }  
};  
}

void PathDiagnosticConsumer::FlushDiagnostics(
                                     PathDiagnosticConsumer::FilesMade *Files) {
  if (flushed)
    return;
  
  flushed = true;
  
  std::vector<const PathDiagnostic *> BatchDiags;
  for (llvm::FoldingSet<PathDiagnostic>::iterator it = Diags.begin(),
       et = Diags.end(); it != et; ++it) {
    BatchDiags.push_back(&*it);
  }
  <commit_msg>Further tweaks to hopefully make the PathDiagnostic emission more deterministic.<commit_after>//===--- PathDiagnostic.cpp - Path-Specific Diagnostic Handling -*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//  This file defines the PathDiagnostic-related interfaces.
//
//===----------------------------------------------------------------------===//

#include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
#include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
#include "clang/Basic/SourceManager.h"
#include "clang/AST/Expr.h"
#include "clang/AST/Decl.h"
#include "clang/AST/DeclCXX.h"
#include "clang/AST/DeclObjC.h"
#include "clang/AST/ParentMap.h"
#include "clang/AST/StmtCXX.h"
#include "llvm/ADT/SmallString.h"

using namespace clang;
using namespace ento;

bool PathDiagnosticMacroPiece::containsEvent() const {
  for (PathPieces::const_iterator I = subPieces.begin(), E = subPieces.end();
       I!=E; ++I) {
    if (isa<PathDiagnosticEventPiece>(*I))
      return true;
    if (PathDiagnosticMacroPiece *MP = dyn_cast<PathDiagnosticMacroPiece>(*I))
      if (MP->containsEvent())
        return true;
  }
  return false;
}

static StringRef StripTrailingDots(StringRef s) {
  for (StringRef::size_type i = s.size(); i != 0; --i)
    if (s[i - 1] != '.')
      return s.substr(0, i);
  return "";
}

PathDiagnosticPiece::PathDiagnosticPiece(StringRef s,
                                         Kind k, DisplayHint hint)
  : str(StripTrailingDots(s)), kind(k), Hint(hint) {}

PathDiagnosticPiece::PathDiagnosticPiece(Kind k, DisplayHint hint)
  : kind(k), Hint(hint) {}

PathDiagnosticPiece::~PathDiagnosticPiece() {}
PathDiagnosticEventPiece::~PathDiagnosticEventPiece() {}
PathDiagnosticCallPiece::~PathDiagnosticCallPiece() {}
PathDiagnosticControlFlowPiece::~PathDiagnosticControlFlowPiece() {}
PathDiagnosticMacroPiece::~PathDiagnosticMacroPiece() {}


PathPieces::~PathPieces() {}

void PathPieces::flattenTo(PathPieces &Primary, PathPieces &Current,
                           bool ShouldFlattenMacros) const {
  for (PathPieces::const_iterator I = begin(), E = end(); I != E; ++I) {
    PathDiagnosticPiece *Piece = I->getPtr();

    switch (Piece->getKind()) {
    case PathDiagnosticPiece::Call: {
      PathDiagnosticCallPiece *Call = cast<PathDiagnosticCallPiece>(Piece);
      IntrusiveRefCntPtr<PathDiagnosticEventPiece> CallEnter =
        Call->getCallEnterEvent();
      if (CallEnter)
        Current.push_back(CallEnter);
      Call->path.flattenTo(Primary, Primary, ShouldFlattenMacros);
      IntrusiveRefCntPtr<PathDiagnosticEventPiece> callExit =
        Call->getCallExitEvent();
      if (callExit)
        Current.push_back(callExit);
      break;
    }
    case PathDiagnosticPiece::Macro: {
      PathDiagnosticMacroPiece *Macro = cast<PathDiagnosticMacroPiece>(Piece);
      if (ShouldFlattenMacros) {
        Macro->subPieces.flattenTo(Primary, Primary, ShouldFlattenMacros);
      } else {
        Current.push_back(Piece);
        PathPieces NewPath;
        Macro->subPieces.flattenTo(Primary, NewPath, ShouldFlattenMacros);
        // FIXME: This probably shouldn't mutate the original path piece.
        Macro->subPieces = NewPath;
      }
      break;
    }
    case PathDiagnosticPiece::Event:
    case PathDiagnosticPiece::ControlFlow:
      Current.push_back(Piece);
      break;
    }
  }
}


PathDiagnostic::~PathDiagnostic() {}

PathDiagnostic::PathDiagnostic(const Decl *declWithIssue,
                               StringRef bugtype, StringRef verboseDesc,
                               StringRef shortDesc, StringRef category)
  : DeclWithIssue(declWithIssue),
    BugType(StripTrailingDots(bugtype)),
    VerboseDesc(StripTrailingDots(verboseDesc)),
    ShortDesc(StripTrailingDots(shortDesc)),
    Category(StripTrailingDots(category)),
    path(pathImpl) {}

void PathDiagnosticConsumer::anchor() { }

PathDiagnosticConsumer::~PathDiagnosticConsumer() {
  // Delete the contents of the FoldingSet if it isn't empty already.
  for (llvm::FoldingSet<PathDiagnostic>::iterator it =
       Diags.begin(), et = Diags.end() ; it != et ; ++it) {
    delete &*it;
  }
}

void PathDiagnosticConsumer::HandlePathDiagnostic(PathDiagnostic *D) {
  llvm::OwningPtr<PathDiagnostic> OwningD(D);
  
  if (!D || D->path.empty())
    return;
  
  // We need to flatten the locations (convert Stmt* to locations) because
  // the referenced statements may be freed by the time the diagnostics
  // are emitted.
  D->flattenLocations();

  // If the PathDiagnosticConsumer does not support diagnostics that
  // cross file boundaries, prune out such diagnostics now.
  if (!supportsCrossFileDiagnostics()) {
    // Verify that the entire path is from the same FileID.
    FileID FID;
    const SourceManager &SMgr = (*D->path.begin())->getLocation().getManager();
    llvm::SmallVector<const PathPieces *, 5> WorkList;
    WorkList.push_back(&D->path);

    while (!WorkList.empty()) {
      const PathPieces &path = *WorkList.back();
      WorkList.pop_back();

      for (PathPieces::const_iterator I = path.begin(), E = path.end();
           I != E; ++I) {
        const PathDiagnosticPiece *piece = I->getPtr();
        FullSourceLoc L = piece->getLocation().asLocation().getExpansionLoc();
      
        if (FID.isInvalid()) {
          FID = SMgr.getFileID(L);
        } else if (SMgr.getFileID(L) != FID)
          return; // FIXME: Emit a warning?
      
        // Check the source ranges.
        ArrayRef<SourceRange> Ranges = piece->getRanges();
        for (ArrayRef<SourceRange>::iterator I = Ranges.begin(),
                                             E = Ranges.end(); I != E; ++I) {
          SourceLocation L = SMgr.getExpansionLoc(I->getBegin());
          if (!L.isFileID() || SMgr.getFileID(L) != FID)
            return; // FIXME: Emit a warning?
          L = SMgr.getExpansionLoc(I->getEnd());
          if (!L.isFileID() || SMgr.getFileID(L) != FID)
            return; // FIXME: Emit a warning?
        }
        
        if (const PathDiagnosticCallPiece *call =
            dyn_cast<PathDiagnosticCallPiece>(piece)) {
          WorkList.push_back(&call->path);
        }
        else if (const PathDiagnosticMacroPiece *macro =
                 dyn_cast<PathDiagnosticMacroPiece>(piece)) {
          WorkList.push_back(&macro->subPieces);
        }
      }
    }
    
    if (FID.isInvalid())
      return; // FIXME: Emit a warning?
  }  

  // Profile the node to see if we already have something matching it
  llvm::FoldingSetNodeID profile;
  D->Profile(profile);
  void *InsertPos = 0;

  if (PathDiagnostic *orig = Diags.FindNodeOrInsertPos(profile, InsertPos)) {
    // Keep the PathDiagnostic with the shorter path.
    // Note, the enclosing routine is called in deterministic order, so the
    // results will be consistent between runs (no reason to break ties if the
    // size is the same).
    const unsigned orig_size = orig->full_size();
    const unsigned new_size = D->full_size();
    if (orig_size <= new_size)
      return;

    Diags.RemoveNode(orig);
    delete orig;
  }
  
  Diags.InsertNode(OwningD.take());
}


namespace {
  
static llvm::Optional<bool> comparePiece(PathDiagnosticPiece &X,
                                         PathDiagnosticPiece &Y) {
  PathDiagnosticPiece::Kind XK = X.getKind(), YK = Y.getKind();
  if (XK != YK) {
    return XK < YK;
  }
  FullSourceLoc XL = X.getLocation().asLocation();
  FullSourceLoc YL = Y.getLocation().asLocation();
  if (XL != YL)
    return XL < YL;
  const std::string &XS = X.getString();
  const std::string &YS = Y.getString();
  if (XS != YS)
    return XS < YS;
  return llvm::Optional<bool>();
}
  
static bool comparePathPieces(const PathPieces &X, const PathPieces &Y) {
  if (X.size() != Y.size())
    return X.size() < Y.size();
  // Compare individual parts of the path.
  assert(X.size() == Y.size());
  for (unsigned i = 0, n = X.size(); i < n; ++i) {
    llvm::Optional<bool> B = comparePiece(*X[i], *Y[i]);
    if (B.hasValue())
      return B.getValue();
  }
  return false;
}
  
struct CompareDiagnostics {
  // Compare if 'X' is "<" than 'Y'.
  bool operator()(const PathDiagnostic *X, const PathDiagnostic *Y) const {
    // First compare by location
    const FullSourceLoc &XLoc = X->getLocation().asLocation();
    const FullSourceLoc &YLoc = Y->getLocation().asLocation();
    if (XLoc != YLoc)
      return XLoc < YLoc;
    
    // Next, compare by bug type.
    StringRef XBugType = X->getBugType();
    StringRef YBugType = Y->getBugType();
    if (XBugType != YBugType)
      return XBugType < YBugType;
    
    // Next, compare by bug description.
    StringRef XDesc = X->getVerboseDescription();
    StringRef YDesc = Y->getVerboseDescription();
    if (XDesc != YDesc)
      return XDesc < YDesc;
    
    // Fall back to comparing path pieces.
    return comparePathPieces(X->path, Y->path);
  }  
};  
}

void PathDiagnosticConsumer::FlushDiagnostics(
                                     PathDiagnosticConsumer::FilesMade *Files) {
  if (flushed)
    return;
  
  flushed = true;
  
  std::vector<const PathDiagnostic *> BatchDiags;
  for (llvm::FoldingSet<PathDiagnostic>::iterator it = Diags.begin(),
       et = Diags.end(); it != et; ++it) {
    BatchDiags.push_back(&*it);
  }
  
{'fraction_non_alphanumeric': 0.1436109079511023, 'fraction_numerical': 0.0006268937415108139, 'mean_word_length': 2.8080366023473244, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 104, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2355306', 'n_tokens_mistral': 6085, 'n_tokens_neox': 6002, 'n_words': 1371}