<commit_before>//===--- Parser.cpp - C Language Family Parser ----------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//  This file implements the Parser interfaces.
//
//===----------------------------------------------------------------------===//

#include "clang/Parse/Parser.h"
#include "clang/Parse/ParseDiagnostic.h"
#include "clang/Parse/DeclSpec.h"
#include "clang/Parse/Scope.h"
#include "llvm/Support/raw_ostream.h"
#include "ExtensionRAIIObject.h"
#include "ParsePragma.h"
using namespace clang;

/// \brief A comment handler that passes comments found by the preprocessor
/// to the parser action.
class ActionCommentHandler : public CommentHandler {
  Action &Actions;

public:
  explicit ActionCommentHandler(Action &Actions) : Actions(Actions) { }

  virtual void HandleComment(Preprocessor &PP, SourceRange Comment) {
    Actions.ActOnComment(Comment);
  }
};

Parser::Parser(Preprocessor &pp, Action &actions)
  : CrashInfo(*this), PP(pp), Actions(actions), Diags(PP.getDiagnostics()),
    GreaterThanIsOperator(true), TemplateParameterDepth(0) {
  Tok.setKind(tok::eof);
  CurScope = 0;
  NumCachedScopes = 0;
  ParenCount = BracketCount = BraceCount = 0;
  ObjCImpDecl = DeclPtrTy();

  // Add #pragma handlers. These are removed and destroyed in the
  // destructor.
  PackHandler.reset(new
          PragmaPackHandler(&PP.getIdentifierTable().get("pack"), actions));
  PP.AddPragmaHandler(0, PackHandler.get());

  UnusedHandler.reset(new
          PragmaUnusedHandler(&PP.getIdentifierTable().get("unused"), actions,
                              *this));
  PP.AddPragmaHandler(0, UnusedHandler.get());

  WeakHandler.reset(new
          PragmaWeakHandler(&PP.getIdentifierTable().get("weak"), actions));
  PP.AddPragmaHandler(0, WeakHandler.get());

  CommentHandler.reset(new ActionCommentHandler(actions));
  PP.AddCommentHandler(CommentHandler.get());
}

/// If a crash happens while the parser is active, print out a line indicating
/// what the current token is.
void PrettyStackTraceParserEntry::print(llvm::raw_ostream &OS) const {
  const Token &Tok = P.getCurToken();
  if (Tok.is(tok::eof)) {
    OS << "<eof> parser at end of file\n";
    return;
  }

  if (Tok.getLocation().isInvalid()) {
    OS << "<unknown> parser at unknown location\n";
    return;
  }

  const Preprocessor &PP = P.getPreprocessor();
  Tok.getLocation().print(OS, PP.getSourceManager());
  OS << ": current parser token '" << PP.getSpelling(Tok) << "'\n";
}


DiagnosticBuilder Parser::Diag(SourceLocation Loc, unsigned DiagID) {
  return Diags.Report(FullSourceLoc(Loc, PP.getSourceManager()), DiagID);
}

DiagnosticBuilder Parser::Diag(const Token &Tok, unsigned DiagID) {
  return Diag(Tok.getLocation(), DiagID);
}

/// \brief Emits a diagnostic suggesting parentheses surrounding a
/// given range.
///
/// \param Loc The location where we'll emit the diagnostic.
/// \param Loc The kind of diagnostic to emit.
/// \param ParenRange Source range enclosing code that should be parenthesized.
void Parser::SuggestParentheses(SourceLocation Loc, unsigned DK,
                                SourceRange ParenRange) {
  SourceLocation EndLoc = PP.getLocForEndOfToken(ParenRange.getEnd());
  if (!ParenRange.getEnd().isFileID() || EndLoc.isInvalid()) {
    // We can't display the parentheses, so just dig the
    // warning/error and return.
    Diag(Loc, DK);
    return;
  }

  Diag(Loc, DK)
    << CodeModificationHint::CreateInsertion(ParenRange.getBegin(), "(")
    << CodeModificationHint::CreateInsertion(EndLoc, ")");<commit_msg>Don't crash when dumping pretty stack traces, if the current tok is an annotation token.  - I'm not sure what the best thing to print is, for now we just print the token    location and 'at annotation token'.<commit_after>//===--- Parser.cpp - C Language Family Parser ----------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//  This file implements the Parser interfaces.
//
//===----------------------------------------------------------------------===//

#include "clang/Parse/Parser.h"
#include "clang/Parse/ParseDiagnostic.h"
#include "clang/Parse/DeclSpec.h"
#include "clang/Parse/Scope.h"
#include "llvm/Support/raw_ostream.h"
#include "ExtensionRAIIObject.h"
#include "ParsePragma.h"
using namespace clang;

/// \brief A comment handler that passes comments found by the preprocessor
/// to the parser action.
class ActionCommentHandler : public CommentHandler {
  Action &Actions;

public:
  explicit ActionCommentHandler(Action &Actions) : Actions(Actions) { }

  virtual void HandleComment(Preprocessor &PP, SourceRange Comment) {
    Actions.ActOnComment(Comment);
  }
};

Parser::Parser(Preprocessor &pp, Action &actions)
  : CrashInfo(*this), PP(pp), Actions(actions), Diags(PP.getDiagnostics()),
    GreaterThanIsOperator(true), TemplateParameterDepth(0) {
  Tok.setKind(tok::eof);
  CurScope = 0;
  NumCachedScopes = 0;
  ParenCount = BracketCount = BraceCount = 0;
  ObjCImpDecl = DeclPtrTy();

  // Add #pragma handlers. These are removed and destroyed in the
  // destructor.
  PackHandler.reset(new
          PragmaPackHandler(&PP.getIdentifierTable().get("pack"), actions));
  PP.AddPragmaHandler(0, PackHandler.get());

  UnusedHandler.reset(new
          PragmaUnusedHandler(&PP.getIdentifierTable().get("unused"), actions,
                              *this));
  PP.AddPragmaHandler(0, UnusedHandler.get());

  WeakHandler.reset(new
          PragmaWeakHandler(&PP.getIdentifierTable().get("weak"), actions));
  PP.AddPragmaHandler(0, WeakHandler.get());

  CommentHandler.reset(new ActionCommentHandler(actions));
  PP.AddCommentHandler(CommentHandler.get());
}

/// If a crash happens while the parser is active, print out a line indicating
/// what the current token is.
void PrettyStackTraceParserEntry::print(llvm::raw_ostream &OS) const {
  const Token &Tok = P.getCurToken();
  if (Tok.is(tok::eof)) {
    OS << "<eof> parser at end of file\n";
    return;
  }

  if (Tok.getLocation().isInvalid()) {
    OS << "<unknown> parser at unknown location\n";
    return;
  }

  const Preprocessor &PP = P.getPreprocessor();
  Tok.getLocation().print(OS, PP.getSourceManager());
  if (Tok.isAnnotation())
    OS << ": at annotation token \n";
  else
    OS << ": current parser token '" << PP.getSpelling(Tok) << "'\n";
}


DiagnosticBuilder Parser::Diag(SourceLocation Loc, unsigned DiagID) {
  return Diags.Report(FullSourceLoc(Loc, PP.getSourceManager()), DiagID);
}

DiagnosticBuilder Parser::Diag(const Token &Tok, unsigned DiagID) {
  return Diag(Tok.getLocation(), DiagID);
}

/// \brief Emits a diagnostic suggesting parentheses surrounding a
/// given range.
///
/// \param Loc The location where we'll emit the diagnostic.
/// \param Loc The kind of diagnostic to emit.
/// \param ParenRange Source range enclosing code that should be parenthesized.
void Parser::SuggestParentheses(SourceLocation Loc, unsigned DK,
                                SourceRange ParenRange) {
  SourceLocation EndLoc = PP.getLocForEndOfToken(ParenRange.getEnd());
  if (!ParenRange.getEnd().isFileID() || EndLoc.isInvalid()) {
    // We can't display the parentheses, so just dig the
    // warning/error and return.
    Diag(Loc, DK);
    return;
  }

  Diag(Loc, DK)
    << CodeModificationHint::CreateInsertion(ParenRange.getBegin(), "(")
    << CodeModificationHint::CreateInsertion(EndLoc, ")");
{'fraction_non_alphanumeric': 0.17325344212136665, 'fraction_numerical': 0.0017848036715961244, 'mean_word_length': 3.1398416886543536, 'pattern_counts': {'":': 3, '<': 37, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2716860', 'n_tokens_mistral': 2302, 'n_tokens_neox': 2235, 'n_words': 600}