<commit_before>//===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
///
/// \file
/// \brief This file contains the implementation of the UnwrappedLineParser,
/// which turns a stream of tokens into UnwrappedLines.
///
//===----------------------------------------------------------------------===//

#define DEBUG_TYPE "format-parser"

#include "UnwrappedLineParser.h"
#include "llvm/Support/Debug.h"

namespace clang {
namespace format {

class FormatTokenSource {
public:
  virtual ~FormatTokenSource() {}
  virtual FormatToken *getNextToken() = 0;

  virtual unsigned getPosition() = 0;
  virtual FormatToken *setPosition(unsigned Position) = 0;
};

class ScopedDeclarationState {
public:
  ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
                         bool MustBeDeclaration)
      : Line(Line), Stack(Stack) {
    Line.MustBeDeclaration = MustBeDeclaration;
    Stack.push_back(MustBeDeclaration);
  }
  ~ScopedDeclarationState() {
    Stack.pop_back();
    if (!Stack.empty())
      Line.MustBeDeclaration = Stack.back();
    else
      Line.MustBeDeclaration = true;
  }

private:
  UnwrappedLine &Line;
  std::vector<bool> &Stack;
};

class ScopedMacroState : public FormatTokenSource {
public:
  ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
                   FormatToken *&ResetToken, bool &StructuralError)
      : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
        PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
        StructuralError(StructuralError),
        PreviousStructuralError(StructuralError), Token(NULL) {
    TokenSource = this;
    Line.Level = 0;
    Line.InPPDirective = true;
  }

  ~ScopedMacroState() {
    TokenSource = PreviousTokenSource;
    ResetToken = Token;
    Line.InPPDirective = false;
    Line.Level = PreviousLineLevel;
    StructuralError = PreviousStructuralError;
  }

  virtual FormatToken *getNextToken() {
    // The \c UnwrappedLineParser guards against this by never calling
    // \c getNextToken() after it has encountered the first eof token.
    assert(!eof());
    Token = PreviousTokenSource->getNextToken();
    if (eof())
      return getFakeEOF();
    return Token;
  }

  virtual unsigned getPosition() { return PreviousTokenSource->getPosition(); }

  virtual FormatToken *setPosition(unsigned Position) {
    Token = PreviousTokenSource->setPosition(Position);
    return Token;
  }

private:
  bool eof() { return Token && Token->HasUnescapedNewline; }

  FormatToken *getFakeEOF() {
    static bool EOFInitialized = false;
    static FormatToken FormatTok;
    if (!EOFInitialized) {
      FormatTok.Tok.startToken();
      FormatTok.Tok.setKind(tok::eof);
      EOFInitialized = true;
    }
    return &FormatTok;
  }

  UnwrappedLine &Line;
  FormatTokenSource *&TokenSource;
  FormatToken *&ResetToken;
  unsigned PreviousLineLevel;
  FormatTokenSource *PreviousTokenSource;
  bool &StructuralError;
  bool PreviousStructuralError;

  FormatToken *Token;
};

class ScopedLineState {
public:
  ScopedLineState(UnwrappedLineParser &Parser,
                  bool SwitchToPreprocessorLines = false)
      : Parser(Parser), SwitchToPreprocessorLines(SwitchToPreprocessorLines) {
    if (SwitchToPreprocessorLines)
      Parser.CurrentLines = &Parser.PreprocessorDirectives;
    PreBlockLine = Parser.Line.take();
    Parser.Line.reset(new UnwrappedLine());
    Parser.Line->Level = PreBlockLine->Level;
    Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
  }

  ~ScopedLineState() {
    if (!Parser.Line->Tokens.empty()) {
      Parser.addUnwrappedLine();
    }
    assert(Parser.Line->Tokens.empty());
    Parser.Line.reset(PreBlockLine);
    Parser.MustBreakBeforeNextToken = true;
    if (SwitchToPreprocessorLines)
      Parser.CurrentLines = &Parser.Lines;
  }

private:
  UnwrappedLineParser &Parser;
  const bool SwitchToPreprocessorLines;

  UnwrappedLine *PreBlockLine;
};

class IndexedTokenSource : public FormatTokenSource {
public:
  IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
      : Tokens(Tokens), Position(-1) {}

  virtual FormatToken *getNextToken() {
    ++Position;
    return Tokens[Position];
  }

  virtual unsigned getPosition() {
    assert(Position >= 0);
    return Position;
  }

  virtual FormatToken *setPosition(unsigned P) {
    Position = P;
    return Tokens[Position];
  }

private:
  ArrayRef<FormatToken *> Tokens;
  int Position;
};

UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
                                         ArrayRef<FormatToken *> Tokens,
                                         UnwrappedLineConsumer &Callback)
    : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
      CurrentLines(&Lines), StructuralError(false), Style(Style), Tokens(NULL),
      Callback(Callback), AllTokens(Tokens) {
  LBraces.resize(Tokens.size(), BS_Unknown);
}

bool UnwrappedLineParser::parse() {
  DEBUG(llvm::dbgs() << "----\n");
  IndexedTokenSource TokenSource(AllTokens);
  Tokens = &TokenSource;
  readToken();
  parseFile();
  for (std::vector<UnwrappedLine>::iterator I = Lines.begin(), E = Lines.end();
       I != E; ++I) {
    Callback.consumeUnwrappedLine(*I);
  }<commit_msg>Put helper classes in an anonymous namespace.<commit_after>//===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
///
/// \file
/// \brief This file contains the implementation of the UnwrappedLineParser,
/// which turns a stream of tokens into UnwrappedLines.
///
//===----------------------------------------------------------------------===//

#define DEBUG_TYPE "format-parser"

#include "UnwrappedLineParser.h"
#include "llvm/Support/Debug.h"

namespace clang {
namespace format {

class FormatTokenSource {
public:
  virtual ~FormatTokenSource() {}
  virtual FormatToken *getNextToken() = 0;

  virtual unsigned getPosition() = 0;
  virtual FormatToken *setPosition(unsigned Position) = 0;
};

namespace {

class ScopedDeclarationState {
public:
  ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
                         bool MustBeDeclaration)
      : Line(Line), Stack(Stack) {
    Line.MustBeDeclaration = MustBeDeclaration;
    Stack.push_back(MustBeDeclaration);
  }
  ~ScopedDeclarationState() {
    Stack.pop_back();
    if (!Stack.empty())
      Line.MustBeDeclaration = Stack.back();
    else
      Line.MustBeDeclaration = true;
  }

private:
  UnwrappedLine &Line;
  std::vector<bool> &Stack;
};

class ScopedMacroState : public FormatTokenSource {
public:
  ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
                   FormatToken *&ResetToken, bool &StructuralError)
      : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
        PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
        StructuralError(StructuralError),
        PreviousStructuralError(StructuralError), Token(NULL) {
    TokenSource = this;
    Line.Level = 0;
    Line.InPPDirective = true;
  }

  ~ScopedMacroState() {
    TokenSource = PreviousTokenSource;
    ResetToken = Token;
    Line.InPPDirective = false;
    Line.Level = PreviousLineLevel;
    StructuralError = PreviousStructuralError;
  }

  virtual FormatToken *getNextToken() {
    // The \c UnwrappedLineParser guards against this by never calling
    // \c getNextToken() after it has encountered the first eof token.
    assert(!eof());
    Token = PreviousTokenSource->getNextToken();
    if (eof())
      return getFakeEOF();
    return Token;
  }

  virtual unsigned getPosition() { return PreviousTokenSource->getPosition(); }

  virtual FormatToken *setPosition(unsigned Position) {
    Token = PreviousTokenSource->setPosition(Position);
    return Token;
  }

private:
  bool eof() { return Token && Token->HasUnescapedNewline; }

  FormatToken *getFakeEOF() {
    static bool EOFInitialized = false;
    static FormatToken FormatTok;
    if (!EOFInitialized) {
      FormatTok.Tok.startToken();
      FormatTok.Tok.setKind(tok::eof);
      EOFInitialized = true;
    }
    return &FormatTok;
  }

  UnwrappedLine &Line;
  FormatTokenSource *&TokenSource;
  FormatToken *&ResetToken;
  unsigned PreviousLineLevel;
  FormatTokenSource *PreviousTokenSource;
  bool &StructuralError;
  bool PreviousStructuralError;

  FormatToken *Token;
};

} // end anonymous namespace

class ScopedLineState {
public:
  ScopedLineState(UnwrappedLineParser &Parser,
                  bool SwitchToPreprocessorLines = false)
      : Parser(Parser), SwitchToPreprocessorLines(SwitchToPreprocessorLines) {
    if (SwitchToPreprocessorLines)
      Parser.CurrentLines = &Parser.PreprocessorDirectives;
    PreBlockLine = Parser.Line.take();
    Parser.Line.reset(new UnwrappedLine());
    Parser.Line->Level = PreBlockLine->Level;
    Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
  }

  ~ScopedLineState() {
    if (!Parser.Line->Tokens.empty()) {
      Parser.addUnwrappedLine();
    }
    assert(Parser.Line->Tokens.empty());
    Parser.Line.reset(PreBlockLine);
    Parser.MustBreakBeforeNextToken = true;
    if (SwitchToPreprocessorLines)
      Parser.CurrentLines = &Parser.Lines;
  }

private:
  UnwrappedLineParser &Parser;
  const bool SwitchToPreprocessorLines;

  UnwrappedLine *PreBlockLine;
};

namespace {

class IndexedTokenSource : public FormatTokenSource {
public:
  IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
      : Tokens(Tokens), Position(-1) {}

  virtual FormatToken *getNextToken() {
    ++Position;
    return Tokens[Position];
  }

  virtual unsigned getPosition() {
    assert(Position >= 0);
    return Position;
  }

  virtual FormatToken *setPosition(unsigned P) {
    Position = P;
    return Tokens[Position];
  }

private:
  ArrayRef<FormatToken *> Tokens;
  int Position;
};

} // end anonymous namespace

UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
                                         ArrayRef<FormatToken *> Tokens,
                                         UnwrappedLineConsumer &Callback)
    : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
      CurrentLines(&Lines), StructuralError(false), Style(Style), Tokens(NULL),
      Callback(Callback), AllTokens(Tokens) {
  LBraces.resize(Tokens.size(), BS_Unknown);
}

bool UnwrappedLineParser::parse() {
  DEBUG(llvm::dbgs() << "----\n");
  IndexedTokenSource TokenSource(AllTokens);
  Tokens = &TokenSource;
  readToken();
  parseFile();
  for (std::vector<UnwrappedLine>::iterator I = Lines.begin(), E = Lines.end();
       I != E; ++I) {
    Callback.consumeUnwrappedLine(*I);
  }
{'fraction_non_alphanumeric': 0.14112903225806453, 'fraction_numerical': 0.001075268817204301, 'mean_word_length': 3.1583457526080476, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4313250', 'n_tokens_mistral': 3233, 'n_tokens_neox': 3153, 'n_words': 686}