<commit_before>//===--- CommentBriefParser.cpp - Dumb comment parser ---------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#include "clang/AST/CommentBriefParser.h"
#include "llvm/ADT/StringSwitch.h"

namespace clang {
namespace comments {

namespace {
/// Convert all whitespace into spaces, remove leading and trailing spaces,
/// compress multiple spaces into one.
void cleanupBrief(std::string &S) {
  bool PrevWasSpace = true;
  std::string::iterator O = S.begin();
  for (std::string::iterator I = S.begin(), E = S.end();
       I != E; ++I) {
    const char C = *I;
    if (C == ' ' || C == '\n' || C == '\r' ||
        C == '\t' || C == '\v' || C == '\f') {
      if (!PrevWasSpace) {
        *O++ = ' ';
        PrevWasSpace = true;
      }
      continue;
    } else {
      *O++ = C;
      PrevWasSpace = false;
    }
  }
  if (O != S.begin() && *(O - 1) == ' ')
    --O;

  S.resize(O - S.begin());
}

bool isBlockCommand(StringRef Name) {
  return llvm::StringSwitch<bool>(Name)
      .Case("brief", true)
      .Case("result", true)
      .Case("return", true)
      .Case("returns", true)
      .Case("author", true)
      .Case("authors", true)
      .Case("pre", true)
      .Case("post", true)
      .Case("param", true)
      .Case("arg", true)
      .Default(false);
}
} // unnamed namespace

std::string BriefParser::Parse() {
  std::string Paragraph;
  bool InFirstParagraph = true;
  bool InBrief = false;

  while (Tok.isNot(tok::eof)) {
    if (Tok.is(tok::text)) {
      if (InFirstParagraph || InBrief)
        Paragraph += Tok.getText();
      ConsumeToken();
      continue;
    }

    if (Tok.is(tok::command)) {
      StringRef Name = Tok.getCommandName();
      if (Name == "brief") {
        Paragraph.clear();
        InBrief = true;
        ConsumeToken();
        continue;
      }
      // Block commands implicitly start a new paragraph.
      if (isBlockCommand(Name)) {
        // We found an implicit paragraph end.
        InFirstParagraph = false;
        if (InBrief) {
          InBrief = false;
          break;
        }
      }
    }

    if (Tok.is(tok::newline)) {
      if (InFirstParagraph || InBrief)
        Paragraph += ' ';
      ConsumeToken();

      if (Tok.is(tok::newline)) {
        ConsumeToken();
        // We found a paragraph end.
        InFirstParagraph = false;
        if (InBrief) {
          InBrief = false;
          break;
        }
      }
      continue;
    }

    // We didn't handle this token, so just drop it.
    ConsumeToken();
  }

  cleanupBrief(Paragraph);
  return Paragraph;
}

BriefParser::BriefParser(Lexer &L) : L(L)
{
  // Get lookahead token.
  ConsumeToken();
}

} // end namespace comments
} // end namespace clang

<commit_msg>CommentBriefParser: remove dead store.  Found by Clang Analyzer.<commit_after>//===--- CommentBriefParser.cpp - Dumb comment parser ---------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#include "clang/AST/CommentBriefParser.h"
#include "llvm/ADT/StringSwitch.h"

namespace clang {
namespace comments {

namespace {
/// Convert all whitespace into spaces, remove leading and trailing spaces,
/// compress multiple spaces into one.
void cleanupBrief(std::string &S) {
  bool PrevWasSpace = true;
  std::string::iterator O = S.begin();
  for (std::string::iterator I = S.begin(), E = S.end();
       I != E; ++I) {
    const char C = *I;
    if (C == ' ' || C == '\n' || C == '\r' ||
        C == '\t' || C == '\v' || C == '\f') {
      if (!PrevWasSpace) {
        *O++ = ' ';
        PrevWasSpace = true;
      }
      continue;
    } else {
      *O++ = C;
      PrevWasSpace = false;
    }
  }
  if (O != S.begin() && *(O - 1) == ' ')
    --O;

  S.resize(O - S.begin());
}

bool isBlockCommand(StringRef Name) {
  return llvm::StringSwitch<bool>(Name)
      .Case("brief", true)
      .Case("result", true)
      .Case("return", true)
      .Case("returns", true)
      .Case("author", true)
      .Case("authors", true)
      .Case("pre", true)
      .Case("post", true)
      .Case("param", true)
      .Case("arg", true)
      .Default(false);
}
} // unnamed namespace

std::string BriefParser::Parse() {
  std::string Paragraph;
  bool InFirstParagraph = true;
  bool InBrief = false;

  while (Tok.isNot(tok::eof)) {
    if (Tok.is(tok::text)) {
      if (InFirstParagraph || InBrief)
        Paragraph += Tok.getText();
      ConsumeToken();
      continue;
    }

    if (Tok.is(tok::command)) {
      StringRef Name = Tok.getCommandName();
      if (Name == "brief") {
        Paragraph.clear();
        InBrief = true;
        ConsumeToken();
        continue;
      }
      // Block commands implicitly start a new paragraph.
      if (isBlockCommand(Name)) {
        // We found an implicit paragraph end.
        InFirstParagraph = false;
        if (InBrief)
          break;
      }
    }

    if (Tok.is(tok::newline)) {
      if (InFirstParagraph || InBrief)
        Paragraph += ' ';
      ConsumeToken();

      if (Tok.is(tok::newline)) {
        ConsumeToken();
        // We found a paragraph end.
        InFirstParagraph = false;
        if (InBrief)
          break;
      }
      continue;
    }

    // We didn't handle this token, so just drop it.
    ConsumeToken();
  }

  cleanupBrief(Paragraph);
  return Paragraph;
}

BriefParser::BriefParser(Lexer &L) : L(L)
{
  // Get lookahead token.
  ConsumeToken();
}

} // end namespace comments
} // end namespace clang


{'fraction_non_alphanumeric': 0.18723476489560345, 'fraction_numerical': 0.00033950093362756747, 'mean_word_length': 1.9415876185721417, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6694318', 'n_tokens_mistral': 1881, 'n_tokens_neox': 1842, 'n_words': 473}