<commit_before>//===--- PPDirectives.cpp - Directive Handling for Preprocessor -----------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//  This file implements # directive processing for the Preprocessor.
//
//===----------------------------------------------------------------------===//

#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/LiteralSupport.h"
#include "clang/Lex/HeaderSearch.h"
#include "clang/Lex/MacroInfo.h"
#include "clang/Lex/LexDiagnostic.h"
#include "clang/Lex/CodeCompletionHandler.h"
#include "clang/Lex/Pragma.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/SourceManager.h"
#include "llvm/ADT/APInt.h"
using namespace clang;

//===----------------------------------------------------------------------===//
// Utility Methods for Preprocessor Directive Handling.
//===----------------------------------------------------------------------===//

MacroInfo *Preprocessor::AllocateMacroInfo() {
  MacroInfo *MI;

  if (!MICache.empty()) {
    MI = MICache.back();
    MICache.pop_back();
  } else {
    MacroInfoChain *MIChain = BP.Allocate<MacroInfoChain>();
    MIChain->Next = MIChainHead;
    MIChainHead = MIChain;
    MI = &(MIChain->MI);
  }
  return MI;
}

MacroInfo *Preprocessor::AllocateMacroInfo(SourceLocation L) {
  MacroInfo *MI = AllocateMacroInfo();
  new (MI) MacroInfo(L);
  return MI;
}

MacroInfo *Preprocessor::CloneMacroInfo(const MacroInfo &MacroToClone) {
  MacroInfo *MI = AllocateMacroInfo();
  new (MI) MacroInfo(MacroToClone, BP);
  return MI;
}

/// ReleaseMacroInfo - Release the specified MacroInfo.  This memory will
///  be reused for allocating new MacroInfo objects.
void Preprocessor::ReleaseMacroInfo(MacroInfo *MI) {
  MICache.push_back(MI);
  // We need to call 'Destroy' as opposed to 'FreeArgumentList' because
  // the MICache object will get reused with a placement new.  This does
  // not cause the underlying SmallVector to get it's memory released, so
  // we need to call Destroy() here.
  MI->Destroy();
}


/// DiscardUntilEndOfDirective - Read and discard all tokens remaining on the
/// current line until the tok::eom token is found.
void Preprocessor::DiscardUntilEndOfDirective() {
  Token Tmp;
  do {
    LexUnexpandedToken(Tmp);
  } while (Tmp.isNot(tok::eom));
}

/// ReadMacroName - Lex and validate a macro name, which occurs after a
/// #define or #undef.  This sets the token kind to eom and discards the rest
/// of the macro line if the macro name is invalid.  isDefineUndef is 1 if
/// this is due to a a #define, 2 if #undef directive, 0 if it is something<commit_msg>Revert most of r116862.  It isn't quite the right fix for a memory leak in Preprocessor.<commit_after>//===--- PPDirectives.cpp - Directive Handling for Preprocessor -----------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//  This file implements # directive processing for the Preprocessor.
//
//===----------------------------------------------------------------------===//

#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/LiteralSupport.h"
#include "clang/Lex/HeaderSearch.h"
#include "clang/Lex/MacroInfo.h"
#include "clang/Lex/LexDiagnostic.h"
#include "clang/Lex/CodeCompletionHandler.h"
#include "clang/Lex/Pragma.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/SourceManager.h"
#include "llvm/ADT/APInt.h"
using namespace clang;

//===----------------------------------------------------------------------===//
// Utility Methods for Preprocessor Directive Handling.
//===----------------------------------------------------------------------===//

MacroInfo *Preprocessor::AllocateMacroInfo() {
  MacroInfo *MI;

  if (!MICache.empty()) {
    MI = MICache.back();
    MICache.pop_back();
  } else {
    MacroInfoChain *MIChain = BP.Allocate<MacroInfoChain>();
    MIChain->Next = MIChainHead;
    MIChainHead = MIChain;
    MI = &(MIChain->MI);
  }
  return MI;
}

MacroInfo *Preprocessor::AllocateMacroInfo(SourceLocation L) {
  MacroInfo *MI = AllocateMacroInfo();
  new (MI) MacroInfo(L);
  return MI;
}

MacroInfo *Preprocessor::CloneMacroInfo(const MacroInfo &MacroToClone) {
  MacroInfo *MI = AllocateMacroInfo();
  new (MI) MacroInfo(MacroToClone, BP);
  return MI;
}

/// ReleaseMacroInfo - Release the specified MacroInfo.  This memory will
///  be reused for allocating new MacroInfo objects.
void Preprocessor::ReleaseMacroInfo(MacroInfo *MI) {
  MICache.push_back(MI);
  MI->FreeArgumentList();
}


/// DiscardUntilEndOfDirective - Read and discard all tokens remaining on the
/// current line until the tok::eom token is found.
void Preprocessor::DiscardUntilEndOfDirective() {
  Token Tmp;
  do {
    LexUnexpandedToken(Tmp);
  } while (Tmp.isNot(tok::eom));
}

/// ReadMacroName - Lex and validate a macro name, which occurs after a
/// #define or #undef.  This sets the token kind to eom and discards the rest
/// of the macro line if the macro name is invalid.  isDefineUndef is 1 if
/// this is due to a a #define, 2 if #undef directive, 0 if it is something
{'fraction_non_alphanumeric': 0.2250644092749356, 'fraction_numerical': 0.002208317997791682, 'mean_word_length': 2.373680943513346, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6166859', 'n_tokens_mistral': 1644, 'n_tokens_neox': 1595, 'n_words': 469}