<commit_before>//===- tools/dsymutil/MachODebugMapParser.cpp - Parse STABS debug maps ----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "BinaryHolder.h"
#include "DebugMap.h"
#include "MachOUtils.h"
#include "llvm/ADT/Optional.h"
#include "llvm/Object/MachO.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/WithColor.h"
#include "llvm/Support/raw_ostream.h"
#include <vector>

namespace {
using namespace llvm;
using namespace llvm::dsymutil;
using namespace llvm::object;

class MachODebugMapParser {
public:
  MachODebugMapParser(StringRef BinaryPath, ArrayRef<std::string> Archs,
                      StringRef PathPrefix = "",
                      bool PaperTrailWarnings = false, bool Verbose = false)
      : BinaryPath(BinaryPath), Archs(Archs.begin(), Archs.end()),
        PathPrefix(PathPrefix), PaperTrailWarnings(PaperTrailWarnings),
        BinHolder(Verbose), CurrentDebugMapObject(nullptr) {}

  /// Parses and returns the DebugMaps of the input binary. The binary contains
  /// multiple maps in case it is a universal binary.
  /// \returns an error in case the provided BinaryPath doesn't exist
  /// or isn't of a supported type.
  ErrorOr<std::vector<std::unique_ptr<DebugMap>>> parse();

  /// Walk the symbol table and dump it.
  bool dumpStab();

private:
  std::string BinaryPath;
  SmallVector<StringRef, 1> Archs;
  std::string PathPrefix;
  bool PaperTrailWarnings;

  /// Owns the MemoryBuffer for the main binary.
  BinaryHolder BinHolder;
  /// Map of the binary symbol addresses.
  StringMap<uint64_t> MainBinarySymbolAddresses;
  StringRef MainBinaryStrings;
  /// The constructed DebugMap.
  std::unique_ptr<DebugMap> Result;
  /// List of common symbols that need to be added to the debug map.
  std::vector<std::string> CommonSymbols;

  /// Map of the currently processed object file symbol addresses.
  StringMap<Optional<uint64_t>> CurrentObjectAddresses;
  /// Element of the debug map corresponding to the current object file.
  DebugMapObject *CurrentDebugMapObject;

  /// Holds function info while function scope processing.
  const char *CurrentFunctionName;
  uint64_t CurrentFunctionAddress;

  std::unique_ptr<DebugMap> parseOneBinary(const MachOObjectFile &MainBinary,
                                           StringRef BinaryPath);

  void
  switchToNewDebugMapObject(StringRef Filename,
                            sys::TimePoint<std::chrono::seconds> Timestamp);
  void resetParserState();
  uint64_t getMainBinarySymbolAddress(StringRef Name);
  std::vector<StringRef> getMainBinarySymbolNames(uint64_t Value);
  void loadMainBinarySymbols(const MachOObjectFile &MainBinary);
  void loadCurrentObjectFileSymbols(const object::MachOObjectFile &Obj);
  void handleStabSymbolTableEntry(uint32_t StringIndex, uint8_t Type,
                                  uint8_t SectionIndex, uint16_t Flags,
                                  uint64_t Value);

  template <typename STEType> void handleStabDebugMapEntry(const STEType &STE) {
    handleStabSymbolTableEntry(STE.n_strx, STE.n_type, STE.n_sect, STE.n_desc,
                               STE.n_value);
  }

  void addCommonSymbols();

  /// Dump the symbol table output header.
  void dumpSymTabHeader(raw_ostream &OS, StringRef Arch);

  /// Dump the contents of nlist entries.
  void dumpSymTabEntry(raw_ostream &OS, uint64_t Index, uint32_t StringIndex,
                       uint8_t Type, uint8_t SectionIndex, uint16_t Flags,
                       uint64_t Value);

  template <typename STEType>
  void dumpSymTabEntry(raw_ostream &OS, uint64_t Index, const STEType &STE) {
    dumpSymTabEntry(OS, Index, STE.n_strx, STE.n_type, STE.n_sect, STE.n_desc,
                    STE.n_value);
  }
  void dumpOneBinaryStab(const MachOObjectFile &MainBinary,
                         StringRef BinaryPath);

  void Warning(const Twine &Msg, StringRef File = StringRef()) {
    WithColor::warning() << "("
                         << MachOUtils::getArchName(
                                Result->getTriple().getArchName())
                         << ") " << File << " " << Msg << "\n";

    if (PaperTrailWarnings) {
      if (!File.empty())
        Result->addDebugMapObject(File, sys::TimePoint<std::chrono::seconds>());
      if (Result->end() != Result->begin())
        (*--Result->end())->addWarning(Msg.str());
    }
  }
};

} // anonymous namespace

/// Reset the parser state corresponding to the current object
/// file. This is to be called after an object file is finished
/// processing.
void MachODebugMapParser::resetParserState() {
  CurrentObjectAddresses.clear();
  CurrentDebugMapObject = nullptr;
}

/// Commons symbols won't show up in the symbol map but might need to be
/// relocated. We can add them to the symbol table ourselves by combining the
/// information in the object file (the symbol name) and the main binary (the
/// address).
void MachODebugMapParser::addCommonSymbols() {
  for (auto &CommonSymbol : CommonSymbols) {
    uint64_t CommonAddr = getMainBinarySymbolAddress(CommonSymbol);
    if (CommonAddr == 0) {
      // The main binary doesn't have an address for the given symbol.
      continue;
    }
    if (!CurrentDebugMapObject->addSymbol(CommonSymbol, None /*ObjectAddress*/,
                                          CommonAddr, 0 /*size*/)) {
      // The symbol is already present.
      continue;
    }
  }
  CommonSymbols.clear();
}

/// Create a new DebugMapObject. This function resets the state of the
/// parser that was referring to the last object file and sets
/// everything up to add symbols to the new one.
void MachODebugMapParser::switchToNewDebugMapObject(
    StringRef Filename, sys::TimePoint<std::chrono::seconds> Timestamp) {
  addCommonSymbols();
  resetParserState();

  SmallString<80> Path(PathPrefix);
  sys::path::append(Path, Filename);

  auto ObjectEntry = BinHolder.getObjectEntry(Path, Timestamp);
  if (!ObjectEntry) {
    auto Err = ObjectEntry.takeError();
    Warning("unable to open object file: " + toString(std::move(Err)),
            Path.str());
    return;
  }

  auto Object = ObjectEntry->getObjectAs<MachOObjectFile>(Result->getTriple());
  if (!Object) {
    auto Err = Object.takeError();
    Warning("unable to open object file: " + toString(std::move(Err)),
            Path.str());
    return;
  }
<commit_msg>[dsymutil] Move CommonSymbols.clear() in resetParserState.<commit_after>//===- tools/dsymutil/MachODebugMapParser.cpp - Parse STABS debug maps ----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "BinaryHolder.h"
#include "DebugMap.h"
#include "MachOUtils.h"
#include "llvm/ADT/Optional.h"
#include "llvm/Object/MachO.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/WithColor.h"
#include "llvm/Support/raw_ostream.h"
#include <vector>

namespace {
using namespace llvm;
using namespace llvm::dsymutil;
using namespace llvm::object;

class MachODebugMapParser {
public:
  MachODebugMapParser(StringRef BinaryPath, ArrayRef<std::string> Archs,
                      StringRef PathPrefix = "",
                      bool PaperTrailWarnings = false, bool Verbose = false)
      : BinaryPath(BinaryPath), Archs(Archs.begin(), Archs.end()),
        PathPrefix(PathPrefix), PaperTrailWarnings(PaperTrailWarnings),
        BinHolder(Verbose), CurrentDebugMapObject(nullptr) {}

  /// Parses and returns the DebugMaps of the input binary. The binary contains
  /// multiple maps in case it is a universal binary.
  /// \returns an error in case the provided BinaryPath doesn't exist
  /// or isn't of a supported type.
  ErrorOr<std::vector<std::unique_ptr<DebugMap>>> parse();

  /// Walk the symbol table and dump it.
  bool dumpStab();

private:
  std::string BinaryPath;
  SmallVector<StringRef, 1> Archs;
  std::string PathPrefix;
  bool PaperTrailWarnings;

  /// Owns the MemoryBuffer for the main binary.
  BinaryHolder BinHolder;
  /// Map of the binary symbol addresses.
  StringMap<uint64_t> MainBinarySymbolAddresses;
  StringRef MainBinaryStrings;
  /// The constructed DebugMap.
  std::unique_ptr<DebugMap> Result;
  /// List of common symbols that need to be added to the debug map.
  std::vector<std::string> CommonSymbols;

  /// Map of the currently processed object file symbol addresses.
  StringMap<Optional<uint64_t>> CurrentObjectAddresses;
  /// Element of the debug map corresponding to the current object file.
  DebugMapObject *CurrentDebugMapObject;

  /// Holds function info while function scope processing.
  const char *CurrentFunctionName;
  uint64_t CurrentFunctionAddress;

  std::unique_ptr<DebugMap> parseOneBinary(const MachOObjectFile &MainBinary,
                                           StringRef BinaryPath);

  void
  switchToNewDebugMapObject(StringRef Filename,
                            sys::TimePoint<std::chrono::seconds> Timestamp);
  void resetParserState();
  uint64_t getMainBinarySymbolAddress(StringRef Name);
  std::vector<StringRef> getMainBinarySymbolNames(uint64_t Value);
  void loadMainBinarySymbols(const MachOObjectFile &MainBinary);
  void loadCurrentObjectFileSymbols(const object::MachOObjectFile &Obj);
  void handleStabSymbolTableEntry(uint32_t StringIndex, uint8_t Type,
                                  uint8_t SectionIndex, uint16_t Flags,
                                  uint64_t Value);

  template <typename STEType> void handleStabDebugMapEntry(const STEType &STE) {
    handleStabSymbolTableEntry(STE.n_strx, STE.n_type, STE.n_sect, STE.n_desc,
                               STE.n_value);
  }

  void addCommonSymbols();

  /// Dump the symbol table output header.
  void dumpSymTabHeader(raw_ostream &OS, StringRef Arch);

  /// Dump the contents of nlist entries.
  void dumpSymTabEntry(raw_ostream &OS, uint64_t Index, uint32_t StringIndex,
                       uint8_t Type, uint8_t SectionIndex, uint16_t Flags,
                       uint64_t Value);

  template <typename STEType>
  void dumpSymTabEntry(raw_ostream &OS, uint64_t Index, const STEType &STE) {
    dumpSymTabEntry(OS, Index, STE.n_strx, STE.n_type, STE.n_sect, STE.n_desc,
                    STE.n_value);
  }
  void dumpOneBinaryStab(const MachOObjectFile &MainBinary,
                         StringRef BinaryPath);

  void Warning(const Twine &Msg, StringRef File = StringRef()) {
    WithColor::warning() << "("
                         << MachOUtils::getArchName(
                                Result->getTriple().getArchName())
                         << ") " << File << " " << Msg << "\n";

    if (PaperTrailWarnings) {
      if (!File.empty())
        Result->addDebugMapObject(File, sys::TimePoint<std::chrono::seconds>());
      if (Result->end() != Result->begin())
        (*--Result->end())->addWarning(Msg.str());
    }
  }
};

} // anonymous namespace

/// Reset the parser state corresponding to the current object
/// file. This is to be called after an object file is finished
/// processing.
void MachODebugMapParser::resetParserState() {
  CommonSymbols.clear();
  CurrentObjectAddresses.clear();
  CurrentDebugMapObject = nullptr;
}

/// Commons symbols won't show up in the symbol map but might need to be
/// relocated. We can add them to the symbol table ourselves by combining the
/// information in the object file (the symbol name) and the main binary (the
/// address).
void MachODebugMapParser::addCommonSymbols() {
  for (auto &CommonSymbol : CommonSymbols) {
    uint64_t CommonAddr = getMainBinarySymbolAddress(CommonSymbol);
    if (CommonAddr == 0) {
      // The main binary doesn't have an address for the given symbol.
      continue;
    }
    if (!CurrentDebugMapObject->addSymbol(CommonSymbol, None /*ObjectAddress*/,
                                          CommonAddr, 0 /*size*/)) {
      // The symbol is already present.
      continue;
    }
  }
}

/// Create a new DebugMapObject. This function resets the state of the
/// parser that was referring to the last object file and sets
/// everything up to add symbols to the new one.
void MachODebugMapParser::switchToNewDebugMapObject(
    StringRef Filename, sys::TimePoint<std::chrono::seconds> Timestamp) {
  addCommonSymbols();
  resetParserState();

  SmallString<80> Path(PathPrefix);
  sys::path::append(Path, Filename);

  auto ObjectEntry = BinHolder.getObjectEntry(Path, Timestamp);
  if (!ObjectEntry) {
    auto Err = ObjectEntry.takeError();
    Warning("unable to open object file: " + toString(std::move(Err)),
            Path.str());
    return;
  }

  auto Object = ObjectEntry->getObjectAs<MachOObjectFile>(Result->getTriple());
  if (!Object) {
    auto Err = Object.takeError();
    Warning("unable to open object file: " + toString(std::move(Err)),
            Path.str());
    return;
  }

{'fraction_non_alphanumeric': 0.11859144263536539, 'fraction_numerical': 0.006209769026883756, 'mean_word_length': 3.0274473924977126, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 61, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '353124', 'n_tokens_mistral': 3826, 'n_tokens_neox': 3736, 'n_words': 1056}