<commit_before>//===-- ManualDWARFIndex.cpp -----------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "Plugins/SymbolFile/DWARF/ManualDWARFIndex.h"
#include "Plugins/Language/ObjC/ObjCLanguage.h"
#include "Plugins/SymbolFile/DWARF/DWARFDebugInfo.h"
#include "Plugins/SymbolFile/DWARF/DWARFDeclContext.h"
#include "Plugins/SymbolFile/DWARF/LogChannelDWARF.h"
#include "Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h"
#include "lldb/Core/Module.h"
#include "lldb/Host/TaskPool.h"
#include "lldb/Symbol/ObjectFile.h"
#include "lldb/Utility/Stream.h"
#include "lldb/Utility/Timer.h"

using namespace lldb_private;
using namespace lldb;

void ManualDWARFIndex::Index() {
  if (!m_debug_info)
    return;

  DWARFDebugInfo &debug_info = *m_debug_info;
  m_debug_info = nullptr;

  static Timer::Category func_cat(LLVM_PRETTY_FUNCTION);
  Timer scoped_timer(func_cat, "%p", static_cast<void *>(&debug_info));

  std::vector<DWARFUnit *> units_to_index;
  units_to_index.reserve(debug_info.GetNumCompileUnits());
  for (size_t U = 0; U < debug_info.GetNumCompileUnits(); ++U) {
    DWARFUnit *unit = debug_info.GetCompileUnitAtIndex(U);
    if (unit && m_units_to_avoid.count(unit->GetOffset()) == 0)
      units_to_index.push_back(unit);
  }
  if (units_to_index.empty())
    return;

  std::vector<IndexSet> sets(units_to_index.size());

  //----------------------------------------------------------------------
  // Keep memory down by clearing DIEs for any compile units if indexing
  // caused us to load the compile unit's DIEs.
  //----------------------------------------------------------------------
  std::vector<llvm::Optional<DWARFUnit::ScopedExtractDIEs>> clear_cu_dies(
      units_to_index.size());
  auto parser_fn = [&](size_t cu_idx) {
    IndexUnit(*units_to_index[cu_idx], sets[cu_idx]);
  };

  auto extract_fn = [&units_to_index, &clear_cu_dies](size_t cu_idx) {
    clear_cu_dies[cu_idx] = units_to_index[cu_idx]->ExtractDIEsScoped();
  };

  // Create a task runner that extracts dies for each DWARF compile unit in a
  // separate thread
  //----------------------------------------------------------------------
  // First figure out which compile units didn't have their DIEs already
  // parsed and remember this.  If no DIEs were parsed prior to this index
  // function call, we are going to want to clear the CU dies after we are
  // done indexing to make sure we don't pull in all DWARF dies, but we need
  // to wait until all compile units have been indexed in case a DIE in one
  // compile unit refers to another and the indexes accesses those DIEs.
  //----------------------------------------------------------------------
  TaskMapOverInt(0, units_to_index.size(), extract_fn);

  // Now create a task runner that can index each DWARF compile unit in a
  // separate thread so we can index quickly.

  TaskMapOverInt(0, units_to_index.size(), parser_fn);

  auto finalize_fn = [this, &sets](NameToDIE(IndexSet::*index)) {
    NameToDIE &result = m_set.*index;
    for (auto &set : sets)
      result.Append(set.*index);
    result.Finalize();
  };

  TaskPool::RunTasks([&]() { finalize_fn(&IndexSet::function_basenames); },
                     [&]() { finalize_fn(&IndexSet::function_fullnames); },
                     [&]() { finalize_fn(&IndexSet::function_methods); },
                     [&]() { finalize_fn(&IndexSet::function_selectors); },
                     [&]() { finalize_fn(&IndexSet::objc_class_selectors); },
                     [&]() { finalize_fn(&IndexSet::globals); },
                     [&]() { finalize_fn(&IndexSet::types); },
                     [&]() { finalize_fn(&IndexSet::namespaces); });
}

void ManualDWARFIndex::IndexUnit(DWARFUnit &unit, IndexSet &set) {
  Log *log = LogChannelDWARF::GetLogIfAll(DWARF_LOG_LOOKUPS);

  if (log) {
    m_module.LogMessage(<commit_msg>Add dropped ManualDWARFIndex assert()<commit_after>//===-- ManualDWARFIndex.cpp -----------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "Plugins/SymbolFile/DWARF/ManualDWARFIndex.h"
#include "Plugins/Language/ObjC/ObjCLanguage.h"
#include "Plugins/SymbolFile/DWARF/DWARFDebugInfo.h"
#include "Plugins/SymbolFile/DWARF/DWARFDeclContext.h"
#include "Plugins/SymbolFile/DWARF/LogChannelDWARF.h"
#include "Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h"
#include "lldb/Core/Module.h"
#include "lldb/Host/TaskPool.h"
#include "lldb/Symbol/ObjectFile.h"
#include "lldb/Utility/Stream.h"
#include "lldb/Utility/Timer.h"

using namespace lldb_private;
using namespace lldb;

void ManualDWARFIndex::Index() {
  if (!m_debug_info)
    return;

  DWARFDebugInfo &debug_info = *m_debug_info;
  m_debug_info = nullptr;

  static Timer::Category func_cat(LLVM_PRETTY_FUNCTION);
  Timer scoped_timer(func_cat, "%p", static_cast<void *>(&debug_info));

  std::vector<DWARFUnit *> units_to_index;
  units_to_index.reserve(debug_info.GetNumCompileUnits());
  for (size_t U = 0; U < debug_info.GetNumCompileUnits(); ++U) {
    DWARFUnit *unit = debug_info.GetCompileUnitAtIndex(U);
    if (unit && m_units_to_avoid.count(unit->GetOffset()) == 0)
      units_to_index.push_back(unit);
  }
  if (units_to_index.empty())
    return;

  std::vector<IndexSet> sets(units_to_index.size());

  //----------------------------------------------------------------------
  // Keep memory down by clearing DIEs for any compile units if indexing
  // caused us to load the compile unit's DIEs.
  //----------------------------------------------------------------------
  std::vector<llvm::Optional<DWARFUnit::ScopedExtractDIEs>> clear_cu_dies(
      units_to_index.size());
  auto parser_fn = [&](size_t cu_idx) {
    IndexUnit(*units_to_index[cu_idx], sets[cu_idx]);
  };

  auto extract_fn = [&units_to_index, &clear_cu_dies](size_t cu_idx) {
    clear_cu_dies[cu_idx] = units_to_index[cu_idx]->ExtractDIEsScoped();
  };

  // Create a task runner that extracts dies for each DWARF compile unit in a
  // separate thread
  //----------------------------------------------------------------------
  // First figure out which compile units didn't have their DIEs already
  // parsed and remember this.  If no DIEs were parsed prior to this index
  // function call, we are going to want to clear the CU dies after we are
  // done indexing to make sure we don't pull in all DWARF dies, but we need
  // to wait until all compile units have been indexed in case a DIE in one
  // compile unit refers to another and the indexes accesses those DIEs.
  //----------------------------------------------------------------------
  TaskMapOverInt(0, units_to_index.size(), extract_fn);

  // Now create a task runner that can index each DWARF compile unit in a
  // separate thread so we can index quickly.

  TaskMapOverInt(0, units_to_index.size(), parser_fn);

  auto finalize_fn = [this, &sets](NameToDIE(IndexSet::*index)) {
    NameToDIE &result = m_set.*index;
    for (auto &set : sets)
      result.Append(set.*index);
    result.Finalize();
  };

  TaskPool::RunTasks([&]() { finalize_fn(&IndexSet::function_basenames); },
                     [&]() { finalize_fn(&IndexSet::function_fullnames); },
                     [&]() { finalize_fn(&IndexSet::function_methods); },
                     [&]() { finalize_fn(&IndexSet::function_selectors); },
                     [&]() { finalize_fn(&IndexSet::objc_class_selectors); },
                     [&]() { finalize_fn(&IndexSet::globals); },
                     [&]() { finalize_fn(&IndexSet::types); },
                     [&]() { finalize_fn(&IndexSet::namespaces); });
}

void ManualDWARFIndex::IndexUnit(DWARFUnit &unit, IndexSet &set) {
  assert(!unit.GetSymbolFileDWARF()->GetBaseCompileUnit() &&
      "DWARFUnit associated with .dwo or .dwp should not be indexed directly");

  Log *log = LogChannelDWARF::GetLogIfAll(DWARF_LOG_LOOKUPS);

  if (log) {
    m_module.LogMessage(
{'fraction_non_alphanumeric': 0.21577698695136419, 'fraction_numerical': 0.0018979833926453143, 'mean_word_length': 2.545416316232128, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 18, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3221733', 'n_tokens_mistral': 2708, 'n_tokens_neox': 2636, 'n_words': 586}