<commit_before>// Copyright 2017-2020 The Verible Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "verilog/analysis/symbol_table.h"

#include <iostream>
#include <sstream>
#include <stack>

#include "absl/status/status.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_join.h"
#include "common/strings/display_utils.h"
#include "common/text/concrete_syntax_leaf.h"
#include "common/text/concrete_syntax_tree.h"
#include "common/text/token_info.h"
#include "common/text/tree_context_visitor.h"
#include "common/text/tree_utils.h"
#include "common/text/visitors.h"
#include "common/util/enum_flags.h"
#include "common/util/logging.h"
#include "common/util/spacer.h"
#include "common/util/value_saver.h"
#include "verilog/CST/class.h"
#include "verilog/CST/declaration.h"
#include "verilog/CST/functions.h"
#include "verilog/CST/macro.h"
#include "verilog/CST/module.h"
#include "verilog/CST/net.h"
#include "verilog/CST/package.h"
#include "verilog/CST/parameters.h"
#include "verilog/CST/port.h"
#include "verilog/CST/seq_block.h"
#include "verilog/CST/statement.h"
#include "verilog/CST/tasks.h"
#include "verilog/CST/type.h"
#include "verilog/CST/verilog_nonterminals.h"
#include "verilog/analysis/verilog_project.h"
#include "verilog/parser/verilog_parser.h"
#include "verilog/parser/verilog_token_enum.h"

namespace verilog {

using verible::AutoTruncate;
using verible::StringSpanOfSymbol;
using verible::SyntaxTreeLeaf;
using verible::SyntaxTreeNode;
using verible::TokenInfo;
using verible::TreeContextVisitor;
using verible::ValueSaver;

// Returns string_view of `text` with outermost double-quotes removed.
// If `text` is not wrapped in quotes, return it as-is.
static absl::string_view StripOuterQuotes(absl::string_view text) {
  return absl::StripSuffix(absl::StripPrefix(text, "\""), "\"");
}

static const verible::EnumNameMap<SymbolMetaType>& SymbolMetaTypeNames() {
  static const verible::EnumNameMap<SymbolMetaType> kSymbolMetaTypeNames({
      // short-hand annotation for identifier reference type
      {"<root>", SymbolMetaType::kRoot},
      {"class", SymbolMetaType::kClass},
      {"module", SymbolMetaType::kModule},
      {"package", SymbolMetaType::kPackage},
      {"parameter", SymbolMetaType::kParameter},
      {"typedef", SymbolMetaType::kTypeAlias},
      {"data/net/var/instance", SymbolMetaType::kDataNetVariableInstance},
      {"function", SymbolMetaType::kFunction},
      {"task", SymbolMetaType::kTask},
      {"struct", SymbolMetaType::kStruct},
      {"enum", SymbolMetaType::kEnumType},
      {"<enum constant>", SymbolMetaType::kEnumConstant},
      {"interface", SymbolMetaType::kInterface},
      {"<unspecified>", SymbolMetaType::kUnspecified},
      {"<callable>", SymbolMetaType::kCallable},
  });
  return kSymbolMetaTypeNames;
}

std::ostream& operator<<(std::ostream& stream, SymbolMetaType symbol_type) {
  return SymbolMetaTypeNames().Unparse(symbol_type, stream);
}

static absl::string_view SymbolMetaTypeAsString(SymbolMetaType type) {
  return SymbolMetaTypeNames().EnumName(type);
}

// Root SymbolTableNode has no key, but we identify it as "$root"
static constexpr absl::string_view kRoot("$root");

std::ostream& SymbolTableNodeFullPath(std::ostream& stream,
                                      const SymbolTableNode& node) {
  if (node.Parent() != nullptr) {
    SymbolTableNodeFullPath(stream, *node.Parent()) << "::" << *node.Key();
  } else {
    stream << kRoot;
  }
  return stream;
}

static std::string ContextFullPath(const SymbolTableNode& context) {
  std::ostringstream stream;
  SymbolTableNodeFullPath(stream, context);
  return stream.str();
}

std::ostream& ReferenceNodeFullPath(std::ostream& stream,
                                    const ReferenceComponentNode& node) {
  if (node.Parent() != nullptr) {
    ReferenceNodeFullPath(stream, *node.Parent());  // recursive
  }
  return node.Value().PrintPathComponent(stream);
}

static std::string ReferenceNodeFullPathString(
    const ReferenceComponentNode& node) {
  std::ostringstream stream;
  ReferenceNodeFullPath(stream, node);
  return stream.str();
}

static std::ostream& operator<<(std::ostream& stream,
                                const ReferenceComponentNode& ref_node) {
  ref_node.PrintTree(
      &stream,
      [](std::ostream& s, const ReferenceComponent& ref_comp) -> std::ostream& {
        return s << ref_comp;
      });
  return stream;
}

// Validates iterator/pointer stability when calling VectorTree::NewChild.
// Detects unwanted reallocation.
static ReferenceComponentNode* CheckedNewChildReferenceNode(
    ReferenceComponentNode* parent, const ReferenceComponent& component) {
  const auto& siblings(parent->Children());
  if (!siblings.empty()) {
    CHECK_LT(siblings.size(), siblings.capacity())
        << "\nReallocation would invalidate pointers to reference nodes at:\n"
        << *parent << "\nFix: pre-allocate child nodes.";
  }
  // Otherwise, this first node had no prior siblings, so no need to check.
  return parent->NewChild(component);  // copy
}

static absl::Status DiagnoseMemberSymbolResolutionFailure(
    absl::string_view name, const SymbolTableNode& context) {
  const absl::string_view context_name =
      context.Parent() == nullptr ? kRoot : *context.Key();
  return absl::NotFoundError(
      absl::StrCat("No member symbol \"", name, "\" in parent scope (",
                   SymbolMetaTypeAsString(context.Value().metatype), ") ",
                   context_name, "."));
}

static const SymbolTableNode* LookupSymbolUpwards(
    const SymbolTableNode& context, absl::string_view symbol);

class SymbolTable::Builder : public TreeContextVisitor {
 public:
  Builder(const VerilogSourceFile& source, SymbolTable* symbol_table,
          VerilogProject* project)
      : source_(&source),
        token_context_(MakeTokenContext()),
        symbol_table_(symbol_table),
        current_scope_(&symbol_table_->MutableRoot()) {}

  std::vector<absl::Status> TakeDiagnostics() {<commit_msg>Make reallocation check in symbol-table more useful.<commit_after>// Copyright 2017-2020 The Verible Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "verilog/analysis/symbol_table.h"

#include <iostream>
#include <sstream>
#include <stack>

#include "absl/status/status.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_join.h"
#include "common/strings/display_utils.h"
#include "common/text/concrete_syntax_leaf.h"
#include "common/text/concrete_syntax_tree.h"
#include "common/text/token_info.h"
#include "common/text/tree_context_visitor.h"
#include "common/text/tree_utils.h"
#include "common/text/visitors.h"
#include "common/util/enum_flags.h"
#include "common/util/logging.h"
#include "common/util/spacer.h"
#include "common/util/value_saver.h"
#include "verilog/CST/class.h"
#include "verilog/CST/declaration.h"
#include "verilog/CST/functions.h"
#include "verilog/CST/macro.h"
#include "verilog/CST/module.h"
#include "verilog/CST/net.h"
#include "verilog/CST/package.h"
#include "verilog/CST/parameters.h"
#include "verilog/CST/port.h"
#include "verilog/CST/seq_block.h"
#include "verilog/CST/statement.h"
#include "verilog/CST/tasks.h"
#include "verilog/CST/type.h"
#include "verilog/CST/verilog_nonterminals.h"
#include "verilog/analysis/verilog_project.h"
#include "verilog/parser/verilog_parser.h"
#include "verilog/parser/verilog_token_enum.h"

namespace verilog {

using verible::AutoTruncate;
using verible::StringSpanOfSymbol;
using verible::SyntaxTreeLeaf;
using verible::SyntaxTreeNode;
using verible::TokenInfo;
using verible::TreeContextVisitor;
using verible::ValueSaver;

// Returns string_view of `text` with outermost double-quotes removed.
// If `text` is not wrapped in quotes, return it as-is.
static absl::string_view StripOuterQuotes(absl::string_view text) {
  return absl::StripSuffix(absl::StripPrefix(text, "\""), "\"");
}

static const verible::EnumNameMap<SymbolMetaType>& SymbolMetaTypeNames() {
  static const verible::EnumNameMap<SymbolMetaType> kSymbolMetaTypeNames({
      // short-hand annotation for identifier reference type
      {"<root>", SymbolMetaType::kRoot},
      {"class", SymbolMetaType::kClass},
      {"module", SymbolMetaType::kModule},
      {"package", SymbolMetaType::kPackage},
      {"parameter", SymbolMetaType::kParameter},
      {"typedef", SymbolMetaType::kTypeAlias},
      {"data/net/var/instance", SymbolMetaType::kDataNetVariableInstance},
      {"function", SymbolMetaType::kFunction},
      {"task", SymbolMetaType::kTask},
      {"struct", SymbolMetaType::kStruct},
      {"enum", SymbolMetaType::kEnumType},
      {"<enum constant>", SymbolMetaType::kEnumConstant},
      {"interface", SymbolMetaType::kInterface},
      {"<unspecified>", SymbolMetaType::kUnspecified},
      {"<callable>", SymbolMetaType::kCallable},
  });
  return kSymbolMetaTypeNames;
}

std::ostream& operator<<(std::ostream& stream, SymbolMetaType symbol_type) {
  return SymbolMetaTypeNames().Unparse(symbol_type, stream);
}

static absl::string_view SymbolMetaTypeAsString(SymbolMetaType type) {
  return SymbolMetaTypeNames().EnumName(type);
}

// Root SymbolTableNode has no key, but we identify it as "$root"
static constexpr absl::string_view kRoot("$root");

std::ostream& SymbolTableNodeFullPath(std::ostream& stream,
                                      const SymbolTableNode& node) {
  if (node.Parent() != nullptr) {
    SymbolTableNodeFullPath(stream, *node.Parent()) << "::" << *node.Key();
  } else {
    stream << kRoot;
  }
  return stream;
}

static std::string ContextFullPath(const SymbolTableNode& context) {
  std::ostringstream stream;
  SymbolTableNodeFullPath(stream, context);
  return stream.str();
}

std::ostream& ReferenceNodeFullPath(std::ostream& stream,
                                    const ReferenceComponentNode& node) {
  if (node.Parent() != nullptr) {
    ReferenceNodeFullPath(stream, *node.Parent());  // recursive
  }
  return node.Value().PrintPathComponent(stream);
}

static std::string ReferenceNodeFullPathString(
    const ReferenceComponentNode& node) {
  std::ostringstream stream;
  ReferenceNodeFullPath(stream, node);
  return stream.str();
}

static std::ostream& operator<<(std::ostream& stream,
                                const ReferenceComponentNode& ref_node) {
  ref_node.PrintTree(
      &stream,
      [](std::ostream& s, const ReferenceComponent& ref_comp) -> std::ostream& {
        return s << ref_comp;
      });
  return stream;
}

// Validates iterator/pointer stability when calling VectorTree::NewChild.
// Detects unwanted reallocation.
static ReferenceComponentNode* CheckedNewChildReferenceNode(
    ReferenceComponentNode* parent, const ReferenceComponent& component) {
  const auto& siblings(parent->Children());
  if (!siblings.empty()) {
    CHECK_LT(siblings.size(), siblings.capacity())
        << "\nReallocation would invalidate pointers to reference nodes at:\n"
        << *parent << "\nWhile attempting to add child:\n"
        << component << "\nFix: pre-allocate child nodes.";
  }
  // Otherwise, this first node had no prior siblings, so no need to check.
  return parent->NewChild(component);  // copy
}

static absl::Status DiagnoseMemberSymbolResolutionFailure(
    absl::string_view name, const SymbolTableNode& context) {
  const absl::string_view context_name =
      context.Parent() == nullptr ? kRoot : *context.Key();
  return absl::NotFoundError(
      absl::StrCat("No member symbol \"", name, "\" in parent scope (",
                   SymbolMetaTypeAsString(context.Value().metatype), ") ",
                   context_name, "."));
}

static const SymbolTableNode* LookupSymbolUpwards(
    const SymbolTableNode& context, absl::string_view symbol);

class SymbolTable::Builder : public TreeContextVisitor {
 public:
  Builder(const VerilogSourceFile& source, SymbolTable* symbol_table,
          VerilogProject* project)
      : source_(&source),
        token_context_(MakeTokenContext()),
        symbol_table_(symbol_table),
        current_scope_(&symbol_table_->MutableRoot()) {}

  std::vector<absl::Status> TakeDiagnostics() {
{'fraction_non_alphanumeric': 0.13185061484742674, 'fraction_numerical': 0.0018217701533323211, 'mean_word_length': 4.49645390070922, 'pattern_counts': {'":': 2, '<': 63, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '225226', 'n_tokens_mistral': 4034, 'n_tokens_neox': 3979, 'n_words': 912}