<commit_before>/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

#include "DexLoader.h"

#include "AggregateException.h"
#include "DexAccess.h"
#include "DexCallSite.h"
#include "DexDefs.h"
#include "DexMethodHandle.h"
#include "IRCode.h"
#include "Macros.h"
#include "Show.h"
#include "Trace.h"
#include "Walkers.h"
#include "WorkQueue.h"

#include <exception>
#include <stdexcept>
#include <vector>

DexLoader::DexLoader(const DexLocation* location)
    : m_idx(nullptr),
      m_file(new boost::iostreams::mapped_file()),
      m_location(location) {}

static void validate_dex_header(const dex_header* dh,
                                size_t dexsize,
                                int support_dex_version) {
  bool supported = false;
  switch (support_dex_version) {
  case 38:
    supported = supported ||
                !memcmp(dh->magic, DEX_HEADER_DEXMAGIC_V38, sizeof(dh->magic));
    FALLTHROUGH_INTENDED; /* intentional fallthrough to also check for v37 */
  case 37:
    supported = supported ||
                !memcmp(dh->magic, DEX_HEADER_DEXMAGIC_V37, sizeof(dh->magic));
    FALLTHROUGH_INTENDED; /* intentional fallthrough to also check for v35 */
  case 35:
    supported = supported ||
                !memcmp(dh->magic, DEX_HEADER_DEXMAGIC_V35, sizeof(dh->magic));
    break;
  default:
    not_reached_log("Unrecognized support_dex_version %d\n",
                    support_dex_version);
  }
  always_assert_log(supported, "Bad dex magic %s for support_dex_version %d\n",
                    dh->magic, support_dex_version);
  always_assert_log(
      dh->file_size == dexsize,
      "Reported size in header (%zu) does not match file size (%u)\n",
      dexsize,
      dh->file_size);
  auto off = (uint64_t)dh->class_defs_off;
  auto limit = off + dh->class_defs_size * sizeof(dex_class_def);
  always_assert_log(off < dexsize, "class_defs_off out of range");
  always_assert_log(limit <= dexsize, "invalid class_defs_size");
}

void DexLoader::gather_input_stats(dex_stats_t* stats, const dex_header* dh) {
  if (!stats) {
    return;
  }
  stats->num_types += dh->type_ids_size;
  stats->num_classes += dh->class_defs_size;
  stats->num_method_refs += dh->method_ids_size;
  stats->num_field_refs += dh->field_ids_size;
  stats->num_strings += dh->string_ids_size;
  stats->num_protos += dh->proto_ids_size;
  stats->num_bytes += dh->file_size;
  // T58562665: TODO - actually update states for callsites/methodhandles
  stats->num_callsites += 0;
  stats->num_methodhandles += 0;

  std::unordered_set<DexEncodedValueArray, boost::hash<DexEncodedValueArray>>
      enc_arrays;
  std::set<DexTypeList*, dextypelists_comparator> type_lists;
  std::unordered_set<uint32_t> anno_offsets;

  for (uint32_t cidx = 0; cidx < dh->class_defs_size; ++cidx) {
    auto* clz = m_classes->at(cidx);
    if (clz == nullptr) {
      // Skip nulls, they may have been introduced by benign duplicate classes
      continue;
    }
    auto* class_def = &m_class_defs[cidx];
    auto anno_off = class_def->annotations_off;
    if (anno_off) {<commit_msg>Add more ids table size bound checks<commit_after>/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

#include "DexLoader.h"

#include "AggregateException.h"
#include "DexAccess.h"
#include "DexCallSite.h"
#include "DexDefs.h"
#include "DexMethodHandle.h"
#include "IRCode.h"
#include "Macros.h"
#include "Show.h"
#include "Trace.h"
#include "Walkers.h"
#include "WorkQueue.h"

#include <exception>
#include <stdexcept>
#include <vector>

DexLoader::DexLoader(const DexLocation* location)
    : m_idx(nullptr),
      m_file(new boost::iostreams::mapped_file()),
      m_location(location) {}

static void validate_dex_header(const dex_header* dh,
                                size_t dexsize,
                                int support_dex_version) {
  always_assert_log(sizeof(dex_header) <= dexsize,
                    "Header size (%lu) is larger than file size (%zu)\n",
                    dexsize,
                    sizeof(dex_header));
  bool supported = false;
  switch (support_dex_version) {
  case 38:
    supported = supported ||
                !memcmp(dh->magic, DEX_HEADER_DEXMAGIC_V38, sizeof(dh->magic));
    FALLTHROUGH_INTENDED; /* intentional fallthrough to also check for v37 */
  case 37:
    supported = supported ||
                !memcmp(dh->magic, DEX_HEADER_DEXMAGIC_V37, sizeof(dh->magic));
    FALLTHROUGH_INTENDED; /* intentional fallthrough to also check for v35 */
  case 35:
    supported = supported ||
                !memcmp(dh->magic, DEX_HEADER_DEXMAGIC_V35, sizeof(dh->magic));
    break;
  default:
    not_reached_log("Unrecognized support_dex_version %d\n",
                    support_dex_version);
  }
  always_assert_log(supported, "Bad dex magic %s for support_dex_version %d\n",
                    dh->magic, support_dex_version);
  always_assert_log(
      dh->file_size == dexsize,
      "Reported size in header (%zu) does not match file size (%u)\n",
      dexsize,
      dh->file_size);

  auto str_ids_off = (uint64_t)dh->string_ids_off;
  auto str_ids_limit =
      str_ids_off + dh->string_ids_size * sizeof(dex_string_id);
  always_assert_log(str_ids_off < dexsize, "string_ids_off out of range");
  always_assert_log(str_ids_limit <= dexsize, "invalid string_ids_size");

  auto type_ids_off = (uint64_t)dh->type_ids_off;
  auto type_ids_limit = type_ids_off + dh->type_ids_size * sizeof(dex_type_id);
  always_assert_log(type_ids_off < dexsize, "type_ids_off out of range");
  always_assert_log(type_ids_limit <= dexsize, "invalid type_ids_size");

  auto proto_ids_off = (uint64_t)dh->proto_ids_off;
  auto proto_ids_limit =
      proto_ids_off + dh->proto_ids_size * sizeof(dex_proto_id);
  always_assert_log(proto_ids_off < dexsize, "proto_ids_off out of range");
  always_assert_log(proto_ids_limit <= dexsize, "invalid proto_ids_size");

  auto field_ids_off = (uint64_t)dh->field_ids_off;
  auto field_ids_limit =
      field_ids_off + dh->field_ids_size * sizeof(dex_field_id);
  always_assert_log(field_ids_off < dexsize, "field_ids_off out of range");
  always_assert_log(field_ids_limit <= dexsize, "invalid field_ids_size");

  auto meth_ids_off = (uint64_t)dh->method_ids_off;
  auto meth_ids_limit =
      meth_ids_off + dh->method_ids_size * sizeof(dex_method_id);
  always_assert_log(meth_ids_off < dexsize, "method_ids_off out of range");
  always_assert_log(meth_ids_limit <= dexsize, "invalid method_ids_size");

  auto cls_defs_off = (uint64_t)dh->class_defs_off;
  auto cls_defs_limit =
      cls_defs_off + dh->class_defs_size * sizeof(dex_class_def);
  always_assert_log(cls_defs_off < dexsize, "class_defs_off out of range");
  always_assert_log(cls_defs_limit <= dexsize, "invalid class_defs_size");
}

void DexLoader::gather_input_stats(dex_stats_t* stats, const dex_header* dh) {
  if (!stats) {
    return;
  }
  stats->num_types += dh->type_ids_size;
  stats->num_classes += dh->class_defs_size;
  stats->num_method_refs += dh->method_ids_size;
  stats->num_field_refs += dh->field_ids_size;
  stats->num_strings += dh->string_ids_size;
  stats->num_protos += dh->proto_ids_size;
  stats->num_bytes += dh->file_size;
  // T58562665: TODO - actually update states for callsites/methodhandles
  stats->num_callsites += 0;
  stats->num_methodhandles += 0;

  std::unordered_set<DexEncodedValueArray, boost::hash<DexEncodedValueArray>>
      enc_arrays;
  std::set<DexTypeList*, dextypelists_comparator> type_lists;
  std::unordered_set<uint32_t> anno_offsets;

  for (uint32_t cidx = 0; cidx < dh->class_defs_size; ++cidx) {
    auto* clz = m_classes->at(cidx);
    if (clz == nullptr) {
      // Skip nulls, they may have been introduced by benign duplicate classes
      continue;
    }
    auto* class_def = &m_class_defs[cidx];
    auto anno_off = class_def->annotations_off;
    if (anno_off) {
{'fraction_non_alphanumeric': 0.114018921243396, 'fraction_numerical': 0.00933775648114019, 'mean_word_length': 3.856801909307876, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4189773', 'n_tokens_mistral': 2972, 'n_tokens_neox': 2806, 'n_words': 595}