<commit_before>/**
 * Copyright (c) 2016-present, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 */

#include "InterDex.h"

#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <vector>
#include <unordered_set>

#include "Creators.h"
#include "Debug.h"
#include "DexClass.h"
#include "DexLoader.h"
#include "DexOutput.h"
#include "DexUtil.h"
#include "ConfigFiles.h"
#include "ReachableClasses.h"
#include "Transform.h"
#include "walkers.h"

namespace {

typedef std::unordered_set<DexMethod*> mrefs_t;
typedef std::unordered_set<DexField*> frefs_t;

size_t global_dmeth_cnt;
size_t global_smeth_cnt;
size_t global_vmeth_cnt;
size_t global_methref_cnt;
size_t global_fieldref_cnt;
size_t global_cls_cnt;
size_t cls_skipped_in_primary = 0;
size_t cls_skipped_in_secondary = 0;

static void gather_mrefs(DexClass* cls, mrefs_t& mrefs, frefs_t& frefs) {
  std::vector<DexMethod*> method_refs;
  std::vector<DexField*> field_refs;
  cls->gather_methods(method_refs);
  cls->gather_fields(field_refs);
  mrefs.insert(method_refs.begin(), method_refs.end());
  frefs.insert(field_refs.begin(), field_refs.end());
}

#ifdef GINGER_BREAD
static const int kMaxLinearAlloc = (2600 * 1024);
#else
static const int kMaxLinearAlloc = (11600 * 1024);
#endif
static const int kMaxMethodRefs = ((64 * 1024) - 1);
static const int kMaxFieldRefs = 64 * 1024 - 1;
static const char* kCanaryPrefix = "Lsecondary/dex";
static const char* kCanaryClassFormat = "Lsecondary/dex%02d/Canary;";
static const int kCanaryClassBufsize = strlen(kCanaryClassFormat) + 1;
static const int kMaxDexNum = 99;

struct dex_emit_tracker {
  int la_size;
  mrefs_t mrefs;
  frefs_t frefs;
  std::vector<DexClass*> outs;
  std::unordered_set<DexClass*> emitted;
  std::unordered_map<std::string, DexClass*> clookup;
};

static void update_dex_stats (
    size_t cls_cnt,
    size_t methrefs_cnt,
    size_t frefs_cnt) {
  global_cls_cnt += cls_cnt;
  global_methref_cnt += methrefs_cnt;
  global_fieldref_cnt += frefs_cnt;
}

static void update_class_stats(DexClass* clazz) {
  int cnt_smeths = 0;
  for (auto const m : clazz->get_dmethods()) {
    if (is_static(m)) {
      cnt_smeths++;
    }
  }
  global_smeth_cnt += cnt_smeths;
  global_dmeth_cnt += clazz->get_dmethods().size();
  global_vmeth_cnt += clazz->get_vmethods().size();
}

static void flush_out_dex(
    dex_emit_tracker& det,
    DexClassesVector& outdex,
    size_t mrefs_size,
    size_t frefs_size) {
  DexClasses dc(det.outs.size());
  for (size_t i = 0; i < det.outs.size(); i++) {
    dc.insert_at(det.outs[i], i);
  }
  outdex.emplace_back(std::move(dc));
  // print out stats
  TRACE(IDEX, 1,
    "terminating dex at classes %lu, lin alloc %d:%d, mrefs %lu:%d, frefs "
    "%lu:%d\n",
    det.outs.size(),
    det.la_size,
    kMaxLinearAlloc,
    mrefs_size,
    kMaxMethodRefs,
    frefs_size,
    kMaxFieldRefs);

  update_dex_stats(det.outs.size(), mrefs_size, frefs_size);

  det.la_size = 0;
  det.mrefs.clear();
  det.frefs.clear();
  det.outs.clear();
}

static void flush_out_dex(
    dex_emit_tracker& det,
    DexClassesVector& outdex) {
  flush_out_dex(det, outdex, det.mrefs.size(), det.frefs.size());
}

static void flush_out_secondary(
    dex_emit_tracker &det,
    DexClassesVector &outdex,
    size_t mrefs_size,
    size_t frefs_size) {
  /* Find the Canary class and add it in. */
  int dexnum = ((int)outdex.size());
  char buf[kCanaryClassBufsize];
  always_assert_log(dexnum <= kMaxDexNum,
                    "Bailing, Max dex number surpassed %d\n", dexnum);<commit_msg>don't emit empty dexes<commit_after>/**
 * Copyright (c) 2016-present, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 */

#include "InterDex.h"

#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <vector>
#include <unordered_set>

#include "Creators.h"
#include "Debug.h"
#include "DexClass.h"
#include "DexLoader.h"
#include "DexOutput.h"
#include "DexUtil.h"
#include "ConfigFiles.h"
#include "ReachableClasses.h"
#include "Transform.h"
#include "walkers.h"

namespace {

typedef std::unordered_set<DexMethod*> mrefs_t;
typedef std::unordered_set<DexField*> frefs_t;

size_t global_dmeth_cnt;
size_t global_smeth_cnt;
size_t global_vmeth_cnt;
size_t global_methref_cnt;
size_t global_fieldref_cnt;
size_t global_cls_cnt;
size_t cls_skipped_in_primary = 0;
size_t cls_skipped_in_secondary = 0;

static void gather_mrefs(DexClass* cls, mrefs_t& mrefs, frefs_t& frefs) {
  std::vector<DexMethod*> method_refs;
  std::vector<DexField*> field_refs;
  cls->gather_methods(method_refs);
  cls->gather_fields(field_refs);
  mrefs.insert(method_refs.begin(), method_refs.end());
  frefs.insert(field_refs.begin(), field_refs.end());
}

#ifdef GINGER_BREAD
static const int kMaxLinearAlloc = (2600 * 1024);
#else
static const int kMaxLinearAlloc = (11600 * 1024);
#endif
static const int kMaxMethodRefs = ((64 * 1024) - 1);
static const int kMaxFieldRefs = 64 * 1024 - 1;
static const char* kCanaryPrefix = "Lsecondary/dex";
static const char* kCanaryClassFormat = "Lsecondary/dex%02d/Canary;";
static const int kCanaryClassBufsize = strlen(kCanaryClassFormat) + 1;
static const int kMaxDexNum = 99;

struct dex_emit_tracker {
  int la_size;
  mrefs_t mrefs;
  frefs_t frefs;
  std::vector<DexClass*> outs;
  std::unordered_set<DexClass*> emitted;
  std::unordered_map<std::string, DexClass*> clookup;
};

static void update_dex_stats (
    size_t cls_cnt,
    size_t methrefs_cnt,
    size_t frefs_cnt) {
  global_cls_cnt += cls_cnt;
  global_methref_cnt += methrefs_cnt;
  global_fieldref_cnt += frefs_cnt;
}

static void update_class_stats(DexClass* clazz) {
  int cnt_smeths = 0;
  for (auto const m : clazz->get_dmethods()) {
    if (is_static(m)) {
      cnt_smeths++;
    }
  }
  global_smeth_cnt += cnt_smeths;
  global_dmeth_cnt += clazz->get_dmethods().size();
  global_vmeth_cnt += clazz->get_vmethods().size();
}

static void flush_out_dex(
    dex_emit_tracker& det,
    DexClassesVector& outdex,
    size_t mrefs_size,
    size_t frefs_size) {
  DexClasses dc(det.outs.size());
  for (size_t i = 0; i < det.outs.size(); i++) {
    dc.insert_at(det.outs[i], i);
  }
  outdex.emplace_back(std::move(dc));
  // print out stats
  TRACE(IDEX, 1,
    "terminating dex at classes %lu, lin alloc %d:%d, mrefs %lu:%d, frefs "
    "%lu:%d\n",
    det.outs.size(),
    det.la_size,
    kMaxLinearAlloc,
    mrefs_size,
    kMaxMethodRefs,
    frefs_size,
    kMaxFieldRefs);

  update_dex_stats(det.outs.size(), mrefs_size, frefs_size);

  det.la_size = 0;
  det.mrefs.clear();
  det.frefs.clear();
  det.outs.clear();
}

static void flush_out_dex(
    dex_emit_tracker& det,
    DexClassesVector& outdex) {
  flush_out_dex(det, outdex, det.mrefs.size(), det.frefs.size());
}

static void flush_out_secondary(
    dex_emit_tracker &det,
    DexClassesVector &outdex,
    size_t mrefs_size,
    size_t frefs_size) {
  // don't emit dex if we don't have any classes
  if (!det.outs.size()) {
    return;
  }
  /* Find the Canary class and add it in. */
  int dexnum = ((int)outdex.size());
  char buf[kCanaryClassBufsize];
  always_assert_log(dexnum <= kMaxDexNum,
                    "Bailing, Max dex number surpassed %d\n", dexnum);
{'fraction_non_alphanumeric': 0.12405030128373068, 'fraction_numerical': 0.012051349227141734, 'mean_word_length': 4.496760259179266, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3679436', 'n_tokens_mistral': 3077, 'n_tokens_neox': 2879, 'n_words': 662}