<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <<EMAIL>>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include "stx/inspect.h"
#include "zbase/JavaScriptContext.h"
#include "js/Conversions.h"

using namespace stx;

namespace zbase {

static JSClass global_class = { "global", JSCLASS_GLOBAL_FLAGS };

static bool write_json_to_buf(const char16_t* str, uint32_t strlen, void* out) {
  *static_cast<String*>(out) += StringUtil::convertUTF16To8(
      std::basic_string<char16_t>(str, strlen));

  return true;
}

JavaScriptContext::JavaScriptContext() {
  runtime_ = JS_NewRuntime(8 * 1024 * 1024);
  if (!runtime_) {
    RAISE(kRuntimeError, "error while initializing JavaScript runtime");
  }

  ctx_ = JS_NewContext(runtime_, 8192);
  if (!ctx_) {
    RAISE(kRuntimeError, "error while initializing JavaScript context");
  }

  JSAutoRequest js_req(ctx_);

  global_ = JS_NewGlobalObject(
      ctx_,
      &global_class,
      nullptr,
      JS::FireOnNewGlobalHook);

  if (!global_) {
    RAISE(kRuntimeError, "error while initializing JavaScript context");
  }

  {
    JSAutoCompartment ac(ctx_, global_);
    JS_InitStandardClasses(ctx_, global_);
  }
}

JavaScriptContext::~JavaScriptContext() {
  JS_DestroyContext(ctx_);
  JS_DestroyRuntime(runtime_);
}

void JavaScriptContext::loadProgram(const String& program) {
  JSAutoRequest js_req(ctx_);
  JSAutoCompartment ac(ctx_, global_);

  JS::RootedValue rval(ctx_);

  JS::CompileOptions opts(ctx_);
  opts.setFileAndLine("<mapreduce>", 1);

  if (!JS::Evaluate(
        ctx_,
        global_,
        opts,
        program.c_str(),
        program.size(),
        &rval)) {
    RAISE(kRuntimeError, "JavaScript execution failed");
  }
}

void JavaScriptContext::callMapFunction(
    const String& method_name,
    const String& json_string,
    Vector<Pair<String, String>>* tuples) {
  auto json_wstring = StringUtil::convertUTF8To16(json_string);

  JSAutoRequest js_req(ctx_);
  JSAutoCompartment js_comp(ctx_, global_);

  JS::RootedValue json(ctx_);
  if (!JS_ParseJSON(ctx_, json_wstring.c_str(), json_wstring.size(), &json)) {
    RAISE(kRuntimeError, "invalid JSON");
  }

  JS::AutoValueArray<1> argv(ctx_);
  argv[0].set(json);

  JS::RootedValue rval(ctx_);
  if (!JS_CallFunctionName(ctx_, global_, method_name.c_str(), argv, &rval)) {
    RAISE(kRuntimeError, "map function failed");
  }

  enumerateTuples(&rval, tuples);
}

void JavaScriptContext::callReduceFunction(
    const String& method_name,
    const String& key,
    const Vector<String>& values,
    Vector<Pair<String, String>>* tuples) {
  JSAutoRequest js_req(ctx_);
  JSAutoCompartment js_comp(ctx_, global_);

  auto val_array_ptr = JS_NewArrayObject(ctx_, values.size());
  if (!val_array_ptr) {
    RAISE(kRuntimeError, "reduce function execution error");
  }

  JS::RootedObject val_array(ctx_, val_array_ptr);
  for (size_t i = 0; i < values.size(); ++i) {
    auto val_str_ptr = JS_NewStringCopyN(
        ctx_,
        values[i].data(),
        values[i].size());
    if (!val_str_ptr) {
      RAISE(kRuntimeError, "reduce function execution error");
    }

    JS::RootedString val_str(ctx_, val_str_ptr);
    if (!JS_SetElement(ctx_, val_array, i, val_str)) {
      RAISE(kRuntimeError, "reduce function execution error");
    }
  }

  auto key_str_ptr = JS_NewStringCopyN(ctx_, key.data(), key.size());
  if (!key_str_ptr) {
    RAISE(kRuntimeError, "reduce function execution error");
  }

  JS::AutoValueArray<2> argv(ctx_);
  argv[0].setString(key_str_ptr);
  argv[1].setObject(*val_array);

  JS::RootedValue rval(ctx_);
  if (!JS_CallFunctionName(ctx_, global_, method_name.c_str(), argv, &rval)) {
    RAISE(kRuntimeError, "reduce function failed");
  }

  enumerateTuples(&rval, tuples);
}

void JavaScriptContext::enumerateTuples(
    JS::RootedValue* src,
    Vector<Pair<String, String>>* dst) const {
  if (!src->isObject()) {
    RAISE(kRuntimeError, "reduce function must return a list/array of tuples");
  }

  JS::RootedObject list(ctx_, &src->toObject());
  JS::AutoIdArray list_enum(ctx_, JS_Enumerate(ctx_, list));
  for (size_t i = 0; i < list_enum.length(); ++i) {
    JS::RootedValue elem(ctx_);
    JS::RootedValue elem_key(ctx_);
    JS::RootedValue elem_value(ctx_);
    JS::Rooted<jsid> elem_id(ctx_, list_enum[i]);<commit_msg>better error messages<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <<EMAIL>>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include "stx/inspect.h"
#include "zbase/JavaScriptContext.h"
#include "js/Conversions.h"

using namespace stx;

namespace zbase {

static JSClass global_class = { "global", JSCLASS_GLOBAL_FLAGS };

static bool write_json_to_buf(const char16_t* str, uint32_t strlen, void* out) {
  *static_cast<String*>(out) += StringUtil::convertUTF16To8(
      std::basic_string<char16_t>(str, strlen));

  return true;
}

JavaScriptContext::JavaScriptContext() {
  runtime_ = JS_NewRuntime(8 * 1024 * 1024);
  if (!runtime_) {
    RAISE(kRuntimeError, "error while initializing JavaScript runtime");
  }

  ctx_ = JS_NewContext(runtime_, 8192);
  if (!ctx_) {
    RAISE(kRuntimeError, "error while initializing JavaScript context");
  }

  JSAutoRequest js_req(ctx_);

  global_ = JS_NewGlobalObject(
      ctx_,
      &global_class,
      nullptr,
      JS::FireOnNewGlobalHook);

  if (!global_) {
    RAISE(kRuntimeError, "error while initializing JavaScript context");
  }

  {
    JSAutoCompartment ac(ctx_, global_);
    JS_InitStandardClasses(ctx_, global_);
  }
}

JavaScriptContext::~JavaScriptContext() {
  JS_DestroyContext(ctx_);
  JS_DestroyRuntime(runtime_);
}

void JavaScriptContext::loadProgram(const String& program) {
  JSAutoRequest js_req(ctx_);
  JSAutoCompartment ac(ctx_, global_);

  JS::RootedValue rval(ctx_);

  JS::CompileOptions opts(ctx_);
  opts.setFileAndLine("<mapreduce>", 1);

  if (!JS::Evaluate(
        ctx_,
        global_,
        opts,
        program.c_str(),
        program.size(),
        &rval)) {
    RAISE(kRuntimeError, "JavaScript execution failed");
  }
}

void JavaScriptContext::callMapFunction(
    const String& method_name,
    const String& json_string,
    Vector<Pair<String, String>>* tuples) {
  auto json_wstring = StringUtil::convertUTF8To16(json_string);

  JSAutoRequest js_req(ctx_);
  JSAutoCompartment js_comp(ctx_, global_);

  JS::RootedValue json(ctx_);
  if (!JS_ParseJSON(ctx_, json_wstring.c_str(), json_wstring.size(), &json)) {
    RAISE(kRuntimeError, "invalid JSON");
  }

  JS::AutoValueArray<1> argv(ctx_);
  argv[0].set(json);

  JS::RootedValue rval(ctx_);
  if (!JS_CallFunctionName(ctx_, global_, method_name.c_str(), argv, &rval)) {
    RAISE(kRuntimeError, "map function failed");
  }

  enumerateTuples(&rval, tuples);
}

void JavaScriptContext::callReduceFunction(
    const String& method_name,
    const String& key,
    const Vector<String>& values,
    Vector<Pair<String, String>>* tuples) {
  JSAutoRequest js_req(ctx_);
  JSAutoCompartment js_comp(ctx_, global_);

  auto val_array_ptr = JS_NewArrayObject(ctx_, values.size());
  if (!val_array_ptr) {
    RAISE(kRuntimeError, "reduce function execution error: out of memory");
  }

  JS::RootedObject val_array(ctx_, val_array_ptr);
  for (size_t i = 0; i < values.size(); ++i) {
    auto val_str_ptr = JS_NewStringCopyN(
        ctx_,
        values[i].data(),
        values[i].size());
    if (!val_str_ptr) {
      RAISE(kRuntimeError, "reduce function execution error: out of memory");
    }

    JS::RootedString val_str(ctx_, val_str_ptr);
    if (!JS_SetElement(ctx_, val_array, i, val_str)) {
      RAISE(kRuntimeError, "reduce function execution error: out of memory");
    }
  }

  auto key_str_ptr = JS_NewStringCopyN(ctx_, key.data(), key.size());
  if (!key_str_ptr) {
    RAISE(kRuntimeError, "reduce function execution error: out of memory");
  }

  JS::AutoValueArray<2> argv(ctx_);
  argv[0].setString(key_str_ptr);
  argv[1].setObject(*val_array);

  JS::RootedValue rval(ctx_);
  if (!JS_CallFunctionName(ctx_, global_, method_name.c_str(), argv, &rval)) {
    RAISE(kRuntimeError, "reduce function failed");
  }

  enumerateTuples(&rval, tuples);
}

void JavaScriptContext::enumerateTuples(
    JS::RootedValue* src,
    Vector<Pair<String, String>>* dst) const {
  if (!src->isObject()) {
    RAISE(kRuntimeError, "reduce function must return a list/array of tuples");
  }

  JS::RootedObject list(ctx_, &src->toObject());
  JS::AutoIdArray list_enum(ctx_, JS_Enumerate(ctx_, list));
  for (size_t i = 0; i < list_enum.length(); ++i) {
    JS::RootedValue elem(ctx_);
    JS::RootedValue elem_key(ctx_);
    JS::RootedValue elem_value(ctx_);
    JS::Rooted<jsid> elem_id(ctx_, list_enum[i]);
{'fraction_non_alphanumeric': 0.1333775713337757, 'fraction_numerical': 0.0081840300818403, 'mean_word_length': 4.457453228726615, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6863502', 'n_tokens_mistral': 3120, 'n_tokens_neox': 3000, 'n_words': 663}