<commit_before>// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "exprs/scalar-fn-call.h"

#include <vector>
#include <llvm/IR/Attributes.h>
#include <llvm/ExecutionEngine/ExecutionEngine.h>
#include "codegen/codegen-anyval.h"
#include "codegen/llvm-codegen.h"
#include "exprs/anyval-util.h"
#include "exprs/expr-context.h"
#include "runtime/hdfs-fs-cache.h"
#include "runtime/lib-cache.h"
#include "runtime/runtime-state.h"
#include "runtime/types.h"
#include "udf/udf-internal.h"
#include "util/debug-util.h"
#include "util/dynamic-util.h"
#include "util/symbols-util.h"

using namespace impala;
using namespace impala_udf;
using namespace std;

ScalarFnCall::ScalarFnCall(const TExprNode& node)
  : Expr(node),
    vararg_start_idx_(node.__isset.vararg_start_idx ?
        node.vararg_start_idx : -1),
    scalar_fn_wrapper_(NULL),
    prepare_fn_(NULL),
    close_fn_(NULL),
    scalar_fn_(NULL) {
  DCHECK_NE(fn_.binary_type, TFunctionBinaryType::HIVE);
}

Status ScalarFnCall::Prepare(RuntimeState* state, const RowDescriptor& desc,
                             ExprContext* context) {
  RETURN_IF_ERROR(Expr::Prepare(state, desc, context));

  if (fn_.scalar_fn.symbol.empty()) {
    // This path is intended to only be used during development to test FE
    // code before the BE has implemented the function.
    // Having the failure in the BE (rather than during analysis) allows for
    // better FE testing.
    DCHECK_EQ(fn_.binary_type, TFunctionBinaryType::BUILTIN);
    stringstream ss;
    ss << "Function " << fn_.name.function_name << " is not implemented.";
    return Status(ss.str());
  }

  FunctionContext::TypeDesc return_type = AnyValUtil::ColumnTypeToTypeDesc(type_);
  vector<FunctionContext::TypeDesc> arg_types;
  bool char_arg = false;
  for (int i = 0; i < children_.size(); ++i) {
    arg_types.push_back(AnyValUtil::ColumnTypeToTypeDesc(children_[i]->type_));
    char_arg = char_arg || (children_[i]->type_.type == TYPE_CHAR);
  }

  // Compute buffer size for varargs
  int varargs_buffer_size = 0;
  if (vararg_start_idx_ != -1) {
    DCHECK_GT(GetNumChildren(), vararg_start_idx_);
    for (int i = vararg_start_idx_; i < GetNumChildren(); ++i) {
      varargs_buffer_size += AnyValUtil::AnyValSize(children_[i]->type());
    }
  }

  context_index_ = context->Register(state, return_type, arg_types, varargs_buffer_size);

  // If the codegen object hasn't been created yet and we're calling a builtin with <= 3
  // non-variadic arguments, we can use the interpreted path and call the builtin without
  // codegen. This saves us the overhead of creating the codegen object when it's not
  // necessary (i.e., in plan fragments with no codegen-enabled operators).
  // TODO: no need to restrict this to builtins.
  // TODO: codegen for char arguments
  if (char_arg || (!state->codegen_created() && NumFixedArgs() <= 3
          && fn_.binary_type == TFunctionBinaryType::BUILTIN)) {
    DCHECK(NumFixedArgs() <= 3 && fn_.binary_type == TFunctionBinaryType::BUILTIN);
    Status status = LibCache::instance()->GetSoFunctionPtr(
        fn_.hdfs_location, fn_.scalar_fn.symbol, &scalar_fn_, &cache_entry_);
    if (!status.ok()) {
      // Builtins symbols should exist unless there is a version mismatch.
      stringstream ss;
      ss << "Builtin '" << fn_.name.function_name << "' with symbol '"
         << fn_.scalar_fn.symbol << "' does not exist. "
         << "Verify that all your impalads are the same version.";
      status.AddErrorMsg(ss.str());
      return status;
    }
  } else {
    // If we got here, either codegen is enabled or we need codegen to run this function.
    LlvmCodeGen* codegen;
    RETURN_IF_ERROR(state->GetCodegen(&codegen));

    if (fn_.binary_type == TFunctionBinaryType::IR) {
      string local_path;
      RETURN_IF_ERROR(LibCache::instance()->GetLocalLibPath(
          fn_.hdfs_location, LibCache::TYPE_IR, &local_path));
      // Link the UDF module into this query's main module (essentially copy the UDF
      // module into the main module) so the UDF's functions are available in the main
      // module.
      RETURN_IF_ERROR(codegen->LinkModule(local_path));
    }

    llvm::Function* ir_udf_wrapper;
    RETURN_IF_ERROR(GetCodegendComputeFn(state, &ir_udf_wrapper));
    // TODO: don't do this for child exprs
    codegen->AddFunctionToJit(ir_udf_wrapper, &scalar_fn_wrapper_);
  }

  if (fn_.scalar_fn.__isset.prepare_fn_symbol) {
    RETURN_IF_ERROR(GetFunction(state, fn_.scalar_fn.prepare_fn_symbol,
        reinterpret_cast<void**>(&prepare_fn_)));
  }
  if (fn_.scalar_fn.__isset.close_fn_symbol) {
    RETURN_IF_ERROR(GetFunction(state, fn_.scalar_fn.close_fn_symbol,
        reinterpret_cast<void**>(&close_fn_)));
  }<commit_msg>IMPALA-1589: allow non-codegen'd native UDFs<commit_after>// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "exprs/scalar-fn-call.h"

#include <vector>
#include <gutil/strings/substitute.h>
#include <llvm/IR/Attributes.h>
#include <llvm/ExecutionEngine/ExecutionEngine.h>

#include "codegen/codegen-anyval.h"
#include "codegen/llvm-codegen.h"
#include "exprs/anyval-util.h"
#include "exprs/expr-context.h"
#include "runtime/hdfs-fs-cache.h"
#include "runtime/lib-cache.h"
#include "runtime/runtime-state.h"
#include "runtime/types.h"
#include "udf/udf-internal.h"
#include "util/debug-util.h"
#include "util/dynamic-util.h"
#include "util/symbols-util.h"

using namespace impala;
using namespace impala_udf;
using namespace std;
using namespace strings;

ScalarFnCall::ScalarFnCall(const TExprNode& node)
  : Expr(node),
    vararg_start_idx_(node.__isset.vararg_start_idx ?
        node.vararg_start_idx : -1),
    scalar_fn_wrapper_(NULL),
    prepare_fn_(NULL),
    close_fn_(NULL),
    scalar_fn_(NULL) {
  DCHECK_NE(fn_.binary_type, TFunctionBinaryType::HIVE);
}

Status ScalarFnCall::Prepare(RuntimeState* state, const RowDescriptor& desc,
                             ExprContext* context) {
  RETURN_IF_ERROR(Expr::Prepare(state, desc, context));

  if (fn_.scalar_fn.symbol.empty()) {
    // This path is intended to only be used during development to test FE
    // code before the BE has implemented the function.
    // Having the failure in the BE (rather than during analysis) allows for
    // better FE testing.
    DCHECK_EQ(fn_.binary_type, TFunctionBinaryType::BUILTIN);
    stringstream ss;
    ss << "Function " << fn_.name.function_name << " is not implemented.";
    return Status(ss.str());
  }

  FunctionContext::TypeDesc return_type = AnyValUtil::ColumnTypeToTypeDesc(type_);
  vector<FunctionContext::TypeDesc> arg_types;
  bool char_arg = false;
  for (int i = 0; i < children_.size(); ++i) {
    arg_types.push_back(AnyValUtil::ColumnTypeToTypeDesc(children_[i]->type_));
    char_arg = char_arg || (children_[i]->type_.type == TYPE_CHAR);
  }

  // Compute buffer size for varargs
  int varargs_buffer_size = 0;
  if (vararg_start_idx_ != -1) {
    DCHECK_GT(GetNumChildren(), vararg_start_idx_);
    for (int i = vararg_start_idx_; i < GetNumChildren(); ++i) {
      varargs_buffer_size += AnyValUtil::AnyValSize(children_[i]->type());
    }
  }

  context_index_ = context->Register(state, return_type, arg_types, varargs_buffer_size);

  // If the codegen object hasn't been created yet and we're calling a builtin or native
  // UDF with <= 3 non-variadic arguments, we can use the interpreted path and call the
  // builtin without codegen. This saves us the overhead of creating the codegen object
  // when it's not necessary (i.e., in plan fragments with no codegen-enabled operators).
  // TODO: codegen for char arguments
  if (char_arg || (!state->codegen_created() && NumFixedArgs() <= 3 &&
                   (fn_.binary_type == TFunctionBinaryType::BUILTIN ||
                    fn_.binary_type == TFunctionBinaryType::NATIVE))) {
    if (char_arg) {
      DCHECK(NumFixedArgs() <= 3 && fn_.binary_type == TFunctionBinaryType::BUILTIN);
    }
    Status status = LibCache::instance()->GetSoFunctionPtr(
        fn_.hdfs_location, fn_.scalar_fn.symbol, &scalar_fn_, &cache_entry_);
    if (!status.ok()) {
      if (fn_.binary_type == TFunctionBinaryType::BUILTIN) {
        // Builtins symbols should exist unless there is a version mismatch.
        status.AddErrorMsg(Substitute(
            "Builtin '$0' with symbol '$1' does not exist. Verify that all your impalads "
            "are the same version.", fn_.name.function_name, fn_.scalar_fn.symbol));
        return status;
      } else {
        DCHECK_EQ(fn_.binary_type, TFunctionBinaryType::NATIVE);
        return Status(Substitute("Problem loading UDF '$0':\n$1",
            fn_.name.function_name, status.GetErrorMsg()));
        return status;
      }
    }
  } else {
    // If we got here, either codegen is enabled or we need codegen to run this function.
    LlvmCodeGen* codegen;
    RETURN_IF_ERROR(state->GetCodegen(&codegen));

    if (fn_.binary_type == TFunctionBinaryType::IR) {
      string local_path;
      RETURN_IF_ERROR(LibCache::instance()->GetLocalLibPath(
          fn_.hdfs_location, LibCache::TYPE_IR, &local_path));
      // Link the UDF module into this query's main module (essentially copy the UDF
      // module into the main module) so the UDF's functions are available in the main
      // module.
      RETURN_IF_ERROR(codegen->LinkModule(local_path));
    }

    llvm::Function* ir_udf_wrapper;
    RETURN_IF_ERROR(GetCodegendComputeFn(state, &ir_udf_wrapper));
    // TODO: don't do this for child exprs
    codegen->AddFunctionToJit(ir_udf_wrapper, &scalar_fn_wrapper_);
  }

  if (fn_.scalar_fn.__isset.prepare_fn_symbol) {
    RETURN_IF_ERROR(GetFunction(state, fn_.scalar_fn.prepare_fn_symbol,
        reinterpret_cast<void**>(&prepare_fn_)));
  }
  if (fn_.scalar_fn.__isset.close_fn_symbol) {
    RETURN_IF_ERROR(GetFunction(state, fn_.scalar_fn.close_fn_symbol,
        reinterpret_cast<void**>(&close_fn_)));
  }
{'fraction_non_alphanumeric': 0.11847133757961784, 'fraction_numerical': 0.003457688808007279, 'mean_word_length': 3.7641959254443, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4600913', 'n_tokens_mistral': 3476, 'n_tokens_neox': 3417, 'n_words': 908}