<commit_before>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include <vespa/log/log.h>
LOG_SETUP("dense_dot_product_function_test");

#include <vespa/vespalib/testkit/test_kit.h>
#include <vespa/eval/eval/tensor_function.h>
#include <vespa/eval/eval/operation.h>
#include <vespa/eval/eval/simple_tensor.h>
#include <vespa/eval/eval/simple_tensor_engine.h>
#include <vespa/eval/tensor/default_tensor_engine.h>
#include <vespa/eval/tensor/dense/dense_xw_product_function.h>
#include <vespa/eval/tensor/dense/dense_tensor.h>
#include <vespa/eval/tensor/dense/dense_tensor_builder.h>
#include <vespa/eval/tensor/dense/dense_tensor_view.h>

#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/util/stash.h>

using namespace vespalib;
using namespace vespalib::eval;
using namespace vespalib::tensor;
using namespace vespalib::eval::tensor_function;

const TensorEngine &ref_engine = SimpleTensorEngine::ref();
const TensorEngine &prod_engine = DefaultTensorEngine::ref();

void verify_equal(const Value &expect, const Value &value) {
    const eval::Tensor *tensor = value.as_tensor();
    ASSERT_TRUE(tensor != nullptr);
    const eval::Tensor *expect_tensor = expect.as_tensor();
    ASSERT_TRUE(expect_tensor != nullptr);
    auto expect_spec = expect_tensor->engine().to_spec(expect);
    auto value_spec = tensor->engine().to_spec(value);
    EXPECT_EQUAL(expect_spec, value_spec);
}

SimpleObjectParams wrap(std::vector<eval::Value::CREF> params) {
    return SimpleObjectParams(params);
}

void verify_result(const TensorSpec &v, const TensorSpec &m, bool happy) {
    Stash stash;
    Value::UP ref_vec = ref_engine.from_spec(v);
    Value::UP ref_mat = ref_engine.from_spec(m);
    const Value &joined = ref_engine.join(*ref_vec, *ref_mat, operation::Mul::f, stash);
    const Value &expect = ref_engine.reduce(joined, Aggr::SUM, {"x"}, stash);

    Value::UP prod_vec = prod_engine.from_spec(v);
    Value::UP prod_mat = prod_engine.from_spec(m);

    Inject vec_first(prod_vec->type(), 0);
    Inject mat_last(prod_mat->type(), 1);

    DenseXWProductFunction fun1(expect.type(), vec_first, mat_last,
                                prod_vec->type().dimensions()[0].size,
                                expect.type().dimensions()[0].size,
                                happy);
    const Value &actual1 = fun1.eval(prod_engine, wrap({*prod_vec, *prod_mat}), stash);
    TEST_DO(verify_equal(expect, actual1));

    Inject vec_last(prod_vec->type(), 1);
    Inject mat_first(prod_mat->type(), 0);

    DenseXWProductFunction fun2(expect.type(), vec_last, mat_first,
                                prod_vec->type().dimensions()[0].size,
                                expect.type().dimensions()[0].size,
                                happy);
    const Value &actual2 = fun2.eval(prod_engine, wrap({*prod_mat, *prod_vec}), stash);
    TEST_DO(verify_equal(expect, actual2));
}<commit_msg>use interpreted function to evaluate tensor function<commit_after>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include <vespa/log/log.h>
LOG_SETUP("dense_dot_product_function_test");

#include <vespa/vespalib/testkit/test_kit.h>
#include <vespa/eval/eval/tensor_function.h>
#include <vespa/eval/eval/operation.h>
#include <vespa/eval/eval/simple_tensor.h>
#include <vespa/eval/eval/simple_tensor_engine.h>
#include <vespa/eval/tensor/default_tensor_engine.h>
#include <vespa/eval/tensor/dense/dense_xw_product_function.h>
#include <vespa/eval/tensor/dense/dense_tensor.h>
#include <vespa/eval/tensor/dense/dense_tensor_builder.h>
#include <vespa/eval/tensor/dense/dense_tensor_view.h>

#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/util/stash.h>

using namespace vespalib;
using namespace vespalib::eval;
using namespace vespalib::tensor;
using namespace vespalib::eval::tensor_function;

const TensorEngine &ref_engine = SimpleTensorEngine::ref();
const TensorEngine &prod_engine = DefaultTensorEngine::ref();

void verify_equal(const Value &expect, const Value &value) {
    const eval::Tensor *tensor = value.as_tensor();
    ASSERT_TRUE(tensor != nullptr);
    const eval::Tensor *expect_tensor = expect.as_tensor();
    ASSERT_TRUE(expect_tensor != nullptr);
    auto expect_spec = expect_tensor->engine().to_spec(expect);
    auto value_spec = tensor->engine().to_spec(value);
    EXPECT_EQUAL(expect_spec, value_spec);
}

SimpleObjectParams wrap(std::vector<eval::Value::CREF> params) {
    return SimpleObjectParams(params);
}

void verify_result(const TensorSpec &v, const TensorSpec &m, bool happy) {
    Stash stash;
    Value::UP ref_vec = ref_engine.from_spec(v);
    Value::UP ref_mat = ref_engine.from_spec(m);
    const Value &joined = ref_engine.join(*ref_vec, *ref_mat, operation::Mul::f, stash);
    const Value &expect = ref_engine.reduce(joined, Aggr::SUM, {"x"}, stash);

    Value::UP prod_vec = prod_engine.from_spec(v);
    Value::UP prod_mat = prod_engine.from_spec(m);

    Inject vec_first(prod_vec->type(), 0);
    Inject mat_last(prod_mat->type(), 1);

    DenseXWProductFunction fun1(expect.type(), vec_first, mat_last,
                                prod_vec->type().dimensions()[0].size,
                                expect.type().dimensions()[0].size,
                                happy);
    InterpretedFunction ifun1(prod_engine, fun1);
    InterpretedFunction::Context ictx1(ifun1);
    const Value &actual1 = ifun1.eval(ictx1, wrap({*prod_vec, *prod_mat}));
    TEST_DO(verify_equal(expect, actual1));

    Inject vec_last(prod_vec->type(), 1);
    Inject mat_first(prod_mat->type(), 0);

    DenseXWProductFunction fun2(expect.type(), vec_last, mat_first,
                                prod_vec->type().dimensions()[0].size,
                                expect.type().dimensions()[0].size,
                                happy);
    InterpretedFunction ifun2(prod_engine, fun2);
    InterpretedFunction::Context ictx2(ifun2);
    const Value &actual2 = ifun2.eval(ictx2, wrap({*prod_mat, *prod_vec}));
    TEST_DO(verify_equal(expect, actual2));
}
{'fraction_non_alphanumeric': 0.13354890864995958, 'fraction_numerical': 0.008730800323362976, 'mean_word_length': 4.0333604556550045, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1223053', 'n_tokens_mistral': 2065, 'n_tokens_neox': 1995, 'n_words': 374}