<commit_before>// Copyright 2017 <NAME>. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "default_tensor_engine.h"
#include "tensor.h"
#include "wrapped_simple_value.h"
#include "serialization/typed_binary_format.h"
#include "sparse/sparse_tensor_address_builder.h"
#include "sparse/direct_sparse_tensor_builder.h"
#include "dense/dense_tensor.h"
#include "dense/typed_dense_tensor_builder.h"
#include "dense/dense_dot_product_function.h"
#include "dense/dense_xw_product_function.h"
#include "dense/dense_matmul_function.h"
#include "dense/dense_multi_matmul_function.h"
#include "dense/dense_fast_rename_optimizer.h"
#include "dense/dense_add_dimension_optimizer.h"
#include "dense/dense_single_reduce_function.h"
#include "dense/dense_remove_dimension_optimizer.h"
#include "dense/dense_lambda_peek_optimizer.h"
#include "dense/dense_lambda_function.h"
#include "dense/dense_simple_expand_function.h"
#include "dense/dense_simple_join_function.h"
#include "dense/dense_number_join_function.h"
#include "dense/dense_pow_as_map_optimizer.h"
#include "dense/dense_simple_map_function.h"
#include "dense/vector_from_doubles_function.h"
#include "dense/dense_tensor_create_function.h"
#include "dense/dense_tensor_peek_function.h"
#include <vespa/eval/eval/value.h>
#include <vespa/eval/eval/tensor_spec.h>
#include <vespa/eval/eval/simple_value.h>
#include <vespa/eval/eval/operation.h>
#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/vespalib/util/exceptions.h>
#include <cassert>

#include <vespa/log/log.h>
LOG_SETUP(".eval.tensor.default_tensor_engine");

namespace vespalib::tensor {

using eval::Aggr;
using eval::Aggregator;
using eval::DoubleValue;
using eval::TensorFunction;
using eval::TensorSpec;
using eval::Value;
using eval::ValueType;
using CellType = eval::ValueType::CellType;
using vespalib::IllegalArgumentException;
using vespalib::make_string;

using map_fun_t = vespalib::eval::operation::op1_t;
using join_fun_t = vespalib::eval::operation::op2_t;

namespace {

constexpr size_t UNDEFINED_IDX = std::numeric_limits<size_t>::max();

const eval::EngineOrFactory &simple_engine() {
    static eval::EngineOrFactory engine(eval::SimpleValueBuilderFactory::get());
    return engine;
}
const eval::TensorEngine &default_engine() { return DefaultTensorEngine::ref(); }

// map tensors to simple tensors before fall-back evaluation

const Value &to_simple(const Value &value, Stash &stash) {
    if (auto tensor = value.as_tensor()) {
        if (auto wrapped = dynamic_cast<const WrappedSimpleValue *>(tensor)) {
            return wrapped->unwrap();
        }
        nbostream data;
        tensor->engine().encode(*tensor, data);
        return *stash.create<Value::UP>(simple_engine().decode(data));
    }
    return value;
}

// map tensors to default tensors after fall-back evaluation

const Value &to_default(const Value &value, Stash &stash) {
    // case 1 : a tensor with an engine
    if (auto tensor = value.as_tensor()) {
        // case [1A]: it's already one of "our" tensors
        if (&tensor->engine() == &default_engine()) {
            return value;
        }
        // case [1B]: it belongs to some other engine
        nbostream data;
        tensor->engine().encode(*tensor, data);
        return *stash.create<Value::UP>(default_engine().decode(data));
    }
    // case 2 : some kind of double (possibly in a SimpleValue or DenseTensor)
    if (value.type().is_double()) {
        // case [2A]: already OK
        if (dynamic_cast<const DoubleValue *>(&value)) {
            return value;
        }
        // case [2B]: simplify to DoubleValue
        return stash.create<DoubleValue>(value.as_double());
    }
    // case 3 : it's a (possibly mixed) SimpleValue
    if (auto simple = dynamic_cast<const eval::SimpleValue *>(&value)) {
        // case [3A]: not one of our supported types, just wrap it
        if (!Tensor::supported({simple->type()})) {
            return stash.create<WrappedSimpleValue>(*simple);
        }
        // case [3B]: we should convert to one of our supported types
    }
    // case [4]: some other kind of Value, convert to one of our
    // supported types or make a WrappedSimpleValue
    nbostream data;
    simple_engine().encode(value, data);
    return *stash.create<Value::UP>(default_engine().decode(data));
}

const Value &to_value(std::unique_ptr<Tensor> tensor, Stash &stash) {
    assert(tensor);
    if (tensor->type().is_tensor()) {
        return *stash.create<Value::UP>(std::move(tensor));
    }
    return stash.create<DoubleValue>(tensor->as_double());
}

Value::UP to_value(std::unique_ptr<Tensor> tensor) {
    if (tensor->type().is_tensor()) {
        return tensor;
    }
    return std::make_unique<DoubleValue>(tensor->as_double());
}

const Value &fallback_join(const Value &a, const Value &b, join_fun_t function, Stash &stash) {
    return to_default(simple_engine().join(to_simple(a, stash), to_simple(b, stash), function, stash), stash);
}

const Value &fallback_merge(const Value &a, const Value &b, join_fun_t function, Stash &stash) {
    return to_default(simple_engine().merge(to_simple(a, stash), to_simple(b, stash), function, stash), stash);
}<commit_msg>simplify to_default()<commit_after>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "default_tensor_engine.h"
#include "tensor.h"
#include "wrapped_simple_value.h"
#include "serialization/typed_binary_format.h"
#include "sparse/sparse_tensor_address_builder.h"
#include "sparse/direct_sparse_tensor_builder.h"
#include "dense/dense_tensor.h"
#include "dense/typed_dense_tensor_builder.h"
#include "dense/dense_dot_product_function.h"
#include "dense/dense_xw_product_function.h"
#include "dense/dense_matmul_function.h"
#include "dense/dense_multi_matmul_function.h"
#include "dense/dense_fast_rename_optimizer.h"
#include "dense/dense_add_dimension_optimizer.h"
#include "dense/dense_single_reduce_function.h"
#include "dense/dense_remove_dimension_optimizer.h"
#include "dense/dense_lambda_peek_optimizer.h"
#include "dense/dense_lambda_function.h"
#include "dense/dense_simple_expand_function.h"
#include "dense/dense_simple_join_function.h"
#include "dense/dense_number_join_function.h"
#include "dense/dense_pow_as_map_optimizer.h"
#include "dense/dense_simple_map_function.h"
#include "dense/vector_from_doubles_function.h"
#include "dense/dense_tensor_create_function.h"
#include "dense/dense_tensor_peek_function.h"
#include <vespa/eval/eval/value.h>
#include <vespa/eval/eval/tensor_spec.h>
#include <vespa/eval/eval/simple_value.h>
#include <vespa/eval/eval/operation.h>
#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/vespalib/util/exceptions.h>
#include <cassert>

#include <vespa/log/log.h>
LOG_SETUP(".eval.tensor.default_tensor_engine");

namespace vespalib::tensor {

using eval::Aggr;
using eval::Aggregator;
using eval::DoubleValue;
using eval::TensorFunction;
using eval::TensorSpec;
using eval::Value;
using eval::ValueType;
using CellType = eval::ValueType::CellType;
using vespalib::IllegalArgumentException;
using vespalib::make_string;

using map_fun_t = vespalib::eval::operation::op1_t;
using join_fun_t = vespalib::eval::operation::op2_t;

namespace {

constexpr size_t UNDEFINED_IDX = std::numeric_limits<size_t>::max();

const eval::EngineOrFactory &simple_engine() {
    static eval::EngineOrFactory engine(eval::SimpleValueBuilderFactory::get());
    return engine;
}
const eval::TensorEngine &default_engine() { return DefaultTensorEngine::ref(); }

// map tensors to simple tensors before fall-back evaluation

const Value &to_simple(const Value &value, Stash &stash) {
    if (auto tensor = value.as_tensor()) {
        if (auto wrapped = dynamic_cast<const WrappedSimpleValue *>(tensor)) {
            return wrapped->unwrap();
        }
        nbostream data;
        tensor->engine().encode(*tensor, data);
        return *stash.create<Value::UP>(simple_engine().decode(data));
    }
    return value;
}

// map tensors to default tensors after fall-back evaluation

const Value &to_default(const Value &value, Stash &stash) {
    if (! value.type().is_double()) {
        if (! Tensor::supported({value.type()})) {
            return stash.create<WrappedSimpleValue>(value);
        }
        nbostream data;
        simple_engine().encode(value, data);
        return *stash.create<Value::UP>(default_engine().decode(data));
    }
    return value;
}

const Value &to_value(std::unique_ptr<Tensor> tensor, Stash &stash) {
    assert(tensor);
    if (tensor->type().is_tensor()) {
        return *stash.create<Value::UP>(std::move(tensor));
    }
    return stash.create<DoubleValue>(tensor->as_double());
}

Value::UP to_value(std::unique_ptr<Tensor> tensor) {
    if (tensor->type().is_tensor()) {
        return tensor;
    }
    return std::make_unique<DoubleValue>(tensor->as_double());
}

const Value &fallback_join(const Value &a, const Value &b, join_fun_t function, Stash &stash) {
    return to_default(simple_engine().join(to_simple(a, stash), to_simple(b, stash), function, stash), stash);
}

const Value &fallback_merge(const Value &a, const Value &b, join_fun_t function, Stash &stash) {
    return to_default(simple_engine().merge(to_simple(a, stash), to_simple(b, stash), function, stash), stash);
}
{'fraction_non_alphanumeric': 0.132928259247415, 'fraction_numerical': 0.0027715595352307857, 'mean_word_length': 4.991060025542784, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5212000', 'n_tokens_mistral': 3164, 'n_tokens_neox': 3035, 'n_words': 642}