<commit_before>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "tensor_buffer_store.h"
#include <vespa/document/util/serializableexceptions.h>
#include <vespa/eval/eval/value_codec.h>
#include <vespa/eval/streamed/streamed_value_builder_factory.h>
#include <vespa/vespalib/datastore/array_store.hpp>
#include <vespa/vespalib/datastore/buffer_type.hpp>
#include <vespa/vespalib/datastore/compaction_context.h>
#include <vespa/vespalib/datastore/compaction_strategy.h>
#include <vespa/vespalib/datastore/datastore.hpp>
#include <vespa/vespalib/util/size_literals.h>

using document::DeserializeException;
using vespalib::alloc::MemoryAllocator;
using vespalib::datastore::CompactionContext;
using vespalib::datastore::CompactionStrategy;
using vespalib::datastore::EntryRef;
using vespalib::eval::StreamedValueBuilderFactory;
using vespalib::eval::Value;
using vespalib::eval::ValueType;

namespace search::tensor {

namespace {

constexpr float ALLOC_GROW_FACTOR = 0.2;

}

TensorBufferStore::TensorBufferStore(const ValueType& tensor_type, std::shared_ptr<MemoryAllocator> allocator, uint32_t max_small_subspaces_type_id)
    : TensorStore(ArrayStoreType::get_data_store_base(_array_store)),
      _tensor_type(tensor_type),
      _ops(_tensor_type),
      _array_store(ArrayStoreType::optimizedConfigForHugePage(max_small_subspaces_type_id,
                                                              TensorBufferTypeMapper(max_small_subspaces_type_id, &_ops),
                                                              MemoryAllocator::HUGEPAGE_SIZE, 4_Ki, 8_Ki, ALLOC_GROW_FACTOR),
                   std::move(allocator), TensorBufferTypeMapper(max_small_subspaces_type_id, &_ops))
{
}

TensorBufferStore::~TensorBufferStore() = default;

void
TensorBufferStore::holdTensor(EntryRef ref)
{
    _array_store.remove(ref);
}

EntryRef
TensorBufferStore::move_on_compact(EntryRef ref)
{
    if (!ref.valid()) {
        return EntryRef();
    }
    auto buf = _array_store.get(ref);
    auto new_ref = _array_store.add(buf);
    _ops.copied_labels(unconstify(buf));
    return new_ref;
}

vespalib::MemoryUsage
TensorBufferStore::update_stat(const CompactionStrategy& compaction_strategy)
{
    auto array_store_address_space_usage = _store.getAddressSpaceUsage();
    auto array_store_memory_usage = _store.getMemoryUsage();
    _compaction_spec = compaction_strategy.should_compact(array_store_memory_usage, array_store_address_space_usage);
    return array_store_memory_usage;
}

std::unique_ptr<vespalib::datastore::ICompactionContext>
TensorBufferStore::start_compact(const CompactionStrategy& compaction_strategy)<commit_msg>Ask array store for writable buffer.<commit_after>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "tensor_buffer_store.h"
#include <vespa/document/util/serializableexceptions.h>
#include <vespa/eval/eval/value_codec.h>
#include <vespa/eval/streamed/streamed_value_builder_factory.h>
#include <vespa/vespalib/datastore/array_store.hpp>
#include <vespa/vespalib/datastore/buffer_type.hpp>
#include <vespa/vespalib/datastore/compaction_context.h>
#include <vespa/vespalib/datastore/compaction_strategy.h>
#include <vespa/vespalib/datastore/datastore.hpp>
#include <vespa/vespalib/util/size_literals.h>

using document::DeserializeException;
using vespalib::alloc::MemoryAllocator;
using vespalib::datastore::CompactionContext;
using vespalib::datastore::CompactionStrategy;
using vespalib::datastore::EntryRef;
using vespalib::eval::StreamedValueBuilderFactory;
using vespalib::eval::Value;
using vespalib::eval::ValueType;

namespace search::tensor {

namespace {

constexpr float ALLOC_GROW_FACTOR = 0.2;

}

TensorBufferStore::TensorBufferStore(const ValueType& tensor_type, std::shared_ptr<MemoryAllocator> allocator, uint32_t max_small_subspaces_type_id)
    : TensorStore(ArrayStoreType::get_data_store_base(_array_store)),
      _tensor_type(tensor_type),
      _ops(_tensor_type),
      _array_store(ArrayStoreType::optimizedConfigForHugePage(max_small_subspaces_type_id,
                                                              TensorBufferTypeMapper(max_small_subspaces_type_id, &_ops),
                                                              MemoryAllocator::HUGEPAGE_SIZE, 4_Ki, 8_Ki, ALLOC_GROW_FACTOR),
                   std::move(allocator), TensorBufferTypeMapper(max_small_subspaces_type_id, &_ops))
{
}

TensorBufferStore::~TensorBufferStore() = default;

void
TensorBufferStore::holdTensor(EntryRef ref)
{
    _array_store.remove(ref);
}

EntryRef
TensorBufferStore::move_on_compact(EntryRef ref)
{
    if (!ref.valid()) {
        return EntryRef();
    }
    auto buf = _array_store.get_writable(ref);
    auto new_ref = _array_store.add(buf);
    _ops.copied_labels(buf);
    return new_ref;
}

vespalib::MemoryUsage
TensorBufferStore::update_stat(const CompactionStrategy& compaction_strategy)
{
    auto array_store_address_space_usage = _store.getAddressSpaceUsage();
    auto array_store_memory_usage = _store.getMemoryUsage();
    _compaction_spec = compaction_strategy.should_compact(array_store_memory_usage, array_store_address_space_usage);
    return array_store_memory_usage;
}

std::unique_ptr<vespalib::datastore::ICompactionContext>
TensorBufferStore::start_compact(const CompactionStrategy& compaction_strategy)
{'fraction_non_alphanumeric': 0.09786214522668632, 'fraction_numerical': 0.002948765204570586, 'mean_word_length': 5.362250879249707, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6539300', 'n_tokens_mistral': 1765, 'n_tokens_neox': 1716, 'n_words': 248}