<commit_before>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#pragma once

#include "sparse_tensor_address_ref.h"
#include <vespa/vespalib/stllike/string.h>

namespace vespalib::tensor {


/**
 * A writer to serialize tensor addresses into a compact representation.
 * All dimensions in the tensors are present, empty label is the "undefined"
 * value.
 *
 * Format: (labelStr NUL)*
 */
class SparseTensorAddressBuilder
{
private:
    vespalib::Array<char> _address;

protected:
    void append(vespalib::stringref str) {
        for (size_t i(0); i < str.size(); i++) {
            _address.push_back_fast(str[i]);
        }
    }
    void ensure_room(size_t additional) { _address.reserve(_address.size() + additional); }
public:
    SparseTensorAddressBuilder() : _address() {}
    void add(vespalib::stringref label) {
        ensure_room(label.size()+1);
        append(label);
    }
    void addUndefined() { _address.push_back('\0'); }
    void clear() { _address.clear(); }
    SparseTensorAddressRef getAddressRef() const {
        return SparseTensorAddressRef(&_address[0], _address.size());
    }
    bool empty() const { return _address.empty(); }
};<commit_msg>Also copy the zero termination.<commit_after>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#pragma once

#include "sparse_tensor_address_ref.h"
#include <vespa/vespalib/stllike/string.h>

namespace vespalib::tensor {


/**
 * A writer to serialize tensor addresses into a compact representation.
 * All dimensions in the tensors are present, empty label is the "undefined"
 * value.
 *
 * Format: (labelStr NUL)*
 */
class SparseTensorAddressBuilder
{
private:
    vespalib::Array<char> _address;

protected:
    void append(vespalib::stringref str) {
        for (size_t i(0); i < str.size() + 1; i++) {
            _address.push_back_fast(str[i]);
        }
    }
    void ensure_room(size_t additional) { _address.reserve(_address.size() + additional); }
public:
    SparseTensorAddressBuilder() : _address() {}
    void add(vespalib::stringref label) {
        ensure_room(label.size()+1);
        append(label);
    }
    void addUndefined() { _address.push_back('\0'); }
    void clear() { _address.clear(); }
    SparseTensorAddressRef getAddressRef() const {
        return SparseTensorAddressRef(&_address[0], _address.size());
    }
    bool empty() const { return _address.empty(); }
};
{'fraction_non_alphanumeric': 0.1246031746031746, 'fraction_numerical': 0.008333333333333333, 'mean_word_length': 3.6685185185185185, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '532402', 'n_tokens_mistral': 789, 'n_tokens_neox': 765, 'n_words': 205}