<commit_before>// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "value_codec.h"
#include "tensor_spec.h"
#include "array_array_map.h"
#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/vespalib/util/exceptions.h>
#include <vespa/vespalib/util/typify.h>
#include <vespa/vespalib/util/small_vector.h>
#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/util/shared_string_repo.h>

using vespalib::make_string_short::fmt;

namespace vespalib::eval {

namespace {

constexpr uint32_t DOUBLE_CELL_TYPE = 0;
constexpr uint32_t FLOAT_CELL_TYPE = 1;

inline uint32_t cell_type_to_id(CellType cell_type) {
    switch (cell_type) {
    case CellType::DOUBLE: return DOUBLE_CELL_TYPE;
    case CellType::FLOAT: return FLOAT_CELL_TYPE;
    }
    throw IllegalArgumentException(fmt("Unknown CellType=%u", (uint32_t)cell_type));
}

inline CellType id_to_cell_type(uint32_t id) {
    switch (id) {
    case DOUBLE_CELL_TYPE: return CellType::DOUBLE;
    case FLOAT_CELL_TYPE: return CellType::FLOAT;
    }
    throw IllegalArgumentException(fmt("Unknown CellType id=%u", id));
}

struct Format {
    bool     has_sparse;
    bool     has_dense;
    bool     with_cell_type;
    uint32_t tag;
    explicit Format(const ValueType &type)
        : has_sparse(type.count_mapped_dimensions() > 0),
          has_dense((type.count_indexed_dimensions() > 0) || !has_sparse),
          with_cell_type(type.cell_type() != CellType::DOUBLE),
          tag((has_sparse ? 0x1 : 0) | (has_dense ? 0x2 : 0) | (with_cell_type ? 0x4 : 0)) {}
    explicit Format(uint32_t tag_in)
        : has_sparse((tag_in & 0x1) != 0),
          has_dense((tag_in & 0x2) != 0),
          with_cell_type((tag_in & 0x4) != 0),
          tag(tag_in)
    {
        if (tag & ~7u) {
            throw IllegalArgumentException(fmt("Unknown tensor format tag=%u", tag));
        }
    }
    ~Format() {}
};

void maybe_encode_cell_type(nbostream &output, const Format &format, CellType cell_type) {
    if (format.with_cell_type) {
        output.putInt1_4Bytes(cell_type_to_id(cell_type));
    }
}

void encode_type(nbostream &output, const Format &format, const ValueType &type) {
    maybe_encode_cell_type(output, format, type.cell_type());
    if (format.has_sparse) {
        output.putInt1_4Bytes(type.count_mapped_dimensions());
        for (const auto & dim : type.dimensions()) {
            if (dim.is_mapped()) {
                output.writeSmallString(dim.name);
            }
        }
    }
    if (format.has_dense) {
        output.putInt1_4Bytes(type.count_indexed_dimensions());
        for (const auto & dim : type.dimensions()) {
            if (dim.is_indexed()) {
                output.writeSmallString(dim.name);
                output.putInt1_4Bytes(dim.size);
            }
        }
    }
}

void maybe_encode_num_blocks(nbostream &output, bool has_mapped_dims, size_t num_blocks) {
    if (has_mapped_dims) {
        output.putInt1_4Bytes(num_blocks);
    }
}

CellType maybe_decode_cell_type(nbostream &input, const Format &format) {
    if (format.with_cell_type) {
        return id_to_cell_type(input.getInt1_4Bytes());
    }
    return CellType::DOUBLE;
}

ValueType decode_type(nbostream &input, const Format &format) {
    CellType cell_type = maybe_decode_cell_type(input, format);
    std::vector<ValueType::Dimension> dim_list;
    if (format.has_sparse) {
        size_t cnt = input.getInt1_4Bytes();
        for (size_t i = 0; i < cnt; ++i) {
            vespalib::string name;
            input.readSmallString(name);
            dim_list.emplace_back(name);
        }
    }
    if (format.has_dense) {
        size_t cnt = input.getInt1_4Bytes();
        for (size_t i = 0; i < cnt; ++i) {
            vespalib::string name;
            input.readSmallString(name);
            dim_list.emplace_back(name, input.getInt1_4Bytes());
        }
    }
    if (dim_list.empty()) {
        assert(cell_type == CellType::DOUBLE);<commit_msg>do not assert when receiving invalid data (from network)<commit_after>// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "value_codec.h"
#include "tensor_spec.h"
#include "array_array_map.h"
#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/vespalib/util/exceptions.h>
#include <vespa/vespalib/util/typify.h>
#include <vespa/vespalib/util/small_vector.h>
#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/util/shared_string_repo.h>

using vespalib::make_string_short::fmt;

namespace vespalib::eval {

namespace {

constexpr uint32_t DOUBLE_CELL_TYPE = 0;
constexpr uint32_t FLOAT_CELL_TYPE = 1;

inline uint32_t cell_type_to_id(CellType cell_type) {
    switch (cell_type) {
    case CellType::DOUBLE: return DOUBLE_CELL_TYPE;
    case CellType::FLOAT: return FLOAT_CELL_TYPE;
    }
    throw IllegalArgumentException(fmt("Unknown CellType=%u", (uint32_t)cell_type));
}

inline CellType id_to_cell_type(uint32_t id) {
    switch (id) {
    case DOUBLE_CELL_TYPE: return CellType::DOUBLE;
    case FLOAT_CELL_TYPE: return CellType::FLOAT;
    }
    throw IllegalArgumentException(fmt("Unknown CellType id=%u", id));
}

struct Format {
    bool     has_sparse;
    bool     has_dense;
    bool     with_cell_type;
    uint32_t tag;
    explicit Format(const ValueType &type)
        : has_sparse(type.count_mapped_dimensions() > 0),
          has_dense((type.count_indexed_dimensions() > 0) || !has_sparse),
          with_cell_type(type.cell_type() != CellType::DOUBLE),
          tag((has_sparse ? 0x1 : 0) | (has_dense ? 0x2 : 0) | (with_cell_type ? 0x4 : 0)) {}
    explicit Format(uint32_t tag_in)
        : has_sparse((tag_in & 0x1) != 0),
          has_dense((tag_in & 0x2) != 0),
          with_cell_type((tag_in & 0x4) != 0),
          tag(tag_in)
    {
        if (tag & ~7u) {
            throw IllegalArgumentException(fmt("Unknown tensor format tag=%u", tag));
        }
    }
    ~Format() {}
};

void maybe_encode_cell_type(nbostream &output, const Format &format, CellType cell_type) {
    if (format.with_cell_type) {
        output.putInt1_4Bytes(cell_type_to_id(cell_type));
    }
}

void encode_type(nbostream &output, const Format &format, const ValueType &type) {
    maybe_encode_cell_type(output, format, type.cell_type());
    if (format.has_sparse) {
        output.putInt1_4Bytes(type.count_mapped_dimensions());
        for (const auto & dim : type.dimensions()) {
            if (dim.is_mapped()) {
                output.writeSmallString(dim.name);
            }
        }
    }
    if (format.has_dense) {
        output.putInt1_4Bytes(type.count_indexed_dimensions());
        for (const auto & dim : type.dimensions()) {
            if (dim.is_indexed()) {
                output.writeSmallString(dim.name);
                output.putInt1_4Bytes(dim.size);
            }
        }
    }
}

void maybe_encode_num_blocks(nbostream &output, bool has_mapped_dims, size_t num_blocks) {
    if (has_mapped_dims) {
        output.putInt1_4Bytes(num_blocks);
    }
}

CellType maybe_decode_cell_type(nbostream &input, const Format &format) {
    if (format.with_cell_type) {
        return id_to_cell_type(input.getInt1_4Bytes());
    }
    return CellType::DOUBLE;
}

ValueType decode_type(nbostream &input, const Format &format) {
    CellType cell_type = maybe_decode_cell_type(input, format);
    std::vector<ValueType::Dimension> dim_list;
    if (format.has_sparse) {
        size_t cnt = input.getInt1_4Bytes();
        for (size_t i = 0; i < cnt; ++i) {
            vespalib::string name;
            input.readSmallString(name);
            dim_list.emplace_back(name);
        }
    }
    if (format.has_dense) {
        size_t cnt = input.getInt1_4Bytes();
        for (size_t i = 0; i < cnt; ++i) {
            vespalib::string name;
            input.readSmallString(name);
            dim_list.emplace_back(name, input.getInt1_4Bytes());
        }
    }
    if (dim_list.empty() && (cell_type != CellType::DOUBLE)) {
        throw IllegalArgumentException(fmt("Scalar values must have cell type DOUBLE (was %u)",
                                           (uint32_t)cell_type));

{'fraction_non_alphanumeric': 0.11838913148956817, 'fraction_numerical': 0.01455604075691412, 'mean_word_length': 3.031784841075795, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5887130', 'n_tokens_mistral': 2877, 'n_tokens_neox': 2830, 'n_words': 532}