<commit_before>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "values.h"
#include <vespa/fnet/databuffer.h>
#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/util/stash.h>
#include <vespa/vespalib/data/databuffer.h>
#include <cassert>

static_assert(sizeof(uint8_t) == 1, "uint8_t must be 1 byte.");
static_assert(sizeof(float)   == sizeof(uint32_t), "float must be same size as uint32_t");
static_assert(sizeof(double)  == sizeof(uint64_t), "double must be same size as uint64_t");

constexpr size_t SHARED_LIMIT = 1024;

namespace fnet {

char * copyString(char *dst, const char *src, size_t len) {
    memcpy(dst, src, len);
    dst[len] = '\0';
    return dst;
}

char * copyData(char *dst, const void *src, size_t len) {
    memcpy(dst, src, len);
    return dst;
}

using vespalib::alloc::Alloc;
class LocalBlob : public FRT_ISharedBlob
{
public:
    LocalBlob(Alloc data, uint32_t len) :
            _data(std::move(data)),
            _len(len)
    { }
    LocalBlob(const char *data, uint32_t len);
    void addRef() override {}
    void subRef() override { Alloc().swap(_data); }
    uint32_t getLen() override { return _len; }
    const char *getData() override { return static_cast<const char *>(_data.get()); }
    char *getInternalData() { return static_cast<char *>(_data.get()); }
private:
    LocalBlob(const LocalBlob &);
    LocalBlob &operator=(const LocalBlob &);

    Alloc _data;
    uint32_t _len;
};

struct BlobRef
{
    FRT_DataValue   *_value; // for blob inside data array
    uint32_t         _idx;   // for blob as single data value
    FRT_ISharedBlob *_blob;  // interface to shared data
    BlobRef         *_next;  // next in list

    BlobRef(FRT_DataValue *value, uint32_t idx, FRT_ISharedBlob *blob, BlobRef *next)
            : _value(value), _idx(idx), _blob(blob), _next(next) { blob->addRef(); }
    ~BlobRef() { discard(); }
    void discard() {
        if (_blob != nullptr) {
            _blob->subRef();
            _blob = nullptr;
        }
    }
private:
    BlobRef(const BlobRef &);
    BlobRef &operator=(const BlobRef &);
};

}

using fnet::BlobRef;
using fnet::LocalBlob;

FRT_Values::FRT_Values(Stash &stash)
    : _maxValues(0),
      _numValues(0),
      _typeString(nullptr),
      _values(nullptr),
      _blobs(nullptr),
      _stash(stash)
{ }

FRT_Values::~FRT_Values() = default;

LocalBlob::LocalBlob(const char *data, uint32_t len) :
        _data(Alloc::alloc(len)),
        _len(len)
{
    if (data != nullptr) {
        memcpy(_data.get(), data, len);
    }
}

void
FRT_Values::DiscardBlobs()
{
    while (_blobs != nullptr) {
        BlobRef *ref = _blobs;
        _blobs = ref->_next;
        FRT_ISharedBlob *blob = ref->_blob;
        FRT_DataValue *value = ref->_value;
        if (value == nullptr) {
            uint32_t idx = ref->_idx;
            assert(_numValues > idx);
            assert(_typeString[idx] == 'x');
            value = &_values[idx]._data;
        }
        if ((value->_buf == blob->getData()) && (value->_len == blob->getLen())) {
            value->_buf = nullptr;
            value->_len = 0;
        }
        ref->discard();
    }
}

void
FRT_Values::EnsureFree(uint32_t need)
{
    if (_numValues + need <= _maxValues)
        return;

    uint32_t cnt = _maxValues * 2;
    if (cnt < _numValues + need)
        cnt = _numValues + need;
    if (cnt < 16)
        cnt = 16;

    char *types = (char *) _stash.alloc(cnt + 1);
    memcpy(types, _typeString, _numValues);
    memset(types + _numValues, FRT_VALUE_NONE, cnt + 1 - _numValues);
    FRT_Value *values = (FRT_Value *) (void *)_stash.alloc(cnt * sizeof(FRT_Value));
    memcpy(values, _values, _numValues * sizeof(FRT_Value));
    _maxValues  = cnt;
    _typeString = types;
    _values     = values;
}<commit_msg>Avoid undefined behavior for zero-sized memcpy with nullptr argument<commit_after>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "values.h"
#include <vespa/fnet/databuffer.h>
#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/util/stash.h>
#include <vespa/vespalib/data/databuffer.h>
#include <cassert>

static_assert(sizeof(uint8_t) == 1, "uint8_t must be 1 byte.");
static_assert(sizeof(float)   == sizeof(uint32_t), "float must be same size as uint32_t");
static_assert(sizeof(double)  == sizeof(uint64_t), "double must be same size as uint64_t");

constexpr size_t SHARED_LIMIT = 1024;

namespace fnet {

char * copyString(char *dst, const char *src, size_t len) {
    memcpy(dst, src, len);
    dst[len] = '\0';
    return dst;
}

char * copyData(char *dst, const void *src, size_t len) {
    memcpy(dst, src, len);
    return dst;
}

using vespalib::alloc::Alloc;
class LocalBlob : public FRT_ISharedBlob
{
public:
    LocalBlob(Alloc data, uint32_t len) :
            _data(std::move(data)),
            _len(len)
    { }
    LocalBlob(const char *data, uint32_t len);
    void addRef() override {}
    void subRef() override { Alloc().swap(_data); }
    uint32_t getLen() override { return _len; }
    const char *getData() override { return static_cast<const char *>(_data.get()); }
    char *getInternalData() { return static_cast<char *>(_data.get()); }
private:
    LocalBlob(const LocalBlob &);
    LocalBlob &operator=(const LocalBlob &);

    Alloc _data;
    uint32_t _len;
};

struct BlobRef
{
    FRT_DataValue   *_value; // for blob inside data array
    uint32_t         _idx;   // for blob as single data value
    FRT_ISharedBlob *_blob;  // interface to shared data
    BlobRef         *_next;  // next in list

    BlobRef(FRT_DataValue *value, uint32_t idx, FRT_ISharedBlob *blob, BlobRef *next)
            : _value(value), _idx(idx), _blob(blob), _next(next) { blob->addRef(); }
    ~BlobRef() { discard(); }
    void discard() {
        if (_blob != nullptr) {
            _blob->subRef();
            _blob = nullptr;
        }
    }
private:
    BlobRef(const BlobRef &);
    BlobRef &operator=(const BlobRef &);
};

}

using fnet::BlobRef;
using fnet::LocalBlob;

FRT_Values::FRT_Values(Stash &stash)
    : _maxValues(0),
      _numValues(0),
      _typeString(nullptr),
      _values(nullptr),
      _blobs(nullptr),
      _stash(stash)
{ }

FRT_Values::~FRT_Values() = default;

LocalBlob::LocalBlob(const char *data, uint32_t len) :
        _data(Alloc::alloc(len)),
        _len(len)
{
    if (data != nullptr) {
        memcpy(_data.get(), data, len);
    }
}

void
FRT_Values::DiscardBlobs()
{
    while (_blobs != nullptr) {
        BlobRef *ref = _blobs;
        _blobs = ref->_next;
        FRT_ISharedBlob *blob = ref->_blob;
        FRT_DataValue *value = ref->_value;
        if (value == nullptr) {
            uint32_t idx = ref->_idx;
            assert(_numValues > idx);
            assert(_typeString[idx] == 'x');
            value = &_values[idx]._data;
        }
        if ((value->_buf == blob->getData()) && (value->_len == blob->getLen())) {
            value->_buf = nullptr;
            value->_len = 0;
        }
        ref->discard();
    }
}

void
FRT_Values::EnsureFree(uint32_t need)
{
    if (_numValues + need <= _maxValues)
        return;

    uint32_t cnt = _maxValues * 2;
    if (cnt < _numValues + need)
        cnt = _numValues + need;
    if (cnt < 16)
        cnt = 16;

    char *types = (char *) _stash.alloc(cnt + 1);
    if (_numValues > 0) {
        assert(_typeString != nullptr);
        memcpy(types, _typeString, _numValues);
    }
    memset(types + _numValues, FRT_VALUE_NONE, cnt + 1 - _numValues);
    FRT_Value *values = (FRT_Value *) (void *)_stash.alloc(cnt * sizeof(FRT_Value));
    if (_numValues > 0) {
        assert(_values != nullptr);
        memcpy(values, _values, _numValues * sizeof(FRT_Value));
    }
    _maxValues  = cnt;
    _typeString = types;
    _values     = values;
}
{'fraction_non_alphanumeric': 0.1379179810725552, 'fraction_numerical': 0.012618296529968454, 'mean_word_length': 2.832688588007737, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2536127', 'n_tokens_mistral': 2803, 'n_tokens_neox': 2730, 'n_words': 654}