<commit_before>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "filechunk.h"
#include "data_store_file_chunk_stats.h"
#include "summaryexceptions.h"
#include "randreaders.h"
#include <vespa/vespalib/data/fileheader.h>
#include <vespa/vespalib/stllike/asciistream.h>
#include <vespa/vespalib/util/array.hpp>
#include <vespa/vespalib/stllike/hash_map.hpp>
#include <vespa/searchlib/util/filekit.h>
#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/fastos/file.h>

#include <vespa/log/log.h>
LOG_SETUP(".search.filechunk");

using vespalib::GenericHeader;
using vespalib::getErrorString;

namespace search {

namespace {

constexpr size_t ALIGNMENT=0x1000;
constexpr size_t ENTRY_BIAS_SIZE=8;
const vespalib::string DOC_ID_LIMIT_KEY("docIdLimit");

}

using vespalib::make_string;

FileChunk::ChunkInfo::ChunkInfo(uint64_t offset, uint32_t size, uint64_t lastSerial)
    : _lastSerial(lastSerial),
      _offset(offset),
      _size(size)
{
    assert(valid());
}

vespalib::string
FileChunk::NameId::createName(const vespalib::string &baseName) const {
    vespalib::asciistream os;
    os << baseName << '/' << vespalib::setfill('0') << vespalib::setw(19) << getId();
    return os.str();
}

vespalib::string
FileChunk::createIdxFileName(const vespalib::string & name) {
    return name + ".idx";
}

vespalib::string
FileChunk::createDatFileName(const vespalib::string & name) {
    return name + ".dat";
}

FileChunk::FileChunk(FileId fileId, NameId nameId, const vespalib::string & baseName,
                     const TuneFileSummary & tune, const IBucketizer * bucketizer, bool skipCrcOnRead)
    : _fileId(fileId),
      _nameId(nameId),
      _name(nameId.createName(baseName)),
      _skipCrcOnRead(skipCrcOnRead),
      _erasedCount(0),
      _erasedBytes(0),
      _diskFootprint(0),
      _sumNumBuckets(0),
      _numChunksWithBuckets(0),
      _numUniqueBuckets(0),
      _file(),
      _bucketizer(bucketizer),
      _addedBytes(0),
      _tune(tune),
      _dataFileName(createDatFileName(_name)),
      _idxFileName(createIdxFileName(_name)),
      _chunkInfo(),
      _dataHeaderLen(0u),
      _idxHeaderLen(0u),
      _lastPersistedSerialNum(0),
      _docIdLimit(std::numeric_limits<uint32_t>::max()),
      _modificationTime()
{
    FastOS_File dataFile(_dataFileName.c_str());
    if (dataFile.OpenReadOnly()) {
        if (!dataFile.Sync()) {
            throw SummaryException("Failed syncing dat file", dataFile, VESPA_STRLOC);
        }
        _diskFootprint += dataFile.GetSize();
        FastOS_File idxFile(_idxFileName.c_str());
        if (idxFile.OpenReadOnly()) {
            if (!idxFile.Sync()) {
                throw SummaryException("Failed syncing idx file", idxFile, VESPA_STRLOC);
            }
            _diskFootprint += idxFile.GetSize();
            _modificationTime = FileKit::getModificationTime(_idxFileName);
        } else {
            dataFile.Close();
            throw SummaryException("Failed opening idx file", idxFile, VESPA_STRLOC);
        }
    }
}

FileChunk::~FileChunk() { }

void
FileChunk::addNumBuckets(size_t numBucketsInChunk)
{
    _sumNumBuckets += numBucketsInChunk;
    if (numBucketsInChunk != 0) {
        ++_numChunksWithBuckets;
    }
}

class TmpChunkMeta : public ChunkMeta,
                     public std::vector<LidMeta>
{
public:
    void fill(vespalib::nbostream & is) {
        resize(getNumEntries());
        for (LidMeta & lm : *this) {
            lm.deserialize(is);
        }
    }
};

typedef vespalib::Array<TmpChunkMeta> TmpChunkMetaV;

namespace {

void
verifyOrAssert(const TmpChunkMetaV & v)
{
    for (auto prev(v.begin()), it(prev); it != v.end(); ++it) {
        assert(prev->getLastSerial() <= it->getLastSerial());
        prev = it;
    }
}

vespalib::string eraseErrorMsg(const vespalib::string & fileName, int error) {
    return make_string("Error erasing file '%s'. Error is '%s'",
                       fileName.c_str(), getErrorString(error).c_str());
}

}

void
FileChunk::erase()
{
    _file.reset();
    if (!FastOS_File::Delete(_idxFileName.c_str()) && (errno != ENOENT)) {
        throw std::runtime_error(eraseErrorMsg(_idxFileName, errno));
    }
    if (!FastOS_File::Delete(_dataFileName.c_str()) && (errno != ENOENT)) {
        throw std::runtime_error(eraseErrorMsg(_dataFileName, errno));
    }
}

size_t
FileChunk::updateLidMap(const LockGuard &guard, ISetLid &ds, uint64_t serialNum, uint32_t docIdLimit)
{
    size_t sz(0);
    assert(_chunkInfo.empty());

    FastOS_File idxFile(_idxFileName.c_str());
    idxFile.enableMemoryMap(0);
    if (idxFile.OpenReadOnly()) {
        if (idxFile.IsMemoryMapped()) {
            const int64_t fileSize = idxFile.GetSize();
            if (_idxHeaderLen == 0) {
                _idxHeaderLen = readIdxHeader(idxFile, _docIdLimit);
            }
            vespalib::nbostream is(static_cast<const char *>(idxFile.MemoryMapPtr(0)) + _idxHeaderLen,
                                   fileSize - _idxHeaderLen);
            TmpChunkMetaV tempVector;
            tempVector.reserve(fileSize/(sizeof(ChunkMeta)+sizeof(LidMeta)));
            while ( ! is.empty() && is.good()) {
                const int64_t lastKnownGoodPos = _idxHeaderLen + is.rp();
                tempVector.push_back(TmpChunkMeta());
                TmpChunkMeta & chunkMeta(tempVector.back());
                try {
                    chunkMeta.deserialize(is);
                    chunkMeta.fill(is);
                } catch (const vespalib::IllegalStateException & e) {
                    LOG(warning, "Exception deserializing idx file : %s", e.what());
                    LOG(warning, "File '%s' seems to be partially truncated. Will truncate from size=%ld to %ld",
                                 _idxFileName.c_str(), fileSize, lastKnownGoodPos);
                    FastOS_File toTruncate(_idxFileName.c_str());
                    if ( toTruncate.OpenReadWrite()) {
                        if (toTruncate.SetSize(lastKnownGoodPos)) {
                            tempVector.resize(tempVector.size() - 1);
                        } else {
                            throw SummaryException("SetSize(%ld) failed.", toTruncate, VESPA_STRLOC);
                        }
                    } else {
                        throw SummaryException("Open for truncation failed.", toTruncate, VESPA_STRLOC);
                    }
                }
            }
            if ( ! tempVector.empty()) {
                verifyOrAssert(tempVector);
                if (tempVector[0].getLastSerial() < serialNum) {
                    LOG(warning,
                        "last serial num(%ld) from previous file is "
                        "bigger than my first(%ld). That is odd."
                        "Current filename is '%s'",
                        serialNum, tempVector[0].getLastSerial(),
                        _idxFileName.c_str());
                    serialNum = tempVector[0].getLastSerial();
                }
                BucketDensityComputer globalBucketMap(_bucketizer);
                // Guard comes from the same bucketizer so the same guard can be used
                // for both local and global BucketDensityComputer
                vespalib::GenerationHandler::Guard bucketizerGuard = globalBucketMap.getGuard();
                for (const TmpChunkMeta & chunkMeta : tempVector) {
                    assert(serialNum <= chunkMeta.getLastSerial());<commit_msg>Break out on error before the end.<commit_after>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "filechunk.h"
#include "data_store_file_chunk_stats.h"
#include "summaryexceptions.h"
#include "randreaders.h"
#include <vespa/vespalib/data/fileheader.h>
#include <vespa/vespalib/stllike/asciistream.h>
#include <vespa/vespalib/util/array.hpp>
#include <vespa/vespalib/stllike/hash_map.hpp>
#include <vespa/searchlib/util/filekit.h>
#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/fastos/file.h>

#include <vespa/log/log.h>
LOG_SETUP(".search.filechunk");

using vespalib::GenericHeader;
using vespalib::getErrorString;

namespace search {

namespace {

constexpr size_t ALIGNMENT=0x1000;
constexpr size_t ENTRY_BIAS_SIZE=8;
const vespalib::string DOC_ID_LIMIT_KEY("docIdLimit");

}

using vespalib::make_string;

FileChunk::ChunkInfo::ChunkInfo(uint64_t offset, uint32_t size, uint64_t lastSerial)
    : _lastSerial(lastSerial),
      _offset(offset),
      _size(size)
{
    assert(valid());
}

vespalib::string
FileChunk::NameId::createName(const vespalib::string &baseName) const {
    vespalib::asciistream os;
    os << baseName << '/' << vespalib::setfill('0') << vespalib::setw(19) << getId();
    return os.str();
}

vespalib::string
FileChunk::createIdxFileName(const vespalib::string & name) {
    return name + ".idx";
}

vespalib::string
FileChunk::createDatFileName(const vespalib::string & name) {
    return name + ".dat";
}

FileChunk::FileChunk(FileId fileId, NameId nameId, const vespalib::string & baseName,
                     const TuneFileSummary & tune, const IBucketizer * bucketizer, bool skipCrcOnRead)
    : _fileId(fileId),
      _nameId(nameId),
      _name(nameId.createName(baseName)),
      _skipCrcOnRead(skipCrcOnRead),
      _erasedCount(0),
      _erasedBytes(0),
      _diskFootprint(0),
      _sumNumBuckets(0),
      _numChunksWithBuckets(0),
      _numUniqueBuckets(0),
      _file(),
      _bucketizer(bucketizer),
      _addedBytes(0),
      _tune(tune),
      _dataFileName(createDatFileName(_name)),
      _idxFileName(createIdxFileName(_name)),
      _chunkInfo(),
      _dataHeaderLen(0u),
      _idxHeaderLen(0u),
      _lastPersistedSerialNum(0),
      _docIdLimit(std::numeric_limits<uint32_t>::max()),
      _modificationTime()
{
    FastOS_File dataFile(_dataFileName.c_str());
    if (dataFile.OpenReadOnly()) {
        if (!dataFile.Sync()) {
            throw SummaryException("Failed syncing dat file", dataFile, VESPA_STRLOC);
        }
        _diskFootprint += dataFile.GetSize();
        FastOS_File idxFile(_idxFileName.c_str());
        if (idxFile.OpenReadOnly()) {
            if (!idxFile.Sync()) {
                throw SummaryException("Failed syncing idx file", idxFile, VESPA_STRLOC);
            }
            _diskFootprint += idxFile.GetSize();
            _modificationTime = FileKit::getModificationTime(_idxFileName);
        } else {
            dataFile.Close();
            throw SummaryException("Failed opening idx file", idxFile, VESPA_STRLOC);
        }
    }
}

FileChunk::~FileChunk() { }

void
FileChunk::addNumBuckets(size_t numBucketsInChunk)
{
    _sumNumBuckets += numBucketsInChunk;
    if (numBucketsInChunk != 0) {
        ++_numChunksWithBuckets;
    }
}

class TmpChunkMeta : public ChunkMeta,
                     public std::vector<LidMeta>
{
public:
    void fill(vespalib::nbostream & is) {
        resize(getNumEntries());
        for (LidMeta & lm : *this) {
            lm.deserialize(is);
        }
    }
};

typedef vespalib::Array<TmpChunkMeta> TmpChunkMetaV;

namespace {

void
verifyOrAssert(const TmpChunkMetaV & v)
{
    for (auto prev(v.begin()), it(prev); it != v.end(); ++it) {
        assert(prev->getLastSerial() <= it->getLastSerial());
        prev = it;
    }
}

vespalib::string eraseErrorMsg(const vespalib::string & fileName, int error) {
    return make_string("Error erasing file '%s'. Error is '%s'",
                       fileName.c_str(), getErrorString(error).c_str());
}

}

void
FileChunk::erase()
{
    _file.reset();
    if (!FastOS_File::Delete(_idxFileName.c_str()) && (errno != ENOENT)) {
        throw std::runtime_error(eraseErrorMsg(_idxFileName, errno));
    }
    if (!FastOS_File::Delete(_dataFileName.c_str()) && (errno != ENOENT)) {
        throw std::runtime_error(eraseErrorMsg(_dataFileName, errno));
    }
}

size_t
FileChunk::updateLidMap(const LockGuard &guard, ISetLid &ds, uint64_t serialNum, uint32_t docIdLimit)
{
    size_t sz(0);
    assert(_chunkInfo.empty());

    FastOS_File idxFile(_idxFileName.c_str());
    idxFile.enableMemoryMap(0);
    if (idxFile.OpenReadOnly()) {
        if (idxFile.IsMemoryMapped()) {
            const int64_t fileSize = idxFile.GetSize();
            if (_idxHeaderLen == 0) {
                _idxHeaderLen = readIdxHeader(idxFile, _docIdLimit);
            }
            vespalib::nbostream is(static_cast<const char *>(idxFile.MemoryMapPtr(0)) + _idxHeaderLen,
                                   fileSize - _idxHeaderLen);
            TmpChunkMetaV tempVector;
            tempVector.reserve(fileSize/(sizeof(ChunkMeta)+sizeof(LidMeta)));
            while ( ! is.empty() && is.good()) {
                const int64_t lastKnownGoodPos = _idxHeaderLen + is.rp();
                tempVector.push_back(TmpChunkMeta());
                TmpChunkMeta & chunkMeta(tempVector.back());
                try {
                    chunkMeta.deserialize(is);
                    chunkMeta.fill(is);
                } catch (const vespalib::IllegalStateException & e) {
                    LOG(warning, "Exception deserializing idx file : %s", e.what());
                    LOG(warning, "File '%s' seems to be partially truncated. Will truncate from size=%ld to %ld",
                                 _idxFileName.c_str(), fileSize, lastKnownGoodPos);
                    FastOS_File toTruncate(_idxFileName.c_str());
                    if ( toTruncate.OpenReadWrite()) {
                        if (toTruncate.SetSize(lastKnownGoodPos)) {
                            tempVector.resize(tempVector.size() - 1);
                        } else {
                            throw SummaryException("SetSize(%ld) failed.", toTruncate, VESPA_STRLOC);
                        }
                    } else {
                        throw SummaryException("Open for truncation failed.", toTruncate, VESPA_STRLOC);
                    }
                    break;
                }
            }
            if ( ! tempVector.empty()) {
                verifyOrAssert(tempVector);
                if (tempVector[0].getLastSerial() < serialNum) {
                    LOG(warning,
                        "last serial num(%ld) from previous file is "
                        "bigger than my first(%ld). That is odd."
                        "Current filename is '%s'",
                        serialNum, tempVector[0].getLastSerial(),
                        _idxFileName.c_str());
                    serialNum = tempVector[0].getLastSerial();
                }
                BucketDensityComputer globalBucketMap(_bucketizer);
                // Guard comes from the same bucketizer so the same guard can be used
                // for both local and global BucketDensityComputer
                vespalib::GenerationHandler::Guard bucketizerGuard = globalBucketMap.getGuard();
                for (const TmpChunkMeta & chunkMeta : tempVector) {
                    assert(serialNum <= chunkMeta.getLastSerial());
{'fraction_non_alphanumeric': 0.11936936936936937, 'fraction_numerical': 0.006624271330153683, 'mean_word_length': 2.4858000461787118, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7244460', 'n_tokens_mistral': 4680, 'n_tokens_neox': 4516, 'n_words': 850}