<commit_before>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#pragma once

#include "fileconfigmanager.h"
#include "ifeedview.h"
#include "isummaryadapter.h"
#include "replaypacketdispatcher.h"
#include "searchcontext.h"
#include "tlcproxy.h"
#include <vespa/searchcore/proton/common/doctypename.h>
#include <vespa/searchcore/proton/common/feeddebugger.h>
#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h>
#include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h>
#include <vespa/searchcore/proton/feedoperation/feedoperation.h>
#include <vespa/searchcore/proton/persistenceengine/resulthandler.h>
#include <vespa/searchcorespi/index/ithreadingservice.h>
#include <vespa/searchlib/query/base.h>
#include <vespa/vespalib/util/threadstackexecutorbase.h>

namespace search { class IDestructorCallback; }

namespace proton {

class IReplayConfig;
class PerDocTypeFeedMetrics;
class ForceCommitContext;
class OperationDoneContext;
class PutDoneContext;
class RemoveDoneContext;
class CommitTimeTracker;

namespace documentmetastore { class ILidReuseDelayer; }


/**
 * The feed view used by the store-only sub database.
 *
 * Handles inserting/updating/removing of documents to the underlying document store.
 */
class StoreOnlyFeedView : public IFeedView,
                          protected FeedDebugger
{
protected:
    typedef search::transactionlog::Packet Packet;
public:
    typedef std::unique_ptr<StoreOnlyFeedView> UP;
    typedef std::shared_ptr<StoreOnlyFeedView> SP;
    typedef search::SerialNum SerialNum;
    typedef LidVectorContext::LidVector LidVector;
    using OnWriteDoneType =const std::shared_ptr<search::IDestructorCallback> &;
    using OnForceCommitDoneType =const std::shared_ptr<ForceCommitContext> &;
    using OnOperationDoneType = const std::shared_ptr<OperationDoneContext> &;
    using OnPutDoneType = const std::shared_ptr<PutDoneContext> &;
    using OnRemoveDoneType = const std::shared_ptr<RemoveDoneContext> &;
    using FeedTokenUP = std::unique_ptr<FeedToken>;

    struct Context
    {
        const ISummaryAdapter::SP               &_summaryAdapter;
        const search::index::Schema::SP         &_schema;
        const IDocumentMetaStoreContext::SP     &_documentMetaStoreContext;
        const document::DocumentTypeRepo::SP    &_repo;
        searchcorespi::index::IThreadingService &_writeService;
        documentmetastore::ILidReuseDelayer     &_lidReuseDelayer;
        CommitTimeTracker                       &_commitTimeTracker;

        Context(const ISummaryAdapter::SP &summaryAdapter,
                const search::index::Schema::SP &schema,
                const IDocumentMetaStoreContext::SP &documentMetaStoreContext,
                const document::DocumentTypeRepo::SP &repo,
                searchcorespi::index::IThreadingService &writeService,
                documentmetastore::ILidReuseDelayer &lidReuseDelayer,
                CommitTimeTracker &commitTimeTracker)
            : _summaryAdapter(summaryAdapter),
              _schema(schema),
              _documentMetaStoreContext(documentMetaStoreContext),
              _repo(repo),
              _writeService(writeService),
              _lidReuseDelayer(lidReuseDelayer),
              _commitTimeTracker(commitTimeTracker)
        {}
    };

    struct PersistentParams
    {
        const SerialNum        _flushedDocumentMetaStoreSerialNum;
        const SerialNum        _flushedDocumentStoreSerialNum;
        const DocTypeName      _docTypeName;
        PerDocTypeFeedMetrics &_metrics;
        const uint32_t         _subDbId;
        const SubDbType        _subDbType;

        PersistentParams(SerialNum flushedDocumentMetaStoreSerialNum,
                         SerialNum flushedDocumentStoreSerialNum,
                         const DocTypeName &docTypeName,
                         PerDocTypeFeedMetrics &metrics,
                         uint32_t subDbId,
                         SubDbType subDbType)
            : _flushedDocumentMetaStoreSerialNum(flushedDocumentMetaStoreSerialNum),
              _flushedDocumentStoreSerialNum(flushedDocumentStoreSerialNum),
              _docTypeName(docTypeName),
              _metrics(metrics),
              _subDbId(subDbId),
              _subDbType(subDbType)
        {}
    };

protected:
    struct UpdateScope
    {
        bool _indexedFields;
        bool _nonAttributeFields;

        UpdateScope()
            : _indexedFields(false),
              _nonAttributeFields(false)
        {}
        bool hasIndexOrNonAttributeFields() const {
            return _indexedFields || _nonAttributeFields;
        }
    };

protected:
    const ISummaryAdapter::SP                _summaryAdapter;
    const search::index::Schema::SP          _schema;
    const IDocumentMetaStoreContext::SP      _documentMetaStoreContext;
    const document::DocumentTypeRepo::SP     _repo;
    searchcorespi::index::IThreadingService &_writeService;
    PersistentParams                         _params;
    IDocumentMetaStore                      &_metaStore;
    const document::DocumentType            *_docType;
    documentmetastore::ILidReuseDelayer     &_lidReuseDelayer;
    CommitTimeTracker                       &_commitTimeTracker;

    bool useDocumentStore(SerialNum replaySerialNum) const {
        return replaySerialNum > _params._flushedDocumentStoreSerialNum;
    }

private:
    bool
    useDocumentMetaStore(SerialNum replaySerialNum) const
    {
        return replaySerialNum > _params._flushedDocumentMetaStoreSerialNum;
    }

    void adjustMetaStore(const DocumentOperation &op, const document::DocumentId &docId);
    void internalPut(FeedTokenUP token, const PutOperation &putOp);
    void internalUpdate(FeedTokenUP token, const UpdateOperation &updOp);

    bool lookupDocId(const document::DocumentId &gid, search::DocumentIdT & lid) const;
    void internalRemove(FeedTokenUP token, const RemoveOperation &rmOp);

    // Removes documents from meta store and document store.
    // returns the number of documents removed.
    size_t removeDocuments(const RemoveDocumentsOperation &op, bool remove_index_and_attribute_fields,
                           bool immediateCommit);
<commit_msg>Revert a premature style change.<commit_after>// Copyright 2017 <NAME>. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#pragma once

#include "fileconfigmanager.h"
#include "ifeedview.h"
#include "isummaryadapter.h"
#include "replaypacketdispatcher.h"
#include "searchcontext.h"
#include "tlcproxy.h"
#include <vespa/searchcore/proton/common/doctypename.h>
#include <vespa/searchcore/proton/common/feeddebugger.h>
#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h>
#include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h>
#include <vespa/searchcore/proton/feedoperation/feedoperation.h>
#include <vespa/searchcore/proton/persistenceengine/resulthandler.h>
#include <vespa/searchcorespi/index/ithreadingservice.h>
#include <vespa/searchlib/query/base.h>
#include <vespa/vespalib/util/threadstackexecutorbase.h>

namespace search { class IDestructorCallback; }

namespace proton {

class IReplayConfig;
class PerDocTypeFeedMetrics;
class ForceCommitContext;
class OperationDoneContext;
class PutDoneContext;
class RemoveDoneContext;
class CommitTimeTracker;

namespace documentmetastore { class ILidReuseDelayer; }


/**
 * The feed view used by the store-only sub database.
 *
 * Handles inserting/updating/removing of documents to the underlying document store.
 */
class StoreOnlyFeedView : public IFeedView,
                          protected FeedDebugger
{
protected:
    typedef search::transactionlog::Packet Packet;
public:
    typedef std::unique_ptr<StoreOnlyFeedView> UP;
    typedef std::shared_ptr<StoreOnlyFeedView> SP;
    typedef search::SerialNum SerialNum;
    typedef LidVectorContext::LidVector LidVector;
    using OnWriteDoneType =const std::shared_ptr<search::IDestructorCallback> &;
    using OnForceCommitDoneType =const std::shared_ptr<ForceCommitContext> &;
    using OnOperationDoneType = const std::shared_ptr<OperationDoneContext> &;
    using OnPutDoneType = const std::shared_ptr<PutDoneContext> &;
    using OnRemoveDoneType = const std::shared_ptr<RemoveDoneContext> &;
    using FeedTokenUP = std::unique_ptr<FeedToken>;

    struct Context
    {
        const ISummaryAdapter::SP               &_summaryAdapter;
        const search::index::Schema::SP         &_schema;
        const IDocumentMetaStoreContext::SP     &_documentMetaStoreContext;
        const document::DocumentTypeRepo::SP    &_repo;
        searchcorespi::index::IThreadingService &_writeService;
        documentmetastore::ILidReuseDelayer     &_lidReuseDelayer;
        CommitTimeTracker                       &_commitTimeTracker;

        Context(const ISummaryAdapter::SP &summaryAdapter,
                const search::index::Schema::SP &schema,
                const IDocumentMetaStoreContext::SP &documentMetaStoreContext,
                const document::DocumentTypeRepo::SP &repo,
                searchcorespi::index::IThreadingService &writeService,
                documentmetastore::ILidReuseDelayer &lidReuseDelayer,
                CommitTimeTracker &commitTimeTracker)
            : _summaryAdapter(summaryAdapter),
              _schema(schema),
              _documentMetaStoreContext(documentMetaStoreContext),
              _repo(repo),
              _writeService(writeService),
              _lidReuseDelayer(lidReuseDelayer),
              _commitTimeTracker(commitTimeTracker)
        {}
    };

    struct PersistentParams
    {
        const SerialNum        _flushedDocumentMetaStoreSerialNum;
        const SerialNum        _flushedDocumentStoreSerialNum;
        const DocTypeName      _docTypeName;
        PerDocTypeFeedMetrics &_metrics;
        const uint32_t         _subDbId;
        const SubDbType        _subDbType;

        PersistentParams(SerialNum flushedDocumentMetaStoreSerialNum,
                         SerialNum flushedDocumentStoreSerialNum,
                         const DocTypeName &docTypeName,
                         PerDocTypeFeedMetrics &metrics,
                         uint32_t subDbId,
                         SubDbType subDbType)
            : _flushedDocumentMetaStoreSerialNum(flushedDocumentMetaStoreSerialNum),
              _flushedDocumentStoreSerialNum(flushedDocumentStoreSerialNum),
              _docTypeName(docTypeName),
              _metrics(metrics),
              _subDbId(subDbId),
              _subDbType(subDbType)
        {}
    };

protected:
    struct UpdateScope
    {
        bool _indexedFields;
        bool _nonAttributeFields;

        UpdateScope()
            : _indexedFields(false),
              _nonAttributeFields(false)
        {}
        bool hasIndexOrNonAttributeFields() const {
            return _indexedFields || _nonAttributeFields;
        }
    };

protected:
    const ISummaryAdapter::SP                _summaryAdapter;
    const search::index::Schema::SP          _schema;
    const IDocumentMetaStoreContext::SP      _documentMetaStoreContext;
    const document::DocumentTypeRepo::SP     _repo;
    searchcorespi::index::IThreadingService &_writeService;
    PersistentParams                         _params;
    IDocumentMetaStore                      &_metaStore;
    const document::DocumentType            *_docType;
    documentmetastore::ILidReuseDelayer     &_lidReuseDelayer;
    CommitTimeTracker                       &_commitTimeTracker;

    bool useDocumentStore(SerialNum replaySerialNum) const {
        return replaySerialNum > _params._flushedDocumentStoreSerialNum;
    }

private:
    bool
    useDocumentMetaStore(SerialNum replaySerialNum) const
    {
        return replaySerialNum > _params._flushedDocumentMetaStoreSerialNum;
    }

    void adjustMetaStore(const DocumentOperation &op, const document::DocumentId &docId);
    void internalPut(FeedTokenUP token, const PutOperation &putOp);
    void internalUpdate(FeedTokenUP token, const UpdateOperation &updOp);

    void
    updateIndexAndDocumentStore(bool indexedFieldsInScope,
                                SerialNum serialNum,
                                search::DocumentIdT lid,
                                const document::DocumentUpdate &upd,
                                bool immediateCommit,
                                OnOperationDoneType onWriteDone);

    bool
    lookupDocId(const document::DocumentId &gid,
                search::DocumentIdT & lid) const;

    void
    internalRemove(FeedTokenUP token,
                   const RemoveOperation &rmOp);

    // Removes documents from meta store and document store.
    // returns the number of documents removed.
    size_t removeDocuments(const RemoveDocumentsOperation &op, bool remove_index_and_attribute_fields,
                           bool immediateCommit);

{'fraction_non_alphanumeric': 0.07087520135000383, 'fraction_numerical': 0.0015340952673161003, 'mean_word_length': 2.9318455971049455, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7249634', 'n_tokens_mistral': 3511, 'n_tokens_neox': 3414, 'n_words': 685}