<commit_before>/////////////////////////////////////////////////////////////////////////////
// Copyright (c) 2009-2010 <NAME>. All rights reserved.
// Distributable under the terms of either the Apache License (Version 2.0)
// or the GNU Lesser General Public License.
/////////////////////////////////////////////////////////////////////////////

#include "LuceneInc.h"
#include "FreqProxTermsWriterPerField.h"
#include "FreqProxTermsWriter.h"
#include "FieldInfo.h"
#include "Fieldable.h"
#include "TermsHashPerField.h"
#include "FieldInvertState.h"
#include "AttributeSource.h"
#include "Payload.h"
#include "PayloadAttribute.h"
#include "DocumentsWriter.h"
#include "RawPostingList.h"

namespace Lucene
{
    FreqProxTermsWriterPerField::FreqProxTermsWriterPerField(TermsHashPerFieldPtr termsHashPerField, FreqProxTermsWriterPerThreadPtr perThread, FieldInfoPtr fieldInfo)
    {
        this->hasPayloads = false;
        this->_termsHashPerField = termsHashPerField;
        this->_perThread = perThread;
        this->fieldInfo = fieldInfo;
        docState = termsHashPerField->docState;
        fieldState = termsHashPerField->fieldState;
        omitTermFreqAndPositions = fieldInfo->omitTermFreqAndPositions;
    }
    
    FreqProxTermsWriterPerField::~FreqProxTermsWriterPerField()
    {
    }
    
    int32_t FreqProxTermsWriterPerField::getStreamCount()
    {
        return fieldInfo->omitTermFreqAndPositions ? 1 : 2;
    }
    
    void FreqProxTermsWriterPerField::finish()
    {
    }
    
    void FreqProxTermsWriterPerField::skippingLongTerm()
    {
    }
    
    int32_t FreqProxTermsWriterPerField::compareTo(LuceneObjectPtr other)
    {
        return fieldInfo->name.compare(boost::static_pointer_cast<FreqProxTermsWriterPerField>(other)->fieldInfo->name);
    }
    
    void FreqProxTermsWriterPerField::reset()
    {
        // Record, up front, whether our in-RAM format will be with or without term freqs
        omitTermFreqAndPositions = fieldInfo->omitTermFreqAndPositions;
        payloadAttribute.reset();
    }
    
    bool FreqProxTermsWriterPerField::start(Collection<FieldablePtr> fields, int32_t count)
    {
        for (int32_t i = 0; i < count; ++i)
        {
            if (fields[i]->isIndexed())
                return true;
        }
        return false;
    }
    
    void FreqProxTermsWriterPerField::start(FieldablePtr field)
    {
        if (fieldState->attributeSource->hasAttribute<PayloadAttribute>())
            payloadAttribute = fieldState->attributeSource->getAttribute<PayloadAttribute>();
        else
            payloadAttribute.reset();
    }
    
    void FreqProxTermsWriterPerField::writeProx(FreqProxTermsWriterPostingListPtr p, int32_t proxCode)
    {
        PayloadPtr payload;
        if (payloadAttribute)
            payload = payloadAttribute->getPayload();
        
        TermsHashPerFieldPtr termsHashPerField(_termsHashPerField);
        
        if (payload && payload->length() > 0)
        {
            termsHashPerField->writeVInt(1, (proxCode << 1) | 1);
            termsHashPerField->writeVInt(1, payload->length());
            termsHashPerField->writeBytes(1, payload->getData().get(), payload->getOffset(), payload->length());
            hasPayloads = true;
        }
        else
            termsHashPerField->writeVInt(1, proxCode << 1);
        p->lastPosition = fieldState->position;
    }
    
    void FreqProxTermsWriterPerField::newTerm(RawPostingListPtr p)
    {
        // First time we're seeing this term since the last flush
        bool testPoint = docState->testPoint(L"FreqProxTermsWriterPerField.newTerm start");
        BOOST_ASSERT(testPoint);
        FreqProxTermsWriterPostingListPtr newPostingList(boost::static_pointer_cast<FreqProxTermsWriterPostingList>(p));
        newPostingList->lastDocID = docState->docID;
        if (omitTermFreqAndPositions)
            newPostingList->lastDocCode = docState->docID;
        else
        {
            newPostingList->lastDocCode = docState->docID << 1;
            newPostingList->docFreq = 1;
            writeProx(newPostingList, fieldState->position);
        }
    }
    
    void FreqProxTermsWriterPerField::addTerm(RawPostingListPtr p)
    {
        bool testPoint = docState->testPoint(L"FreqProxTermsWriterPerField.addTerm start");
        BOOST_ASSERT(testPoint);
        
        FreqProxTermsWriterPostingListPtr addPostingList(boost::static_pointer_cast<FreqProxTermsWriterPostingList>(p));
        
        BOOST_ASSERT(omitTermFreqAndPositions || addPostingList->docFreq > 0);
        TermsHashPerFieldPtr termsHashPerField(_termsHashPerField);
        
        if (omitTermFreqAndPositions)
        {
            if (docState->docID != addPostingList->lastDocID)
            {
                BOOST_ASSERT(docState->docID > addPostingList->lastDocID);
                termsHashPerField->writeVInt(0, addPostingList->lastDocCode);
                addPostingList->lastDocCode = docState->docID - addPostingList->lastDocID;
                addPostingList->lastDocID = docState->docID;
            }
        }
        else
        {
            if (docState->docID != addPostingList->lastDocID)
            {
                BOOST_ASSERT(docState->docID > addPostingList->lastDocID);
                // Term not yet seen in the current doc but previously seen in other doc(s) since 
                // the last flush
                
                // Now that we know doc freq for previous doc, write it & lastDocCode
                if (addPostingList->docFreq == 1)
                    termsHashPerField->writeVInt(0, addPostingList->lastDocCode | 1);
                else<commit_msg>Remove unnecessary test points in release builds<commit_after>/////////////////////////////////////////////////////////////////////////////
// Copyright (c) 2009-2010 <NAME>. All rights reserved.
// Distributable under the terms of either the Apache License (Version 2.0)
// or the GNU Lesser General Public License.
/////////////////////////////////////////////////////////////////////////////

#include "LuceneInc.h"
#include "FreqProxTermsWriterPerField.h"
#include "FreqProxTermsWriter.h"
#include "FieldInfo.h"
#include "Fieldable.h"
#include "TermsHashPerField.h"
#include "FieldInvertState.h"
#include "AttributeSource.h"
#include "Payload.h"
#include "PayloadAttribute.h"
#include "DocumentsWriter.h"
#include "RawPostingList.h"

namespace Lucene
{
    FreqProxTermsWriterPerField::FreqProxTermsWriterPerField(TermsHashPerFieldPtr termsHashPerField, FreqProxTermsWriterPerThreadPtr perThread, FieldInfoPtr fieldInfo)
    {
        this->hasPayloads = false;
        this->_termsHashPerField = termsHashPerField;
        this->_perThread = perThread;
        this->fieldInfo = fieldInfo;
        docState = termsHashPerField->docState;
        fieldState = termsHashPerField->fieldState;
        omitTermFreqAndPositions = fieldInfo->omitTermFreqAndPositions;
    }
    
    FreqProxTermsWriterPerField::~FreqProxTermsWriterPerField()
    {
    }
    
    int32_t FreqProxTermsWriterPerField::getStreamCount()
    {
        return fieldInfo->omitTermFreqAndPositions ? 1 : 2;
    }
    
    void FreqProxTermsWriterPerField::finish()
    {
    }
    
    void FreqProxTermsWriterPerField::skippingLongTerm()
    {
    }
    
    int32_t FreqProxTermsWriterPerField::compareTo(LuceneObjectPtr other)
    {
        return fieldInfo->name.compare(boost::static_pointer_cast<FreqProxTermsWriterPerField>(other)->fieldInfo->name);
    }
    
    void FreqProxTermsWriterPerField::reset()
    {
        // Record, up front, whether our in-RAM format will be with or without term freqs
        omitTermFreqAndPositions = fieldInfo->omitTermFreqAndPositions;
        payloadAttribute.reset();
    }
    
    bool FreqProxTermsWriterPerField::start(Collection<FieldablePtr> fields, int32_t count)
    {
        for (int32_t i = 0; i < count; ++i)
        {
            if (fields[i]->isIndexed())
                return true;
        }
        return false;
    }
    
    void FreqProxTermsWriterPerField::start(FieldablePtr field)
    {
        if (fieldState->attributeSource->hasAttribute<PayloadAttribute>())
            payloadAttribute = fieldState->attributeSource->getAttribute<PayloadAttribute>();
        else
            payloadAttribute.reset();
    }
    
    void FreqProxTermsWriterPerField::writeProx(FreqProxTermsWriterPostingListPtr p, int32_t proxCode)
    {
        PayloadPtr payload;
        if (payloadAttribute)
            payload = payloadAttribute->getPayload();
        
        TermsHashPerFieldPtr termsHashPerField(_termsHashPerField);
        
        if (payload && payload->length() > 0)
        {
            termsHashPerField->writeVInt(1, (proxCode << 1) | 1);
            termsHashPerField->writeVInt(1, payload->length());
            termsHashPerField->writeBytes(1, payload->getData().get(), payload->getOffset(), payload->length());
            hasPayloads = true;
        }
        else
            termsHashPerField->writeVInt(1, proxCode << 1);
        p->lastPosition = fieldState->position;
    }
    
    void FreqProxTermsWriterPerField::newTerm(RawPostingListPtr p)
    {
        // First time we're seeing this term since the last flush
        BOOST_ASSERT(docState->testPoint(L"FreqProxTermsWriterPerField.newTerm start"));
        FreqProxTermsWriterPostingListPtr newPostingList(boost::static_pointer_cast<FreqProxTermsWriterPostingList>(p));
        newPostingList->lastDocID = docState->docID;
        if (omitTermFreqAndPositions)
            newPostingList->lastDocCode = docState->docID;
        else
        {
            newPostingList->lastDocCode = docState->docID << 1;
            newPostingList->docFreq = 1;
            writeProx(newPostingList, fieldState->position);
        }
    }
    
    void FreqProxTermsWriterPerField::addTerm(RawPostingListPtr p)
    {
        BOOST_ASSERT(docState->testPoint(L"FreqProxTermsWriterPerField.addTerm start"));
        
        FreqProxTermsWriterPostingListPtr addPostingList(boost::static_pointer_cast<FreqProxTermsWriterPostingList>(p));
        
        BOOST_ASSERT(omitTermFreqAndPositions || addPostingList->docFreq > 0);
        TermsHashPerFieldPtr termsHashPerField(_termsHashPerField);
        
        if (omitTermFreqAndPositions)
        {
            if (docState->docID != addPostingList->lastDocID)
            {
                BOOST_ASSERT(docState->docID > addPostingList->lastDocID);
                termsHashPerField->writeVInt(0, addPostingList->lastDocCode);
                addPostingList->lastDocCode = docState->docID - addPostingList->lastDocID;
                addPostingList->lastDocID = docState->docID;
            }
        }
        else
        {
            if (docState->docID != addPostingList->lastDocID)
            {
                BOOST_ASSERT(docState->docID > addPostingList->lastDocID);
                // Term not yet seen in the current doc but previously seen in other doc(s) since 
                // the last flush
                
                // Now that we know doc freq for previous doc, write it & lastDocCode
                if (addPostingList->docFreq == 1)
                    termsHashPerField->writeVInt(0, addPostingList->lastDocCode | 1);
                else
{'fraction_non_alphanumeric': 0.11784481995930284, 'fraction_numerical': 0.006723878616296558, 'mean_word_length': 2.4558239070620607, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 141, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6263194', 'n_tokens_mistral': 3355, 'n_tokens_neox': 3240, 'n_words': 549}