<commit_before>package com.pearson.entech.elasticsearch.search.facet.approx.date.collectors;

import java.io.IOException;

import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.BytesValues.Iter;
import org.elasticsearch.index.fielddata.BytesValues.WithOrdinals;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.ScriptDocValues;

public class BytesFieldIterator<B extends AtomicFieldData<? extends ScriptDocValues>>
        extends CollectableIterator<BytesRef> {

    private final IndexFieldData<B> _bytesFieldData;
    private BytesValues _bytesFieldValues;
    private IntsRef _docOrds;
    private int _docOrdPointer;
    private Iter _docIter;

    public BytesFieldIterator(
            final IndexFieldData<B> bytesFieldData) {
        _bytesFieldData = bytesFieldData;
    }

    @Override
    public void collect(final int doc) throws IOException {
        if(_bytesFieldValues instanceof WithOrdinals) {
            _docOrds = ((WithOrdinals) _bytesFieldValues).ordinals().getOrds(doc);
            _docOrdPointer = _docOrds.offset;
        } else {
            _docIter = _bytesFieldValues.getIter(doc);
        }
    }

    @Override
    public void setNextReader(final AtomicReaderContext context) throws IOException {
        _bytesFieldValues = _bytesFieldData.load(context).getBytesValues();
    }

    @Override
    public boolean hasNext() {
        if(_bytesFieldValues instanceof WithOrdinals) {
            return _docOrdPointer < _docOrds.length;
        } else {
            return _docIter.hasNext();
        }
    }

    @Override
    public BytesRef next() {
        if(_bytesFieldValues instanceof WithOrdinals) {
            final BytesRef unsafe = ((WithOrdinals) _bytesFieldValues).getValueByOrd(_docOrds.ints[_docOrdPointer]);
            _docOrdPointer++;
            return unsafe;
        } else {
            return _docIter.next();
        }
    }

    @Override
    public void postCollection() {
        _docOrds = null;
        _docOrdPointer = -1;
        _docIter = null;
    }

}
<commit_msg>Special handling of ordinals is no longer helpful - removed<commit_after>package com.pearson.entech.elasticsearch.search.facet.approx.date.collectors;

import java.io.IOException;

import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.BytesValues.Iter;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.ScriptDocValues;

public class BytesFieldIterator<B extends AtomicFieldData<? extends ScriptDocValues>>
        extends CollectableIterator<BytesRef> {

    private final IndexFieldData<B> _bytesFieldData;
    private BytesValues _bytesFieldValues;
    private IntsRef _docOrds;
    private int _docOrdPointer;
    private Iter _docIter;

    public BytesFieldIterator(
            final IndexFieldData<B> bytesFieldData) {
        _bytesFieldData = bytesFieldData;
    }

    @Override
    public void collect(final int doc) throws IOException {
        _docIter = _bytesFieldValues.getIter(doc);
    }

    @Override
    public void setNextReader(final AtomicReaderContext context) throws IOException {
        _bytesFieldValues = _bytesFieldData.load(context).getBytesValues();
    }

    @Override
    public boolean hasNext() {
        return _docIter.hasNext();
    }

    @Override
    public BytesRef next() {
        return _docIter.next();
    }

    @Override
    public void postCollection() {
        _docIter = null;
    }

}

{'fraction_non_alphanumeric': 0.0813421453990849, 'fraction_numerical': 0.0002541942043721403, 'mean_word_length': 3.406494960806271, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3234441', 'n_tokens_mistral': 1117, 'n_tokens_neox': 1104, 'n_words': 222}