<commit_before>package com.linkedin.pinot.core.segment.index.data.source.mv.block;

import java.util.List;

import com.linkedin.pinot.common.data.FieldSpec.DataType;
import com.linkedin.pinot.core.common.Block;
import com.linkedin.pinot.core.common.BlockDocIdIterator;
import com.linkedin.pinot.core.common.BlockDocIdSet;
import com.linkedin.pinot.core.common.BlockDocIdValueSet;
import com.linkedin.pinot.core.common.BlockId;
import com.linkedin.pinot.core.common.BlockMetadata;
import com.linkedin.pinot.core.common.BlockMultiValIterator;
import com.linkedin.pinot.core.common.BlockValIterator;
import com.linkedin.pinot.core.common.BlockValSet;
import com.linkedin.pinot.core.common.Constants;
import com.linkedin.pinot.core.common.Predicate;
import com.linkedin.pinot.core.segment.index.ColumnMetadata;
import com.linkedin.pinot.core.segment.index.block.BlockUtils;
import com.linkedin.pinot.core.segment.index.data.source.DictionaryIdFilterUtils;
import com.linkedin.pinot.core.segment.index.readers.Dictionary;
import com.linkedin.pinot.core.segment.index.readers.FixedBitCompressedMVForwardIndexReader;
import com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader;


public class MultiValueBlockWithoutInvertedIndex implements Block {

  private final FixedBitCompressedMVForwardIndexReader indexReader;
  private final BlockId id;
  private final ImmutableDictionaryReader dictionary;
  private final ColumnMetadata columnMetadata;
  private Predicate predicate;

  public MultiValueBlockWithoutInvertedIndex(BlockId id, FixedBitCompressedMVForwardIndexReader multiValueReader,
      ImmutableDictionaryReader dict, ColumnMetadata metadata) {
    indexReader = multiValueReader;
    this.id = id;
    dictionary = dict;
    columnMetadata = metadata;
  }

  @Override
  public BlockId getId() {
    return id;
  }

  @Override
  public boolean applyPredicate(Predicate predicate) {
    this.predicate = predicate;
    return true;
  }

  @Override
  public BlockDocIdSet getBlockDocIdSet() {
    if (predicate == null) {
      return BlockUtils.getDummyBlockDocIdSet(columnMetadata.getTotalDocs());
    }

    return new BlockDocIdSet() {

      @Override
      public BlockDocIdIterator iterator() {
        return new BlockDocIdIterator() {
          private final List<Integer> filteredIds = DictionaryIdFilterUtils.filter(predicate, dictionary);
          int counter = -1;

          @Override
          public int skipTo(int targetDocId) {
            if (targetDocId >= columnMetadata.getTotalDocs()) {
              return Constants.EOF;
            }
            counter = targetDocId;
            int[] vals = new int[columnMetadata.getMaxNumberOfMultiValues()];
            int len = indexReader.getIntArray(counter, vals);
            for (int i = 0; i < len; i++) {
              if (filteredIds.contains(vals[i])) {
                return counter;
              }
            }
            while (counter < columnMetadata.getTotalDocs()) {
              counter++;
              int[] mval = new int[columnMetadata.getMaxNumberOfMultiValues()];
              int rlen = indexReader.getIntArray(counter, mval);
              for (int i = 0; i < rlen; i++) {
                if (filteredIds.contains(mval[i])) {
                  break;
                }
              }
            }
            return counter;
          }

          @Override
          public int next() {
            counter++;
            while (counter < columnMetadata.getTotalDocs()) {
              int[] mval = new int[columnMetadata.getMaxNumberOfMultiValues()];
              int rlen = indexReader.getIntArray(counter, mval);
              for (int i = 0; i < rlen; i++) {
                if (filteredIds.contains(mval[i])) {
                  break;
                }
              }
              counter++;<commit_msg>single and multi value blocks without inverted index changes<commit_after>package com.linkedin.pinot.core.segment.index.data.source.mv.block;

import java.util.List;

import com.linkedin.pinot.common.data.FieldSpec.DataType;
import com.linkedin.pinot.core.common.Block;
import com.linkedin.pinot.core.common.BlockDocIdIterator;
import com.linkedin.pinot.core.common.BlockDocIdSet;
import com.linkedin.pinot.core.common.BlockDocIdValueSet;
import com.linkedin.pinot.core.common.BlockId;
import com.linkedin.pinot.core.common.BlockMetadata;
import com.linkedin.pinot.core.common.BlockMultiValIterator;
import com.linkedin.pinot.core.common.BlockValIterator;
import com.linkedin.pinot.core.common.BlockValSet;
import com.linkedin.pinot.core.common.Constants;
import com.linkedin.pinot.core.common.Predicate;
import com.linkedin.pinot.core.segment.index.ColumnMetadata;
import com.linkedin.pinot.core.segment.index.block.BlockUtils;
import com.linkedin.pinot.core.segment.index.data.source.DictionaryIdFilterUtils;
import com.linkedin.pinot.core.segment.index.readers.Dictionary;
import com.linkedin.pinot.core.segment.index.readers.FixedBitCompressedMVForwardIndexReader;
import com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader;


public class MultiValueBlockWithoutInvertedIndex implements Block {

  private final FixedBitCompressedMVForwardIndexReader indexReader;
  private final BlockId id;
  private final ImmutableDictionaryReader dictionary;
  private final ColumnMetadata columnMetadata;
  private Predicate predicate;

  public MultiValueBlockWithoutInvertedIndex(BlockId id, FixedBitCompressedMVForwardIndexReader multiValueReader,
      ImmutableDictionaryReader dict, ColumnMetadata metadata) {
    indexReader = multiValueReader;
    this.id = id;
    dictionary = dict;
    columnMetadata = metadata;
  }

  @Override
  public BlockId getId() {
    return id;
  }

  @Override
  public boolean applyPredicate(Predicate predicate) {
    this.predicate = predicate;
    return true;
  }

  @Override
  public BlockDocIdSet getBlockDocIdSet() {
    if (predicate == null) {
      return BlockUtils.getDummyBlockDocIdSet(columnMetadata.getTotalDocs());
    }

    return new BlockDocIdSet() {

      @Override
      public BlockDocIdIterator iterator() {
        return new BlockDocIdIterator() {
          private final List<Integer> filteredIds = DictionaryIdFilterUtils.filter(predicate, dictionary);
          int counter = -1;

          @Override
          public int skipTo(int targetDocId) {
            if (targetDocId >= columnMetadata.getTotalDocs()) {
              return Constants.EOF;
            }
            counter = targetDocId - 1;
            return next();
          }

          @Override
          public int next() {
            counter++;
            if (counter >= columnMetadata.getTotalDocs()) {
              return Constants.EOF;
            }
            while (counter < columnMetadata.getTotalDocs()) {
              int[] mval = new int[columnMetadata.getMaxNumberOfMultiValues()];
              int rlen = indexReader.getIntArray(counter, mval);
              for (int i = 0; i < rlen; i++) {
                if (filteredIds.contains(mval[i])) {
                  break;
                }
              }
              counter++;
{'fraction_non_alphanumeric': 0.09262892376681614, 'fraction_numerical': 0.0009809417040358745, 'mean_word_length': 2.962798445308162, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '266007', 'n_tokens_mistral': 2029, 'n_tokens_neox': 1978, 'n_words': 384}