<commit_before>/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.prestosql.spi.block;

import io.airlift.slice.Slice;
import io.airlift.slice.SliceInput;
import io.airlift.slice.SliceOutput;
import io.airlift.slice.Slices;

import static io.prestosql.spi.block.EncoderUtil.decodeNullBits;
import static io.prestosql.spi.block.EncoderUtil.encodeNullsAsBits;

public class Int96ArrayBlockEncoding
        implements BlockEncoding
{
    public static final String NAME = "INT96_ARRAY";

    @Override
    public String getName()
    {
        return NAME;
    }

    @Override
    public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block)
    {
        int positionCount = block.getPositionCount();
        sliceOutput.appendInt(positionCount);

        encodeNullsAsBits(sliceOutput, block);

        if (!block.mayHaveNull()) {
            sliceOutput.writeBytes(getHighSlice(block));
            sliceOutput.writeBytes(getLowSlice(block));
        }
        else {
            long[] high = new long[positionCount];
            int[] low = new int[positionCount];
            int nonNullPositionCount = 0;
            for (int i = 0; i < positionCount; i++) {
                high[nonNullPositionCount] = block.getLong(i, 0);
                low[nonNullPositionCount] = block.getInt(i, 8);
                if (!block.isNull(i)) {
                    nonNullPositionCount++;
                }
            }

            sliceOutput.writeInt(nonNullPositionCount);
            sliceOutput.writeBytes(Slices.wrappedLongArray(high, 0, nonNullPositionCount));
            sliceOutput.writeBytes(Slices.wrappedIntArray(low, 0, nonNullPositionCount));
        }
    }

    @Override
    public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput)
    {
        int positionCount = sliceInput.readInt();

        boolean[] valueIsNull = decodeNullBits(sliceInput, positionCount).orElse(null);

        long[] high = new long[positionCount];
        int[] low = new int[positionCount];
        if (valueIsNull == null) {
            sliceInput.readBytes(Slices.wrappedLongArray(high));
            sliceInput.readBytes(Slices.wrappedIntArray(low));
        }
        else {
            int nonNullPositionCount = sliceInput.readInt();
            sliceInput.readBytes(Slices.wrappedLongArray(high, 0, nonNullPositionCount));
            sliceInput.readBytes(Slices.wrappedIntArray(low, 0, nonNullPositionCount));
            int position = nonNullPositionCount - 1;
            for (int i = positionCount - 1; i >= 0 && position >= 0; i--) {
                high[i] = high[position];
                low[i] = low[position];
                if (!valueIsNull[i]) {
                    position--;
                }
            }
        }

        return new Int96ArrayBlock(0, positionCount, valueIsNull, high, low);
    }

    private Slice getHighSlice(Block block)
    {
        if (block instanceof Int96ArrayBlock) {
            return ((Int96ArrayBlock) block).getHighSlice();
        }
        else if (block instanceof Int96ArrayBlockBuilder) {
            return ((Int96ArrayBlockBuilder) block).getHighSlice();
        }

        throw new IllegalArgumentException("Unexpected block type " + block.getClass().getSimpleName());
    }

    private Slice getLowSlice(Block block)
    {
        if (block instanceof Int96ArrayBlock) {
            return ((Int96ArrayBlock) block).getLowSlice();
        }
        else if (block instanceof Int96ArrayBlockBuilder) {<commit_msg>Improve deserialization performance of int 96 blocks<commit_after>/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.prestosql.spi.block;

import io.airlift.slice.Slice;
import io.airlift.slice.SliceInput;
import io.airlift.slice.SliceOutput;
import io.airlift.slice.Slices;

import static io.prestosql.spi.block.EncoderUtil.decodeNullBits;
import static io.prestosql.spi.block.EncoderUtil.encodeNullsAsBits;
import static io.prestosql.spi.block.EncoderUtil.retrieveNullBits;
import static java.lang.System.arraycopy;

public class Int96ArrayBlockEncoding
        implements BlockEncoding
{
    public static final String NAME = "INT96_ARRAY";

    @Override
    public String getName()
    {
        return NAME;
    }

    @Override
    public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block)
    {
        int positionCount = block.getPositionCount();
        sliceOutput.appendInt(positionCount);

        encodeNullsAsBits(sliceOutput, block);

        if (!block.mayHaveNull()) {
            sliceOutput.writeBytes(getHighSlice(block));
            sliceOutput.writeBytes(getLowSlice(block));
        }
        else {
            long[] high = new long[positionCount];
            int[] low = new int[positionCount];
            int nonNullPositionCount = 0;
            for (int i = 0; i < positionCount; i++) {
                high[nonNullPositionCount] = block.getLong(i, 0);
                low[nonNullPositionCount] = block.getInt(i, 8);
                if (!block.isNull(i)) {
                    nonNullPositionCount++;
                }
            }

            sliceOutput.writeInt(nonNullPositionCount);
            sliceOutput.writeBytes(Slices.wrappedLongArray(high, 0, nonNullPositionCount));
            sliceOutput.writeBytes(Slices.wrappedIntArray(low, 0, nonNullPositionCount));
        }
    }

    @Override
    public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput)
    {
        int positionCount = sliceInput.readInt();

        byte[] valueIsNullPacked = retrieveNullBits(sliceInput, positionCount);
        long[] high = new long[positionCount];
        int[] low = new int[positionCount];

        if (valueIsNullPacked == null) {
            sliceInput.readBytes(Slices.wrappedLongArray(high));
            sliceInput.readBytes(Slices.wrappedIntArray(low));
            return new Int96ArrayBlock(0, positionCount, null, high, low);
        }
        boolean[] valueIsNull = decodeNullBits(valueIsNullPacked, positionCount);

        int nonNullPositionCount = sliceInput.readInt();
        sliceInput.readBytes(Slices.wrappedLongArray(high, 0, nonNullPositionCount));
        sliceInput.readBytes(Slices.wrappedIntArray(low, 0, nonNullPositionCount));
        int position = nonNullPositionCount - 1;

        // Handle Last (positionCount % 8) values
        for (int i = positionCount - 1; i >= (positionCount & ~0b111) && position >= 0; i--) {
            high[i] = high[position];
            low[i] = low[position];
            if (!valueIsNull[i]) {
                position--;
            }
        }

        // Handle the remaining positions.
        for (int i = (positionCount & ~0b111) - 8; i >= 0 && position >= 0; i -= 8) {
            byte packed = valueIsNullPacked[i >> 3];
            if (packed == 0) { // Only values
                arraycopy(high, position - 7, high, i, 8);
                arraycopy(low, position - 7, low, i, 8);
                position -= 8;
            }
            else if (packed != -1) { // At least one non-null
                for (int j = i + 7; j >= i && position >= 0; j--) {
                    high[j] = high[position];
                    low[j] = low[position];
                    if (!valueIsNull[j]) {
                        position--;
                    }
                }
            }
            // Do nothing if there are only nulls
        }
        return new Int96ArrayBlock(0, positionCount, valueIsNull, high, low);
    }

    private Slice getHighSlice(Block block)
    {
        if (block instanceof Int96ArrayBlock) {
            return ((Int96ArrayBlock) block).getHighSlice();
        }
        else if (block instanceof Int96ArrayBlockBuilder) {
            return ((Int96ArrayBlockBuilder) block).getHighSlice();
        }

        throw new IllegalArgumentException("Unexpected block type " + block.getClass().getSimpleName());
    }

    private Slice getLowSlice(Block block)
    {
        if (block instanceof Int96ArrayBlock) {
            return ((Int96ArrayBlock) block).getLowSlice();
        }
        else if (block instanceof Int96ArrayBlockBuilder) {
{'fraction_non_alphanumeric': 0.09767190037899297, 'fraction_numerical': 0.010936654033567948, 'mean_word_length': 2.408118081180812, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5398551', 'n_tokens_mistral': 2573, 'n_tokens_neox': 2461, 'n_words': 643}