<commit_before>/*
 * (c) <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>
 * Licensed under the Apache License, Version 2.0.
 */
package org.roaringbitmap.buffer;

import org.roaringbitmap.BitmapContainer;
import org.roaringbitmap.ShortIterator;
import org.roaringbitmap.Util;

import java.io.*;
import java.nio.LongBuffer;
import java.util.Iterator;

/**
 * Simple bitset-like container. Unlike org.roaringbitmap.BitmapContainer, this
 * class uses a LongBuffer to store data.
 */
public final class MappeableBitmapContainer extends MappeableContainer
        implements Cloneable, Serializable {
    protected static final int MAX_CAPACITY = 1 << 16;

    private static final long serialVersionUID = 2L;

    private static boolean USE_IN_PLACE = true; // optimization flag

    LongBuffer bitmap;

    int cardinality;

    /**
     * Create a bitmap container with all bits set to false
     */
    public MappeableBitmapContainer() {
        this.cardinality = 0;
        this.bitmap = LongBuffer.allocate(MAX_CAPACITY / 64);
    }

    /**
     * Create a bitmap container with a run of ones from firstOfRun to
     * lastOfRun, inclusive caller must ensure that the range isn't so small
     * that an ArrayContainer should have been created instead
     * 
     * @param firstOfRun
     *            first index
     * @param lastOfRun
     *            last index (range is inclusive)
     */
    public MappeableBitmapContainer(final int firstOfRun, final int lastOfRun) {
        // TODO: this can be optimized for performance
        this.cardinality = lastOfRun - firstOfRun + 1;
        this.bitmap = LongBuffer.allocate(MAX_CAPACITY / 64);
        if (this.cardinality == MAX_CAPACITY) {// perhaps a common case
            for (int k = 0; k < bitmap.limit(); ++k)
                bitmap.put(k, -1L);
        } else {
            final int firstWord = firstOfRun / 64;
            final int lastWord = lastOfRun / 64;
            final int zeroPrefixLength = firstOfRun & 63;
            final int zeroSuffixLength = 63 - (lastOfRun & 63);
            for (int k = firstWord; k < lastWord + 1; ++k)
                bitmap.put(k, -1L);
            bitmap.put(firstWord, bitmap.get(firstWord)
                    ^ ((1L << zeroPrefixLength) - 1));
            final long blockOfOnes = (1L << zeroSuffixLength) - 1;
            final long maskOnLeft = blockOfOnes << (64 - zeroSuffixLength);
            bitmap.put(lastWord, bitmap.get(lastWord) ^ maskOnLeft);
        }
    }

    MappeableBitmapContainer(int newCardinality, LongBuffer newBitmap) {
        this.cardinality = newCardinality;
        this.bitmap = LongBuffer.allocate(newBitmap.limit());
        newBitmap.rewind();
        this.bitmap.put(newBitmap);
    }

    /**
     * Construct a new BitmapContainer backed by the provided LongBuffer.
     * 
     * @param array
     *            LongBuffer where the data is stored
     * @param initCardinality
     *            cardinality (number of values stored)
     */
    public MappeableBitmapContainer(final LongBuffer array,
            final int initCardinality) {
        if (array.limit() != MAX_CAPACITY / 64)
            throw new RuntimeException(
                    "Mismatch between buffer and storage requirements: "
                            + array.limit() + " vs. " + MAX_CAPACITY / 64);
        this.cardinality = initCardinality;
        this.bitmap = array;
    }

    @Override
    public MappeableContainer add(final short i) {
        final int x = BufferUtil.toIntUnsigned(i);
        final long previous = bitmap.get(x / 64);
        bitmap.put(x / 64, previous | (1l << x));
        cardinality += (previous ^ bitmap.get(x / 64)) >>> x;
        return this;
    }

    @Override
    public MappeableArrayContainer and(final MappeableArrayContainer value2) {

        final MappeableArrayContainer answer = new MappeableArrayContainer(
                value2.content.limit());
        short[] sarray = answer.content.array();
        if (value2.content.hasArray()) {
            short[] c = value2.content.array();
            for (int k = 0; k < value2.getCardinality(); ++k)
                if (this.contains(c[k]))
                    sarray[answer.cardinality++] = c[k];

        } else
            for (int k = 0; k < value2.getCardinality(); ++k)
                if (this.contains(value2.content.get(k)))
                    sarray[answer.cardinality++] = value2.content.get(k);
        return answer;
    }

    @Override
    public MappeableContainer and(final MappeableBitmapContainer value2) {
        int newCardinality = 0;
        if (this.bitmap.hasArray() && value2.bitmap.hasArray()) {
            long[] tb = this.bitmap.array();
            long[] v2b = value2.bitmap.array();
            for (int k = 0; k < this.bitmap.limit(); ++k) {
                newCardinality += Long.bitCount(tb[k] & v2b[k]);
            }
        } else
            for (int k = 0; k < this.bitmap.limit(); ++k) {
                newCardinality += Long.bitCount(this.bitmap.get(k)
                        & value2.bitmap.get(k));
            }
        if (newCardinality > MappeableArrayContainer.DEFAULT_MAX_SIZE) {
            final MappeableBitmapContainer answer = new MappeableBitmapContainer();
            long[] bitArray = answer.bitmap.array();
            if (this.bitmap.hasArray() && value2.bitmap.hasArray()) {
                long[] tb = this.bitmap.array();
                long[] v2b = value2.bitmap.array();
                for (int k = 0; k < answer.bitmap.limit(); ++k) {
                    bitArray[k] = tb[k] & v2b[k];
                }
            } else
                for (int k = 0; k < answer.bitmap.limit(); ++k) {
                    bitArray[k] = this.bitmap.get(k) & value2.bitmap.get(k);
                }
            answer.cardinality = newCardinality;
            return answer;
        }
        final MappeableArrayContainer ac = new MappeableArrayContainer(
                newCardinality);
        if (this.bitmap.hasArray() && value2.bitmap.hasArray())
            org.roaringbitmap.Util.fillArrayAND(ac.content.array(),
                    this.bitmap.array(), value2.bitmap.array());
        else
            BufferUtil.fillArrayAND(ac.content.array(), this.bitmap,
                    value2.bitmap);
        ac.cardinality = newCardinality;
        return ac;
    }

    @Override
    public MappeableContainer andNot(final MappeableArrayContainer value2) {
        final MappeableBitmapContainer answer = clone();
        long[] bitArray = answer.bitmap.array();
        if (value2.content.hasArray() && this.bitmap.hasArray()) {
            short[] v2 = value2.content.array();
            long[] ba = this.bitmap.array();
            for (int k = 0; k < value2.cardinality; ++k) {
                final int i = BufferUtil.toIntUnsigned(v2[k]) >>> 6;
                bitArray[i] &= (~(1l << v2[k]));
                answer.cardinality -= (bitArray[i] ^ ba[i]) >>> v2[k];
            }
        } else
            for (int k = 0; k < value2.cardinality; ++k) {
                final int i = BufferUtil.toIntUnsigned(value2.content.get(k)) >>> 6;
                bitArray[i] &= (~(1l << value2.content.get(k)));
                answer.cardinality -= (bitArray[i] ^ this.bitmap.get(i)) >>> value2.content
                        .get(k);
            }
        if (answer.cardinality <= MappeableArrayContainer.DEFAULT_MAX_SIZE)
            return answer.toArrayContainer();
        return answer;
    }

    @Override
    public MappeableContainer andNot(final MappeableBitmapContainer value2) {

        int newCardinality = 0;
        if (this.bitmap.hasArray() && value2.bitmap.hasArray()) {
            long[] b = this.bitmap.array();
            long[] v2 = value2.bitmap.array();
            for (int k = 0; k < this.bitmap.limit(); ++k) {
                newCardinality += Long.bitCount(b[k] & (~v2[k]));
            }

        } else
            for (int k = 0; k < this.bitmap.limit(); ++k) {
                newCardinality += Long.bitCount(this.bitmap.get(k)
                        & (~value2.bitmap.get(k)));
            }
        if (newCardinality > MappeableArrayContainer.DEFAULT_MAX_SIZE) {
            final MappeableBitmapContainer answer = new MappeableBitmapContainer();
            long[] bitArray = answer.bitmap.array();
            if (this.bitmap.hasArray() && value2.bitmap.hasArray()) {
                long[] b = this.bitmap.array();
                long[] v2 = value2.bitmap.array();

                for (int k = 0; k < answer.bitmap.limit(); ++k) {
                    bitArray[k] = b[k] & (~v2[k]);
                }
            } else
                for (int k = 0; k < answer.bitmap.limit(); ++k) {
                    bitArray[k] = this.bitmap.get(k) & (~value2.bitmap.get(k));
                }
            answer.cardinality = newCardinality;
            return answer;
        }
        final MappeableArrayContainer ac = new MappeableArrayContainer(
                newCardinality);
        if (this.bitmap.hasArray() && value2.bitmap.hasArray())
            org.roaringbitmap.Util.fillArrayANDNOT(ac.content.array(),
                    this.bitmap.array(), value2.bitmap.array());
        else
            BufferUtil.fillArrayANDNOT(ac.content.array(), this.bitmap,
                    value2.bitmap);
        ac.cardinality = newCardinality;
        return ac;
    }

    @Override
    public void clear() {
        if (cardinality != 0) {
            cardinality = 0;
            for (int k = 0; k < bitmap.limit(); ++k)
                bitmap.put(k, 0);
        }
    }

    @Override
    public MappeableBitmapContainer clone() {
        return new MappeableBitmapContainer(this.cardinality, this.bitmap);
    }

    @Override
    public boolean contains(final short i) {
        final int x = BufferUtil.toIntUnsigned(i);
        return (bitmap.get(x / 64) & (1l << x)) != 0;
    }

    @Override
    public boolean equals(Object o) {
        if (o instanceof MappeableBitmapContainer) {
            final MappeableBitmapContainer srb = (MappeableBitmapContainer) o;
            if (srb.cardinality != this.cardinality)
                return false;
            if (this.bitmap.hasArray() && srb.bitmap.hasArray()) {
                long[] b = this.bitmap.array();
                long[] s = srb.bitmap.array();
                for (int k = 0; k < this.bitmap.limit(); ++k)
                    if (b[k] != s[k])
                        return false;
            } else
                for (int k = 0; k < this.bitmap.limit(); ++k)
                    if (this.bitmap.get(k) != srb.bitmap.get(k))
                        return false;
            return true;

        }
        return false;
    }

    /**
     * Fill the array with set bits
     * 
     * @param array
     *            container (should be sufficiently large)
     */
    protected void fillArray(final short[] array) {
        int pos = 0;
        if (bitmap.hasArray()) {
            long[] b = bitmap.array();
            for (int k = 0; k < bitmap.limit(); ++k) {
                long bitset = b[k];
                while (bitset != 0) {
                    final long t = bitset & -bitset;
                    array[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
                    bitset ^= t;
                }
            }

        } else
            for (int k = 0; k < bitmap.limit(); ++k) {
                long bitset = bitmap.get(k);
                while (bitset != 0) {
                    final long t = bitset & -bitset;
                    array[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
                    bitset ^= t;
                }
            }
    }

    @Override
    public void fillLeastSignificant16bits(int[] x, int i, int mask) {
        int pos = i;
        if (bitmap.hasArray()) {
            long[] b = bitmap.array();
            for (int k = 0; k < bitmap.limit(); ++k) {
                long bitset = b[k];
                while (bitset != 0) {
                    final long t = bitset & -bitset;
                    x[pos++] = (k * 64 + Long.bitCount(t - 1)) | mask;
                    bitset ^= t;
                }
            }

        } else
            for (int k = 0; k < bitmap.limit(); ++k) {
                long bitset = bitmap.get(k);
                while (bitset != 0) {
                    final long t = bitset & -bitset;
                    x[pos++] = (k * 64 + Long.bitCount(t - 1)) | mask;
                    bitset ^= t;
                }
            }
    }

    @Override
    protected int getArraySizeInBytes() {
        return MAX_CAPACITY / 8;
    }

    @Override
    public int getCardinality() {
        return cardinality;
    }

    @Override
    public ShortIterator getShortIterator() {
        return new ShortIterator() {
            int i = MappeableBitmapContainer.this.nextSetBit(0);
            int max = MappeableBitmapContainer.this.bitmap.limit() * 64 - 1;
            int j;

            @Override
            public boolean hasNext() {
                return i >= 0;
            }

            @Override
            public short next() {
                j = i;
                i = i < max ? MappeableBitmapContainer.this.nextSetBit(i + 1) : -1;
                return (short) j;
            }
            
            @Override
            public ShortIterator clone() {
                try {
                    return (ShortIterator) super.clone();
                } catch (CloneNotSupportedException e) {
                    return null;// will not happen
                }
            }

            @Override
            public void remove() {
                //TODO: implement
                throw new RuntimeException("unsupported operation: remove");                
            }

            
        };
    }

    @Override
    public ShortIterator getReverseShortIterator() {
        return new ShortIterator() {
            int i = MappeableBitmapContainer.this.prevSetBit(64 * MappeableBitmapContainer.this.bitmap.limit() - 1);

            int j;

            @Override
            public boolean hasNext() {
                return i >= 0;
            }

            @Override
            public short next() {
                j = i;
                i = i > 0 ? MappeableBitmapContainer.this.prevSetBit(i - 1) : -1;
                return (short) j;
            }

            @Override
            public ShortIterator clone() {
                try {
                    return (ShortIterator) super.clone();
                } catch (CloneNotSupportedException e) {
                    return null;// will not happen
                }
            }

            @Override
            public void remove() {
                //TODO: implement
                throw new RuntimeException("unsupported operation: remove");
            }


        };
    }

    @Override
    public int getSizeInBytes() {
        return this.bitmap.limit() * 8;
    }

    @Override
    public int hashCode() {
        long hash = 0;
        for (int k = 0; k < this.bitmap.limit(); ++k)
            hash += 31 * hash + this.bitmap.get(k);
        return (int) (hash >> 32);
    }

    @Override
    public MappeableContainer iand(final MappeableArrayContainer b2) {
        return b2.and(this);// no inplace possible
    }

    @Override
    public MappeableContainer iand(final MappeableBitmapContainer b2) {
        int newCardinality = 0;
        for (int k = 0; k < this.bitmap.limit(); ++k) {
            newCardinality += Long.bitCount(this.bitmap.get(k)
                    & b2.bitmap.get(k));
        }
        if (newCardinality > MappeableArrayContainer.DEFAULT_MAX_SIZE) {
            for (int k = 0; k < this.bitmap.limit(); ++k) {
                this.bitmap.put(k, this.bitmap.get(k) & b2.bitmap.get(k));
            }
            this.cardinality = newCardinality;
            return this;
        }
        final MappeableArrayContainer ac = new MappeableArrayContainer(
                newCardinality);<commit_msg>More fixes for hash functions.<commit_after>/*
 * (c) <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>
 * Licensed under the Apache License, Version 2.0.
 */
package org.roaringbitmap.buffer;

import org.roaringbitmap.BitmapContainer;
import org.roaringbitmap.ShortIterator;
import org.roaringbitmap.Util;

import java.io.*;
import java.nio.LongBuffer;
import java.util.Iterator;

/**
 * Simple bitset-like container. Unlike org.roaringbitmap.BitmapContainer, this
 * class uses a LongBuffer to store data.
 */
public final class MappeableBitmapContainer extends MappeableContainer
        implements Cloneable, Serializable {
    protected static final int MAX_CAPACITY = 1 << 16;

    private static final long serialVersionUID = 2L;

    private static boolean USE_IN_PLACE = true; // optimization flag

    LongBuffer bitmap;

    int cardinality;

    /**
     * Create a bitmap container with all bits set to false
     */
    public MappeableBitmapContainer() {
        this.cardinality = 0;
        this.bitmap = LongBuffer.allocate(MAX_CAPACITY / 64);
    }

    /**
     * Create a bitmap container with a run of ones from firstOfRun to
     * lastOfRun, inclusive caller must ensure that the range isn't so small
     * that an ArrayContainer should have been created instead
     * 
     * @param firstOfRun
     *            first index
     * @param lastOfRun
     *            last index (range is inclusive)
     */
    public MappeableBitmapContainer(final int firstOfRun, final int lastOfRun) {
        // TODO: this can be optimized for performance
        this.cardinality = lastOfRun - firstOfRun + 1;
        this.bitmap = LongBuffer.allocate(MAX_CAPACITY / 64);
        if (this.cardinality == MAX_CAPACITY) {// perhaps a common case
            for (int k = 0; k < bitmap.limit(); ++k)
                bitmap.put(k, -1L);
        } else {
            final int firstWord = firstOfRun / 64;
            final int lastWord = lastOfRun / 64;
            final int zeroPrefixLength = firstOfRun & 63;
            final int zeroSuffixLength = 63 - (lastOfRun & 63);
            for (int k = firstWord; k < lastWord + 1; ++k)
                bitmap.put(k, -1L);
            bitmap.put(firstWord, bitmap.get(firstWord)
                    ^ ((1L << zeroPrefixLength) - 1));
            final long blockOfOnes = (1L << zeroSuffixLength) - 1;
            final long maskOnLeft = blockOfOnes << (64 - zeroSuffixLength);
            bitmap.put(lastWord, bitmap.get(lastWord) ^ maskOnLeft);
        }
    }

    MappeableBitmapContainer(int newCardinality, LongBuffer newBitmap) {
        this.cardinality = newCardinality;
        this.bitmap = LongBuffer.allocate(newBitmap.limit());
        newBitmap.rewind();
        this.bitmap.put(newBitmap);
    }

    /**
     * Construct a new BitmapContainer backed by the provided LongBuffer.
     * 
     * @param array
     *            LongBuffer where the data is stored
     * @param initCardinality
     *            cardinality (number of values stored)
     */
    public MappeableBitmapContainer(final LongBuffer array,
            final int initCardinality) {
        if (array.limit() != MAX_CAPACITY / 64)
            throw new RuntimeException(
                    "Mismatch between buffer and storage requirements: "
                            + array.limit() + " vs. " + MAX_CAPACITY / 64);
        this.cardinality = initCardinality;
        this.bitmap = array;
    }

    @Override
    public MappeableContainer add(final short i) {
        final int x = BufferUtil.toIntUnsigned(i);
        final long previous = bitmap.get(x / 64);
        bitmap.put(x / 64, previous | (1l << x));
        cardinality += (previous ^ bitmap.get(x / 64)) >>> x;
        return this;
    }

    @Override
    public MappeableArrayContainer and(final MappeableArrayContainer value2) {

        final MappeableArrayContainer answer = new MappeableArrayContainer(
                value2.content.limit());
        short[] sarray = answer.content.array();
        if (value2.content.hasArray()) {
            short[] c = value2.content.array();
            for (int k = 0; k < value2.getCardinality(); ++k)
                if (this.contains(c[k]))
                    sarray[answer.cardinality++] = c[k];

        } else
            for (int k = 0; k < value2.getCardinality(); ++k)
                if (this.contains(value2.content.get(k)))
                    sarray[answer.cardinality++] = value2.content.get(k);
        return answer;
    }

    @Override
    public MappeableContainer and(final MappeableBitmapContainer value2) {
        int newCardinality = 0;
        if (this.bitmap.hasArray() && value2.bitmap.hasArray()) {
            long[] tb = this.bitmap.array();
            long[] v2b = value2.bitmap.array();
            for (int k = 0; k < this.bitmap.limit(); ++k) {
                newCardinality += Long.bitCount(tb[k] & v2b[k]);
            }
        } else
            for (int k = 0; k < this.bitmap.limit(); ++k) {
                newCardinality += Long.bitCount(this.bitmap.get(k)
                        & value2.bitmap.get(k));
            }
        if (newCardinality > MappeableArrayContainer.DEFAULT_MAX_SIZE) {
            final MappeableBitmapContainer answer = new MappeableBitmapContainer();
            long[] bitArray = answer.bitmap.array();
            if (this.bitmap.hasArray() && value2.bitmap.hasArray()) {
                long[] tb = this.bitmap.array();
                long[] v2b = value2.bitmap.array();
                for (int k = 0; k < answer.bitmap.limit(); ++k) {
                    bitArray[k] = tb[k] & v2b[k];
                }
            } else
                for (int k = 0; k < answer.bitmap.limit(); ++k) {
                    bitArray[k] = this.bitmap.get(k) & value2.bitmap.get(k);
                }
            answer.cardinality = newCardinality;
            return answer;
        }
        final MappeableArrayContainer ac = new MappeableArrayContainer(
                newCardinality);
        if (this.bitmap.hasArray() && value2.bitmap.hasArray())
            org.roaringbitmap.Util.fillArrayAND(ac.content.array(),
                    this.bitmap.array(), value2.bitmap.array());
        else
            BufferUtil.fillArrayAND(ac.content.array(), this.bitmap,
                    value2.bitmap);
        ac.cardinality = newCardinality;
        return ac;
    }

    @Override
    public MappeableContainer andNot(final MappeableArrayContainer value2) {
        final MappeableBitmapContainer answer = clone();
        long[] bitArray = answer.bitmap.array();
        if (value2.content.hasArray() && this.bitmap.hasArray()) {
            short[] v2 = value2.content.array();
            long[] ba = this.bitmap.array();
            for (int k = 0; k < value2.cardinality; ++k) {
                final int i = BufferUtil.toIntUnsigned(v2[k]) >>> 6;
                bitArray[i] &= (~(1l << v2[k]));
                answer.cardinality -= (bitArray[i] ^ ba[i]) >>> v2[k];
            }
        } else
            for (int k = 0; k < value2.cardinality; ++k) {
                final int i = BufferUtil.toIntUnsigned(value2.content.get(k)) >>> 6;
                bitArray[i] &= (~(1l << value2.content.get(k)));
                answer.cardinality -= (bitArray[i] ^ this.bitmap.get(i)) >>> value2.content
                        .get(k);
            }
        if (answer.cardinality <= MappeableArrayContainer.DEFAULT_MAX_SIZE)
            return answer.toArrayContainer();
        return answer;
    }

    @Override
    public MappeableContainer andNot(final MappeableBitmapContainer value2) {

        int newCardinality = 0;
        if (this.bitmap.hasArray() && value2.bitmap.hasArray()) {
            long[] b = this.bitmap.array();
            long[] v2 = value2.bitmap.array();
            for (int k = 0; k < this.bitmap.limit(); ++k) {
                newCardinality += Long.bitCount(b[k] & (~v2[k]));
            }

        } else
            for (int k = 0; k < this.bitmap.limit(); ++k) {
                newCardinality += Long.bitCount(this.bitmap.get(k)
                        & (~value2.bitmap.get(k)));
            }
        if (newCardinality > MappeableArrayContainer.DEFAULT_MAX_SIZE) {
            final MappeableBitmapContainer answer = new MappeableBitmapContainer();
            long[] bitArray = answer.bitmap.array();
            if (this.bitmap.hasArray() && value2.bitmap.hasArray()) {
                long[] b = this.bitmap.array();
                long[] v2 = value2.bitmap.array();

                for (int k = 0; k < answer.bitmap.limit(); ++k) {
                    bitArray[k] = b[k] & (~v2[k]);
                }
            } else
                for (int k = 0; k < answer.bitmap.limit(); ++k) {
                    bitArray[k] = this.bitmap.get(k) & (~value2.bitmap.get(k));
                }
            answer.cardinality = newCardinality;
            return answer;
        }
        final MappeableArrayContainer ac = new MappeableArrayContainer(
                newCardinality);
        if (this.bitmap.hasArray() && value2.bitmap.hasArray())
            org.roaringbitmap.Util.fillArrayANDNOT(ac.content.array(),
                    this.bitmap.array(), value2.bitmap.array());
        else
            BufferUtil.fillArrayANDNOT(ac.content.array(), this.bitmap,
                    value2.bitmap);
        ac.cardinality = newCardinality;
        return ac;
    }

    @Override
    public void clear() {
        if (cardinality != 0) {
            cardinality = 0;
            for (int k = 0; k < bitmap.limit(); ++k)
                bitmap.put(k, 0);
        }
    }

    @Override
    public MappeableBitmapContainer clone() {
        return new MappeableBitmapContainer(this.cardinality, this.bitmap);
    }

    @Override
    public boolean contains(final short i) {
        final int x = BufferUtil.toIntUnsigned(i);
        return (bitmap.get(x / 64) & (1l << x)) != 0;
    }

    @Override
    public boolean equals(Object o) {
        if (o instanceof MappeableBitmapContainer) {
            final MappeableBitmapContainer srb = (MappeableBitmapContainer) o;
            if (srb.cardinality != this.cardinality)
                return false;
            if (this.bitmap.hasArray() && srb.bitmap.hasArray()) {
                long[] b = this.bitmap.array();
                long[] s = srb.bitmap.array();
                for (int k = 0; k < this.bitmap.limit(); ++k)
                    if (b[k] != s[k])
                        return false;
            } else
                for (int k = 0; k < this.bitmap.limit(); ++k)
                    if (this.bitmap.get(k) != srb.bitmap.get(k))
                        return false;
            return true;

        }
        return false;
    }

    /**
     * Fill the array with set bits
     * 
     * @param array
     *            container (should be sufficiently large)
     */
    protected void fillArray(final short[] array) {
        int pos = 0;
        if (bitmap.hasArray()) {
            long[] b = bitmap.array();
            for (int k = 0; k < bitmap.limit(); ++k) {
                long bitset = b[k];
                while (bitset != 0) {
                    final long t = bitset & -bitset;
                    array[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
                    bitset ^= t;
                }
            }

        } else
            for (int k = 0; k < bitmap.limit(); ++k) {
                long bitset = bitmap.get(k);
                while (bitset != 0) {
                    final long t = bitset & -bitset;
                    array[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
                    bitset ^= t;
                }
            }
    }

    @Override
    public void fillLeastSignificant16bits(int[] x, int i, int mask) {
        int pos = i;
        if (bitmap.hasArray()) {
            long[] b = bitmap.array();
            for (int k = 0; k < bitmap.limit(); ++k) {
                long bitset = b[k];
                while (bitset != 0) {
                    final long t = bitset & -bitset;
                    x[pos++] = (k * 64 + Long.bitCount(t - 1)) | mask;
                    bitset ^= t;
                }
            }

        } else
            for (int k = 0; k < bitmap.limit(); ++k) {
                long bitset = bitmap.get(k);
                while (bitset != 0) {
                    final long t = bitset & -bitset;
                    x[pos++] = (k * 64 + Long.bitCount(t - 1)) | mask;
                    bitset ^= t;
                }
            }
    }

    @Override
    protected int getArraySizeInBytes() {
        return MAX_CAPACITY / 8;
    }

    @Override
    public int getCardinality() {
        return cardinality;
    }

    @Override
    public ShortIterator getShortIterator() {
        return new ShortIterator() {
            int i = MappeableBitmapContainer.this.nextSetBit(0);
            int max = MappeableBitmapContainer.this.bitmap.limit() * 64 - 1;
            int j;

            @Override
            public boolean hasNext() {
                return i >= 0;
            }

            @Override
            public short next() {
                j = i;
                i = i < max ? MappeableBitmapContainer.this.nextSetBit(i + 1) : -1;
                return (short) j;
            }
            
            @Override
            public ShortIterator clone() {
                try {
                    return (ShortIterator) super.clone();
                } catch (CloneNotSupportedException e) {
                    return null;// will not happen
                }
            }

            @Override
            public void remove() {
                //TODO: implement
                throw new RuntimeException("unsupported operation: remove");                
            }

            
        };
    }

    @Override
    public ShortIterator getReverseShortIterator() {
        return new ShortIterator() {
            int i = MappeableBitmapContainer.this.prevSetBit(64 * MappeableBitmapContainer.this.bitmap.limit() - 1);

            int j;

            @Override
            public boolean hasNext() {
                return i >= 0;
            }

            @Override
            public short next() {
                j = i;
                i = i > 0 ? MappeableBitmapContainer.this.prevSetBit(i - 1) : -1;
                return (short) j;
            }

            @Override
            public ShortIterator clone() {
                try {
                    return (ShortIterator) super.clone();
                } catch (CloneNotSupportedException e) {
                    return null;// will not happen
                }
            }

            @Override
            public void remove() {
                //TODO: implement
                throw new RuntimeException("unsupported operation: remove");
            }


        };
    }

    @Override
    public int getSizeInBytes() {
        return this.bitmap.limit() * 8;
    }

    @Override
    public int hashCode() {
        long hash = 0;
        for (int k = 0; k < this.bitmap.limit(); ++k)
            hash += 31 * hash + this.bitmap.get(k);
        return (int) hash;
    }

    @Override
    public MappeableContainer iand(final MappeableArrayContainer b2) {
        return b2.and(this);// no inplace possible
    }

    @Override
    public MappeableContainer iand(final MappeableBitmapContainer b2) {
        int newCardinality = 0;
        for (int k = 0; k < this.bitmap.limit(); ++k) {
            newCardinality += Long.bitCount(this.bitmap.get(k)
                    & b2.bitmap.get(k));
        }
        if (newCardinality > MappeableArrayContainer.DEFAULT_MAX_SIZE) {
            for (int k = 0; k < this.bitmap.limit(); ++k) {
                this.bitmap.put(k, this.bitmap.get(k) & b2.bitmap.get(k));
            }
            this.cardinality = newCardinality;
            return this;
        }
        final MappeableArrayContainer ac = new MappeableArrayContainer(
                newCardinality);
{'fraction_non_alphanumeric': 0.11516284868318528, 'fraction_numerical': 0.010659186535764377, 'mean_word_length': 1.5921796736144773, 'pattern_counts': {'":': 0, '<': 99, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5034701', 'n_tokens_mistral': 9385, 'n_tokens_neox': 9035, 'n_words': 2214}