<commit_before>package net.kotek.jdbm;

import java.io.IOError;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Arrays;

/**
 * JDBM abstraction over ByteBuffer.
 * <p/>
 * ByteBuffer has maximal size 2GB, so this class chains multiple ByteBuffers together.
 * IO operations must be padded not to cross ByteBuffer boundaries.
 * <p/>
 * Also ByteBuffer is not grow-able, this solves it.
 */
public final class ByteBuffer2 {

    static final int  BUF_SIZE = 1<<30;

    static final int BUF_SIZE_INC = 1024*1024;

    static final int INITIAL_SIZE = 1024*32;

    /** file channel backing this ByteBuffer, null for in-memory-store */
    protected FileChannel fileChannel;
    final protected FileChannel.MapMode mapMode;
    protected ByteBuffer[] buffers;
    final protected boolean inMemory;


    public ByteBuffer2(boolean inMemory, FileChannel fileChannel, FileChannel.MapMode mapMode){
        try{
        this.fileChannel = fileChannel;
        this.inMemory = inMemory;
        this.mapMode = mapMode;
        if(inMemory){
            buffers = new ByteBuffer[]{ByteBuffer.allocate(INITIAL_SIZE)};
        }else{
            final long fileSize = fileChannel.size();
            if(fileSize>0){
                //map existing data
                buffers = new ByteBuffer[(int) (1+fileSize/BUF_SIZE)];
                for(int i=0;i<=fileSize/BUF_SIZE;i++){
                    final long offset = 1L*BUF_SIZE*i;
                    buffers[i] = fileChannel.map(mapMode, offset, Math.min(BUF_SIZE, fileSize-offset));
                    //TODO what if 'fileSize % 8 != 0'?
                }
            }else{
                buffers = new ByteBuffer[1];
                buffers[0] = fileChannel.map(mapMode, 0, INITIAL_SIZE);
            }
        }
        }catch(IOException e){
            throw new IOError(e);
        }
    }


    protected ByteBuffer getByteBuffer(long offset) {
        return buffers[((int) (offset / BUF_SIZE))];
    }

    public void ensureAvailable(long offset) throws IOException {
        int buffersPos = (int) (offset/BUF_SIZE);

        //check for most common case, this is already mapped
        if(buffers[buffersPos]!=null && buffers[buffersPos].capacity()>=offset%BUF_SIZE)
            return;

        //grow array if necessary
        if(buffers.length<buffersPos){
            buffers = Arrays.copyOf(buffers, Math.max(buffersPos,  buffers.length*2));
        }


        if(inMemory){
            final int newBufSize = JdbmUtil.nextPowTwo((int) (offset%BUF_SIZE));
            //double size of existing in-memory-buffer
            ByteBuffer newBuf = ByteBuffer.allocate(newBufSize);
            if(buffers[buffersPos]!=null){
                //copy old buffer if it exists
                buffers[buffersPos].rewind();<commit_msg>Fixed IndexOutOfBoundsException with store larger 1 GB.<commit_after>package net.kotek.jdbm;

import java.io.IOError;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Arrays;

/**
 * JDBM abstraction over ByteBuffer.
 * <p/>
 * ByteBuffer has maximal size 2GB, so this class chains multiple ByteBuffers together.
 * IO operations must be padded not to cross ByteBuffer boundaries.
 * <p/>
 * Also ByteBuffer is not grow-able, this solves it.
 */
public final class ByteBuffer2 {

    static final int  BUF_SIZE = 1<<30;

    static final int BUF_SIZE_INC = 1024*1024;

    static final int INITIAL_SIZE = 1024*32;

    /** file channel backing this ByteBuffer, null for in-memory-store */
    protected FileChannel fileChannel;
    final protected FileChannel.MapMode mapMode;
    protected ByteBuffer[] buffers;
    final protected boolean inMemory;


    public ByteBuffer2(boolean inMemory, FileChannel fileChannel, FileChannel.MapMode mapMode){
        try{
        this.fileChannel = fileChannel;
        this.inMemory = inMemory;
        this.mapMode = mapMode;
        if(inMemory){
            buffers = new ByteBuffer[]{ByteBuffer.allocate(INITIAL_SIZE)};
        }else{
            final long fileSize = fileChannel.size();
            if(fileSize>0){
                //map existing data
                buffers = new ByteBuffer[(int) (1+fileSize/BUF_SIZE)];
                for(int i=0;i<=fileSize/BUF_SIZE;i++){
                    final long offset = 1L*BUF_SIZE*i;
                    buffers[i] = fileChannel.map(mapMode, offset, Math.min(BUF_SIZE, fileSize-offset));
                    //TODO what if 'fileSize % 8 != 0'?
                }
            }else{
                buffers = new ByteBuffer[1];
                buffers[0] = fileChannel.map(mapMode, 0, INITIAL_SIZE);
            }
        }
        }catch(IOException e){
            throw new IOError(e);
        }
    }


    protected ByteBuffer getByteBuffer(long offset) {
        return buffers[((int) (offset / BUF_SIZE))];
    }

    public void ensureAvailable(long offset) throws IOException {
        int buffersPos = (int) (offset/BUF_SIZE);

        //check for most common case, this is already mapped
        if(buffersPos<buffers.length && buffers[buffersPos]!=null &&
                buffers[buffersPos].capacity()>=offset%BUF_SIZE)
            return;

        //grow array if necessary
        if(buffersPos>=buffers.length){
            buffers = Arrays.copyOf(buffers, Math.max(buffersPos,  buffers.length*2));
        }


        if(inMemory){
            final int newBufSize = JdbmUtil.nextPowTwo((int) (offset%BUF_SIZE));
            //double size of existing in-memory-buffer
            ByteBuffer newBuf = ByteBuffer.allocate(newBufSize);
            if(buffers[buffersPos]!=null){
                //copy old buffer if it exists
                buffers[buffersPos].rewind();
{'fraction_non_alphanumeric': 0.10048342541436464, 'fraction_numerical': 0.010531767955801106, 'mean_word_length': 2.323580034423408, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6098245', 'n_tokens_mistral': 1703, 'n_tokens_neox': 1609, 'n_words': 428}