<commit_before>package org.jetbrains.bio

import com.google.common.primitives.*
import com.indeed.util.mmap.MMapBuffer
import org.iq80.snappy.Snappy
import org.jetbrains.bio.big.RTreeIndex
import java.io.Closeable
import java.io.OutputStream
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.channels.Channels
import java.nio.channels.FileChannel
import java.nio.channels.FileChannel.MapMode
import java.nio.file.Path
import java.nio.file.StandardOpenOption
import java.util.zip.Deflater
import java.util.zip.DeflaterOutputStream
import java.util.zip.Inflater

/** A read-only mapped buffer.*/
interface RomBuffer: Closeable, AutoCloseable {
    var position: Long
    val order: ByteOrder

    /**
     * Returns a new buffer sharing the data with its parent.
     *
     * @see ByteBuffer.duplicate for details.
     */
    fun duplicate(): RomBuffer

    fun checkHeader(leMagic: Int) {
        val magic = getInt()
        check(magic == leMagic) {
            val bigMagic = java.lang.Integer.reverseBytes(leMagic)
            "Unexpected header magic: Actual $magic doesn't match expected LE=$leMagic (BE=$bigMagic)"
        }
    }

    fun asIntArray(size: Int): IntArray
    fun asFloatArray(size: Int): FloatArray

    fun get(dst: ByteArray)
    fun get(): Byte

    fun getUnsignedByte() = java.lang.Byte.toUnsignedInt(get())

    fun getShort(): Short

    fun getUnsignedShort() = java.lang.Short.toUnsignedInt(getShort())

    fun getInt(): Int

    fun getLong(): Long

    fun getFloat(): Float

    fun getDouble(): Double

    fun getCString(): String {
        val sb = StringBuilder()
        do {
            val ch = get()
            if (ch == 0.toByte()) {
                break
            }

            sb.append(ch.toChar())
        } while (true)

        return sb.toString()
    }

    fun hasRemaining(): Boolean
}

/** A read-only mapped buffer based on ByteBuffer.*/
class BBRomBuffer(val mapped: ByteBuffer): RomBuffer {
    override var position: Long
        get() = mapped.position().toLong()
        set(value) = ignore(mapped.position(Ints.checkedCast(value)))

    override val order: ByteOrder get() = mapped.order()

    /**
     * Returns a new buffer sharing the data with its parent.
     *
     * @see ByteBuffer.duplicate for details.
     */
    override fun duplicate() = BBRomBuffer(mapped.duplicate().apply {
        order(mapped.order())
        position(Ints.checkedCast(position))
    })

    override fun close() { /* Do nothing */ }

    override fun asIntArray(size: Int) = IntArray(size).apply {
        mapped.asIntBuffer().get(this)
        mapped.position(mapped.position() + size * Ints.BYTES)
    }

    override fun asFloatArray(size: Int) = FloatArray(size).apply {
        mapped.asFloatBuffer().get(this)
        mapped.position(mapped.position() + size * Floats.BYTES)
    }

    override fun get(dst: ByteArray) { mapped.get(dst) }

    override fun get() = mapped.get()

    override fun getShort() = mapped.getShort()

    override fun getInt() = mapped.getInt()

    override fun getLong() = mapped.getLong()

    override fun getFloat() = mapped.getFloat()

    override fun getDouble() = mapped.getDouble()

    override fun hasRemaining() = mapped.hasRemaining()
}

/** A read-only mapped buffer which supports files > 2GB .*/
class MMBRomBuffer(val mapped: MMapBuffer,
                   override var position: Long = 0,
                   limit: Long = mapped.memory().length()) : RomBuffer {

    override val order: ByteOrder get() = mapped.memory().order
    private var limit: Long = limit
        set(value) {
            val length = mapped.memory().length()
            check(value <= length) {
                "Limit $value is greater than buffer length $length"
            }
            field = value
        }

    override fun hasRemaining() = position < limit

    /**
     * Returns a new buffer sharing the data with its parent.
     *<commit_msg>minor changes in BBRomBuffer according to feedback<commit_after>package org.jetbrains.bio

import com.google.common.primitives.*
import com.indeed.util.mmap.MMapBuffer
import org.iq80.snappy.Snappy
import org.jetbrains.bio.big.RTreeIndex
import java.io.Closeable
import java.io.OutputStream
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.channels.Channels
import java.nio.channels.FileChannel
import java.nio.channels.FileChannel.MapMode
import java.nio.file.Path
import java.nio.file.StandardOpenOption
import java.util.zip.Deflater
import java.util.zip.DeflaterOutputStream
import java.util.zip.Inflater

/** A read-only buffer. */
interface RomBuffer: Closeable, AutoCloseable {
    var position: Long
    val order: ByteOrder

    /**
     * Returns a new buffer sharing the data with its parent.
     *
     * @see ByteBuffer.duplicate for details.
     */
    fun duplicate(): RomBuffer

    fun checkHeader(leMagic: Int) {
        val magic = getInt()
        check(magic == leMagic) {
            val bigMagic = java.lang.Integer.reverseBytes(leMagic)
            "Unexpected header magic: Actual $magic doesn't match expected LE=$leMagic (BE=$bigMagic)"
        }
    }

    fun asIntArray(size: Int): IntArray
    fun asFloatArray(size: Int): FloatArray

    fun get(dst: ByteArray)
    fun get(): Byte

    fun getUnsignedByte() = java.lang.Byte.toUnsignedInt(get())

    fun getShort(): Short

    fun getUnsignedShort() = java.lang.Short.toUnsignedInt(getShort())

    fun getInt(): Int

    fun getLong(): Long

    fun getFloat(): Float

    fun getDouble(): Double

    fun getCString(): String {
        val sb = StringBuilder()
        do {
            val ch = get()
            if (ch == 0.toByte()) {
                break
            }

            sb.append(ch.toChar())
        } while (true)

        return sb.toString()
    }

    fun hasRemaining(): Boolean
}

/** A read-only buffer based on [ByteBuffer]. */
class BBRomBuffer(private val buffer: ByteBuffer): RomBuffer {
    override var position: Long
        get() = buffer.position().toLong()
        set(value) = ignore(buffer.position(Ints.checkedCast(value)))

    override val order: ByteOrder get() = buffer.order()

    /**
     * Returns a new buffer sharing the data with its parent.
     *
     * @see ByteBuffer.duplicate for details.
     */
    override fun duplicate() = BBRomBuffer(buffer.duplicate().apply {
        order(buffer.order())
        position(Ints.checkedCast(position))
    })

    override fun close() { /* Do nothing */ }

    override fun asIntArray(size: Int) = IntArray(size).apply {
        buffer.asIntBuffer().get(this)
        buffer.position(buffer.position() + size * Ints.BYTES)
    }

    override fun asFloatArray(size: Int) = FloatArray(size).apply {
        buffer.asFloatBuffer().get(this)
        buffer.position(buffer.position() + size * Floats.BYTES)
    }

    override fun get(dst: ByteArray) { buffer.get(dst) }

    override fun get() = buffer.get()

    override fun getShort() = buffer.getShort()

    override fun getInt() = buffer.getInt()

    override fun getLong() = buffer.getLong()

    override fun getFloat() = buffer.getFloat()

    override fun getDouble() = buffer.getDouble()

    override fun hasRemaining() = buffer.hasRemaining()
}

/** A read-only mapped buffer which supports files > 2GB .*/
class MMBRomBuffer(val mapped: MMapBuffer,
                   override var position: Long = 0,
                   limit: Long = mapped.memory().length()) : RomBuffer {

    override val order: ByteOrder get() = mapped.memory().order
    private var limit: Long = limit
        set(value) {
            val length = mapped.memory().length()
            check(value <= length) {
                "Limit $value is greater than buffer length $length"
            }
            field = value
        }

    override fun hasRemaining() = position < limit

    /**
     * Returns a new buffer sharing the data with its parent.
     *
{'fraction_non_alphanumeric': 0.11213073538654934, 'fraction_numerical': 0.001257071024512885, 'mean_word_length': 2.622950819672131, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1557537', 'n_tokens_mistral': 2338, 'n_tokens_neox': 2232, 'n_words': 621}