<commit_before>package com.devsmart.ubjson;

import java.io.*;
import java.util.ArrayList;
import java.util.TreeMap;


public class UBReader implements Closeable {

    private final InputStream mInputStream;

    public UBReader(InputStream in) {
        mInputStream = in;
    }

    @Override
    public void close() throws IOException {
        mInputStream.close();
    }

    private byte readControl() throws IOException {
        int value = mInputStream.read();
        if(value == -1) {
            throw new IOException("eof");
        } else {
            return (byte)value;
        }
    }

    private byte readInt8() throws IOException {
        return readControl();
    }

    private short readUInt8() throws IOException {
        return (short)(0xFF & readControl());
    }

    private short readInt16() throws IOException {
        return (short)((readControl() & 0xFF) << 8 | (readControl() & 0xFF));
    }

    private int readInt32() throws IOException {
        return ( (readControl() & 0xFF) << 24 | (readControl() & 0xFF) << 16
                | (readControl() & 0xFF) << 8 | (readControl() & 0xFF) );
    }

    private long readInt(byte control) throws IOException {
        long value;
        switch (control) {
            case UBValue.MARKER_INT8:
                value = readInt8();
                break;

            case UBValue.MARKER_UINT8:
                value = readUInt8();
                break;

            case UBValue.MARKER_INT16:
                value = readInt16();
                break;

            case UBValue.MARKER_INT32:
                value = readInt32();
                break;

            case UBValue.MARKER_INT64:
                value = ((long)readControl() & 0xFF) << 56 | ((long)readControl() & 0xFF) << 48
                        | ((long)readControl() & 0xFF) << 40 | ((long)readControl() & 0xFF) << 32
                        | ((long)readControl() & 0xFF) << 24 | ((long)readControl() & 0xFF) << 16
                        | ((long)readControl() & 0xFF) << 8 | ((long)readControl() & 0xFF);
                break;

            default:
                throw new IOException("not an int type");

        }

        return value;
    }

    private char readChar() throws IOException {
        char value = (char) readControl();
        return value;
    }

    private float readFloat32() throws IOException {
        int intvalue = (readControl() & 0xFF) << 24 | (readControl() & 0xFF) << 16
                | (readControl() & 0xFF) << 8 | (readControl() & 0xFF);

        float value = Float.intBitsToFloat(intvalue);
        return value;
    }

    private double readFloat64() throws IOException {
        long intvalue = ((long)readControl() & 0xFF) << 56 | ((long)readControl() & 0xFF) << 48
                | ((long)readControl() & 0xFF) << 40 | ((long)readControl() & 0xFF) << 32
                | ((long)readControl() & 0xFF) << 24 | ((long)readControl() & 0xFF) << 16
                | ((long)readControl() & 0xFF) << 8 | ((long)readControl() & 0xFF);


        double value = Double.longBitsToDouble(intvalue);
        return value;
    }

    private byte[] readString(byte control) throws IOException {
        int size = (int) readInt(control);
        byte[] value = new byte[size];

        int bytesRead = mInputStream.read(value, 0, size);
        if(bytesRead != size) {
            throw new IOException("eof reached");
        }
        return value;
    }

    private byte[] readOptimizedArrayInt8(int size) throws IOException {
        byte[] data = new byte[size];
        for(int i=0;i<size;i++){
            data[i] = readInt8();
        }
        return data;
    }

    private short[] readOptimizedArrayInt16(int size) throws IOException {
        short[] data = new short[size];<commit_msg>fix bug in UBReader<commit_after>package com.devsmart.ubjson;

import java.io.*;
import java.util.ArrayList;
import java.util.TreeMap;


public class UBReader implements Closeable {

    private final InputStream mInputStream;

    public UBReader(InputStream in) {
        mInputStream = in;
    }

    @Override
    public void close() throws IOException {
        mInputStream.close();
    }

    private byte readControl() throws IOException {
        int value = mInputStream.read();
        if(value == -1) {
            throw new IOException("eof");
        } else {
            return (byte)value;
        }
    }

    private byte readInt8() throws IOException {
        return readControl();
    }

    private short readUInt8() throws IOException {
        return (short)(0xFF & readControl());
    }

    private short readInt16() throws IOException {
        return (short)((readControl() & 0xFF) << 8 | (readControl() & 0xFF));
    }

    private int readInt32() throws IOException {
        return ( (readControl() & 0xFF) << 24 | (readControl() & 0xFF) << 16
                | (readControl() & 0xFF) << 8 | (readControl() & 0xFF) );
    }

    private long readInt(byte control) throws IOException {
        long value;
        switch (control) {
            case UBValue.MARKER_INT8:
                value = readInt8();
                break;

            case UBValue.MARKER_UINT8:
                value = readUInt8();
                break;

            case UBValue.MARKER_INT16:
                value = readInt16();
                break;

            case UBValue.MARKER_INT32:
                value = readInt32();
                break;

            case UBValue.MARKER_INT64:
                value = ((long)readControl() & 0xFF) << 56 | ((long)readControl() & 0xFF) << 48
                        | ((long)readControl() & 0xFF) << 40 | ((long)readControl() & 0xFF) << 32
                        | ((long)readControl() & 0xFF) << 24 | ((long)readControl() & 0xFF) << 16
                        | ((long)readControl() & 0xFF) << 8 | ((long)readControl() & 0xFF);
                break;

            default:
                throw new IOException("not an int type");

        }

        return value;
    }

    private char readChar() throws IOException {
        char value = (char) readControl();
        return value;
    }

    private float readFloat32() throws IOException {
        int intvalue = (readControl() & 0xFF) << 24 | (readControl() & 0xFF) << 16
                | (readControl() & 0xFF) << 8 | (readControl() & 0xFF);

        float value = Float.intBitsToFloat(intvalue);
        return value;
    }

    private double readFloat64() throws IOException {
        long intvalue = ((long)readControl() & 0xFF) << 56 | ((long)readControl() & 0xFF) << 48
                | ((long)readControl() & 0xFF) << 40 | ((long)readControl() & 0xFF) << 32
                | ((long)readControl() & 0xFF) << 24 | ((long)readControl() & 0xFF) << 16
                | ((long)readControl() & 0xFF) << 8 | ((long)readControl() & 0xFF);


        double value = Double.longBitsToDouble(intvalue);
        return value;
    }

    private byte[] readString(byte control) throws IOException {
        int size = (int) readInt(control);
        byte[] value = new byte[size];

        int bytesLeft = size;
        int offset = 0;
        while(bytesLeft > 0) {
            int bytesRead = mInputStream.read(value, offset, size);
            if(bytesRead < 0) {
                throw new IOException("eof reached");
            } else {
                bytesLeft -= bytesRead;
                offset += bytesRead;
            }
        }
        return value;
    }

    private byte[] readOptimizedArrayInt8(int size) throws IOException {
        byte[] data = new byte[size];
        for(int i=0;i<size;i++){
            data[i] = readInt8();
        }
        return data;
    }

    private short[] readOptimizedArrayInt16(int size) throws IOException {
        short[] data = new short[size];
{'fraction_non_alphanumeric': 0.125, 'fraction_numerical': 0.02459016393442623, 'mean_word_length': 1.9170713485244677, 'pattern_counts': {'":': 0, '<': 90, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4732348', 'n_tokens_mistral': 2359, 'n_tokens_neox': 2115, 'n_words': 567}