<commit_before>package net.lshift.spki;

import java.io.IOException;
import java.io.InputStream;

/**
 * Tokenize an InputStream into SPKI tokens
 */
public class CanonicalSpkiInputStream extends SpkiInputStream {
    private static final int NO_MORE_DIGITS_BOUND = (Integer.MAX_VALUE - 9)/10;
    private final InputStream is;
    private int atomBytes;

    public CanonicalSpkiInputStream(final InputStream is) {
        this.is = is;
    }

    @Override
    public TokenType doNext()
        throws IOException,
            ParseException {
        final int next = is.read();
        switch (next) {
        case '(':
            return TokenType.OPENPAREN;
        case ')':
            return TokenType.CLOSEPAREN;
        case -1:
            return TokenType.EOF;
        default:
            atomBytes = readInteger(next);
            return TokenType.ATOM;
        }
    }

    private int readInteger(final int next)
        throws ParseException,
            IOException {
        int c = next;
        int r = 0;
        for (;;) {
            if (c < Constants.DIGITBASE || c >= Constants.DIGITBASE + 10) {
                invalidate();
                throw new ParseException("Bad s-expression format");
            }
            r += c - Constants.DIGITBASE;
            c = is.read();
            if (c == Constants.COLON)
                return r;
            if (r > NO_MORE_DIGITS_BOUND) {
                // Could strictly speaking handle it so long as
                // next digit is 0..7 and is last, but let's not go mad.
                invalidate();
                throw new ParseException("Integer too large");
            }
            r *= 10;
        }
    }

    @Override
    public byte[] doAtomBytes()
        throws IOException,
            ParseException {
        final byte[] res = new byte[atomBytes];
        final int c = is.read(res);
        if (c != atomBytes) {
            invalidate();
            throw new ParseException("Failed to read enough bytes");
        }<commit_msg>doAtomBytes shouldn't assume that every read will be full<commit_after>package net.lshift.spki;

import java.io.IOException;
import java.io.InputStream;

/**
 * Tokenize an InputStream into SPKI tokens
 */
public class CanonicalSpkiInputStream extends SpkiInputStream {
    private static final int NO_MORE_DIGITS_BOUND = (Integer.MAX_VALUE - 9)/10;
    private final InputStream is;
    private int atomBytes;

    public CanonicalSpkiInputStream(final InputStream is) {
        this.is = is;
    }

    @Override
    public TokenType doNext()
        throws IOException,
            ParseException {
        final int next = is.read();
        switch (next) {
        case '(':
            return TokenType.OPENPAREN;
        case ')':
            return TokenType.CLOSEPAREN;
        case -1:
            return TokenType.EOF;
        default:
            atomBytes = readInteger(next);
            return TokenType.ATOM;
        }
    }

    private int readInteger(final int next)
        throws ParseException,
            IOException {
        int c = next;
        int r = 0;
        for (;;) {
            if (c < Constants.DIGITBASE || c >= Constants.DIGITBASE + 10) {
                invalidate();
                throw new ParseException("Bad s-expression format");
            }
            r += c - Constants.DIGITBASE;
            c = is.read();
            if (c == Constants.COLON)
                return r;
            if (r > NO_MORE_DIGITS_BOUND) {
                // Could strictly speaking handle it so long as
                // next digit is 0..7 and is last, but let's not go mad.
                invalidate();
                throw new ParseException("Integer too large");
            }
            r *= 10;
        }
    }

    @Override
    public byte[] doAtomBytes()
        throws IOException,
            ParseException {
        final byte[] res = new byte[atomBytes];
        int ix = 0;
        while (ix < atomBytes) {
            final int c = is.read(res, ix, atomBytes-ix);
            if (c < 1) {
                invalidate();
                throw new ParseException("Failed to read enough bytes");
            }
            ix += c;
        }
{'fraction_non_alphanumeric': 0.08948492760503204, 'fraction_numerical': 0.005696653216235461, 'mean_word_length': 1.5868631062001228, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5449497', 'n_tokens_mistral': 1173, 'n_tokens_neox': 1103, 'n_words': 338}