<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.avro.io;

import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Arrays;

import org.apache.avro.AvroRuntimeException;
import org.apache.avro.InvalidNumberEncodingException;
import org.apache.avro.util.Utf8;

/** An {@link Decoder} for binary-format data.
 * <p/>
 * Instances are created using {@link DecoderFactory}.
 * <p/>
 * This class may read-ahead and buffer bytes from the source beyond what is
 * required to serve its read methods.
 * The number of unused bytes in the buffer can be accessed by
 * inputStream().remaining(), if the BinaryDecoder is not 'direct'.
 *
 * @see Encoder
 */

public class BinaryDecoder extends Decoder {

  /**
   * The maximum size of array to allocate.
   * Some VMs reserve some header words in an array.
   * Attempts to allocate larger arrays may result in
   * OutOfMemoryError: Requested array size exceeds VM limit
   */
  private static final long MAX_ARRAY_SIZE = (long) Integer.MAX_VALUE - 8L;

  private ByteSource source = null;
  // we keep the buffer and its state variables in this class and not in a
  // container class for performance reasons. This improves performance
  // over a container object by about 5% to 15%
  // for example, we could have a FastBuffer class with these state variables
  // and keep a private FastBuffer member here. This simplifies the
  // "detach source" code and source access to the buffer, but
  // hurts performance.
  private byte[] buf = null;
  private int minPos = 0;
  private int pos = 0;
  private int limit = 0;

  byte[] getBuf() { return buf; }
  int getPos() { return pos; }
  int getLimit() { return limit; }

  void setBuf(byte[] buf, int pos, int len) {
    this.buf = buf;
    this.pos = pos;
    this.limit = pos+len;
  }
  void clearBuf() { this.buf = null; }

  /** protected constructor for child classes */
  protected BinaryDecoder() {
  }

  BinaryDecoder(InputStream in, int bufferSize) {
    super();
    configure(in, bufferSize);
  }

  BinaryDecoder(byte[] data, int offset, int length) {
    super();
    configure(data, offset, length);
  }

  BinaryDecoder configure(InputStream in, int bufferSize) {
    configureSource(bufferSize, new InputStreamByteSource(in));
    return this;
  }

  BinaryDecoder configure(byte[] data, int offset, int length) {
    configureSource(DecoderFactory.DEFAULT_BUFFER_SIZE, new ByteArrayByteSource(
        data, offset, length));
    return this;
  }

  /**
   * Initializes this decoder with a new ByteSource. Detaches the old source (if
   * it exists) from this Decoder. The old source's state no longer depends on
   * this Decoder and its InputStream interface will continue to drain the
   * remaining buffer and source data.
   * <p/>
   * The decoder will read from the new source. The source will generally
   * replace the buffer with its own. If the source allocates a new buffer, it
   * will create it with size bufferSize.
   */
  private void configureSource(int bufferSize, ByteSource source) {
    if (null != this.source) {
      this.source.detach();
    }
    source.attach(bufferSize, this);
    this.source = source;
  }

  @Override
  public void readNull() throws IOException {
  }

  @Override
  public boolean readBoolean() throws IOException {
    // inlined, shorter version of ensureBounds
    if (limit == pos) {
      limit = source.tryReadRaw(buf, 0, buf.length);
      pos = 0;
      if (limit == 0) {
        throw new EOFException();
      }
    }
    int n = buf[pos++] & 0xff;
    return n == 1;
  }

  @Override
  public int readInt() throws IOException {
    ensureBounds(5); // won't throw index out of bounds
    int len = 1;
    int b = buf[pos] & 0xff;
    int n = b & 0x7f;
    if (b > 0x7f) {
      b = buf[pos + len++] & 0xff;
      n ^= (b & 0x7f) << 7;
      if (b > 0x7f) {
        b = buf[pos + len++] & 0xff;
        n ^= (b & 0x7f) << 14;
        if (b > 0x7f) {
          b = buf[pos + len++] & 0xff;
          n ^= (b & 0x7f) << 21;
          if (b > 0x7f) {
            b = buf[pos + len++] & 0xff;
            n ^= (b & 0x7f) << 28;
            if (b > 0x7f) {
              throw new InvalidNumberEncodingException("Invalid int encoding");
            }
          }
        }
      }
    }
    pos += len;
    if (pos > limit) {
      throw new EOFException();
    }
    return (n >>> 1) ^ -(n & 1); // back to two's-complement
  }

  @Override
  public long readLong() throws IOException {
    ensureBounds(10);
    int b = buf[pos++] & 0xff;
    int n = b & 0x7f;
    long l;
    if (b > 0x7f) {
      b = buf[pos++] & 0xff;
      n ^= (b & 0x7f) << 7;
      if (b > 0x7f) {
        b = buf[pos++] & 0xff;
        n ^= (b & 0x7f) << 14;
        if (b > 0x7f) {
          b = buf[pos++] & 0xff;
          n ^= (b & 0x7f) << 21;
          if (b > 0x7f) {
            // only the low 28 bits can be set, so this won't carry
            // the sign bit to the long
            l = innerLongDecode((long)n);
          } else {
            l = n;
          }
        } else {
          l = n;
        }
      } else {
        l = n;
      }
    } else {
      l = n;
    }
    if (pos > limit) {
      throw new EOFException();
    }
    return (l >>> 1) ^ -(l & 1); // back to two's-complement
  }

  // splitting readLong up makes it faster because of the JVM does more
  // optimizations on small methods
  private long innerLongDecode(long l) throws IOException {
    int len = 1;
    int b = buf[pos] & 0xff;
    l ^= (b & 0x7fL) << 28;
    if (b > 0x7f) {
      b = buf[pos + len++] & 0xff;
      l ^= (b & 0x7fL) << 35;
      if (b > 0x7f) {
        b = buf[pos + len++] & 0xff;
        l ^= (b & 0x7fL) << 42;
        if (b > 0x7f) {
          b = buf[pos + len++] & 0xff;
          l ^= (b & 0x7fL) << 49;
          if (b > 0x7f) {
            b = buf[pos + len++] & 0xff;
            l ^= (b & 0x7fL) << 56;
            if (b > 0x7f) {
              b = buf[pos + len++] & 0xff;
              l ^= (b & 0x7fL) << 63;
              if (b > 0x7f) {
                throw new InvalidNumberEncodingException("Invalid long encoding");
              }
            }
          }
        }
      }
    }
    pos += len;
    return l;
  }

  @Override
  public float readFloat() throws IOException {
    ensureBounds(4);
    int len = 1;
    int n = (buf[pos] & 0xff) | ((buf[pos + len++] & 0xff) << 8)
        | ((buf[pos + len++] & 0xff) << 16) | ((buf[pos + len++] & 0xff) << 24);
    if ((pos + 4) > limit) {
      throw new EOFException();
    }
    pos += 4;
    return Float.intBitsToFloat(n);
  }

  @Override
  public double readDouble() throws IOException {
    ensureBounds(8);
    int len = 1;
    int n1 = (buf[pos] & 0xff) | ((buf[pos + len++] & 0xff) << 8)
        | ((buf[pos + len++] & 0xff) << 16) | ((buf[pos + len++] & 0xff) << 24);
    int n2 = (buf[pos + len++] & 0xff) | ((buf[pos + len++] & 0xff) << 8)
        | ((buf[pos + len++] & 0xff) << 16) | ((buf[pos + len++] & 0xff) << 24);
    if ((pos + 8) > limit) {
      throw new EOFException();
    }
    pos += 8;
    return Double.longBitsToDouble((((long) n1) & 0xffffffffL)
        | (((long) n2) << 32));
  }

  @Override
  public Utf8 readString(Utf8 old) throws IOException {
    long length = readLong();
    if (length > MAX_ARRAY_SIZE) {
      throw new UnsupportedOperationException("Cannot read strings longer than " + MAX_ARRAY_SIZE + " bytes");
    }
    if (length < 0L) {
      throw new AvroRuntimeException("Malformed data. Length is negative: " + length);
    }
    Utf8 result = (old != null ? old : new Utf8());
    result.setByteLength((int) length);
    if (0L != length) {
      doReadBytes(result.getBytes(), 0, (int) length);
    }
    return result;
  }

  private final Utf8 scratchUtf8 = new Utf8();

  @Override
  public String readString() throws IOException {
    return readString(scratchUtf8).toString();
  }

  @Override
  public void skipString() throws IOException {
    doSkipBytes(readInt());
  }

  @Override
  public ByteBuffer readBytes(ByteBuffer old) throws IOException {
    int length = readInt();
    ByteBuffer result;
    if (old != null && length <= old.capacity()) {
      result = old;
      result.clear();
    } else {
      result = ByteBuffer.allocate(length);
    }
    doReadBytes(result.array(), result.position(), length);
    result.limit(length);
    return result;
  }

  @Override
  public void skipBytes() throws IOException {
    doSkipBytes(readInt());
  }

  @Override
  public void readFixed(byte[] bytes, int start, int length) throws IOException {
    doReadBytes(bytes, start, length);
  }

  @Override
  public void skipFixed(int length) throws IOException {
    doSkipBytes(length);
  }

  @Override
  public int readEnum() throws IOException {
    return readInt();
  }

  protected void doSkipBytes(long length) throws IOException {
    int remaining = limit - pos;
    if (length <= remaining) {
      pos += length;
    } else {
      limit = pos = 0;
      length -= remaining;
      source.skipSourceBytes(length);
    }
  }

  /**
   * Reads <tt>length</tt> bytes into <tt>bytes</tt> starting at <tt>start</tt>.
   *
   * @throws EOFException
   *           If there are not enough number of bytes in the source.
   * @throws IOException
   */
  protected void doReadBytes(byte[] bytes, int start, int length)
      throws IOException {
    if (length < 0)
      throw new AvroRuntimeException("Malformed data. Length is negative: "
                                     + length);
    int remaining = limit - pos;
    if (length <= remaining) {
      System.arraycopy(buf, pos, bytes, start, length);
      pos += length;
    } else {
      // read the rest of the buffer
      System.arraycopy(buf, pos, bytes, start, remaining);
      start += remaining;
      length -= remaining;
      pos = limit;
      // finish from the byte source
      source.readRaw(bytes, start, length);
    }
  }

  /**
   * Returns the number of items to follow in the current array or map. Returns
   * 0 if there are no more items in the current array and the array/map has
   * ended.
   *
   * @throws IOException
   */
  protected long doReadItemCount() throws IOException {
    long result = readLong();
    if (result < 0) {
      readLong(); // Consume byte-count if present
      result = -result;
    }
    return result;
  }

  /**
   * Reads the count of items in the current array or map and skip those items,
   * if possible. If it could skip the items, keep repeating until there are no
   * more items left in the array or map. If items cannot be skipped (because
   * byte count to skip is not found in the stream) return the count of the
   * items found. The client needs to skip the items individually.
   *
   * @return Zero if there are no more items to skip and end of array/map is
   *         reached. Positive number if some items are found that cannot be
   *         skipped and the client needs to skip them individually.
   * @throws IOException
   */
  private long doSkipItems() throws IOException {
    long result = readInt();
    while (result < 0) {
      long bytecount = readLong();
      doSkipBytes(bytecount);
      result = readInt();
    }
    return result;
  }

  @Override
  public long readArrayStart() throws IOException {
    return doReadItemCount();
  }

  @Override
  public long arrayNext() throws IOException {
    return doReadItemCount();
  }

  @Override
  public long skipArray() throws IOException {
    return doSkipItems();
  }

  @Override
  public long readMapStart() throws IOException {
    return doReadItemCount();
  }
<commit_msg>AVRO-2351: Long/Int in Binary Decoder<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.avro.io;

import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Arrays;

import org.apache.avro.AvroRuntimeException;
import org.apache.avro.InvalidNumberEncodingException;
import org.apache.avro.util.Utf8;

/** An {@link Decoder} for binary-format data.
 * <p/>
 * Instances are created using {@link DecoderFactory}.
 * <p/>
 * This class may read-ahead and buffer bytes from the source beyond what is
 * required to serve its read methods.
 * The number of unused bytes in the buffer can be accessed by
 * inputStream().remaining(), if the BinaryDecoder is not 'direct'.
 *
 * @see Encoder
 */

public class BinaryDecoder extends Decoder {

  /**
   * The maximum size of array to allocate.
   * Some VMs reserve some header words in an array.
   * Attempts to allocate larger arrays may result in
   * OutOfMemoryError: Requested array size exceeds VM limit
   */
  private static final long MAX_ARRAY_SIZE = (long) Integer.MAX_VALUE - 8L;

  private ByteSource source = null;
  // we keep the buffer and its state variables in this class and not in a
  // container class for performance reasons. This improves performance
  // over a container object by about 5% to 15%
  // for example, we could have a FastBuffer class with these state variables
  // and keep a private FastBuffer member here. This simplifies the
  // "detach source" code and source access to the buffer, but
  // hurts performance.
  private byte[] buf = null;
  private int minPos = 0;
  private int pos = 0;
  private int limit = 0;

  byte[] getBuf() { return buf; }
  int getPos() { return pos; }
  int getLimit() { return limit; }

  void setBuf(byte[] buf, int pos, int len) {
    this.buf = buf;
    this.pos = pos;
    this.limit = pos+len;
  }
  void clearBuf() { this.buf = null; }

  /** protected constructor for child classes */
  protected BinaryDecoder() {
  }

  BinaryDecoder(InputStream in, int bufferSize) {
    super();
    configure(in, bufferSize);
  }

  BinaryDecoder(byte[] data, int offset, int length) {
    super();
    configure(data, offset, length);
  }

  BinaryDecoder configure(InputStream in, int bufferSize) {
    configureSource(bufferSize, new InputStreamByteSource(in));
    return this;
  }

  BinaryDecoder configure(byte[] data, int offset, int length) {
    configureSource(DecoderFactory.DEFAULT_BUFFER_SIZE, new ByteArrayByteSource(
        data, offset, length));
    return this;
  }

  /**
   * Initializes this decoder with a new ByteSource. Detaches the old source (if
   * it exists) from this Decoder. The old source's state no longer depends on
   * this Decoder and its InputStream interface will continue to drain the
   * remaining buffer and source data.
   * <p/>
   * The decoder will read from the new source. The source will generally
   * replace the buffer with its own. If the source allocates a new buffer, it
   * will create it with size bufferSize.
   */
  private void configureSource(int bufferSize, ByteSource source) {
    if (null != this.source) {
      this.source.detach();
    }
    source.attach(bufferSize, this);
    this.source = source;
  }

  @Override
  public void readNull() throws IOException {
  }

  @Override
  public boolean readBoolean() throws IOException {
    // inlined, shorter version of ensureBounds
    if (limit == pos) {
      limit = source.tryReadRaw(buf, 0, buf.length);
      pos = 0;
      if (limit == 0) {
        throw new EOFException();
      }
    }
    int n = buf[pos++] & 0xff;
    return n == 1;
  }

  @Override
  public int readInt() throws IOException {
    ensureBounds(5); // won't throw index out of bounds
    int len = 1;
    int b = buf[pos] & 0xff;
    int n = b & 0x7f;
    if (b > 0x7f) {
      b = buf[pos + len++] & 0xff;
      n ^= (b & 0x7f) << 7;
      if (b > 0x7f) {
        b = buf[pos + len++] & 0xff;
        n ^= (b & 0x7f) << 14;
        if (b > 0x7f) {
          b = buf[pos + len++] & 0xff;
          n ^= (b & 0x7f) << 21;
          if (b > 0x7f) {
            b = buf[pos + len++] & 0xff;
            n ^= (b & 0x7f) << 28;
            if (b > 0x7f) {
              throw new InvalidNumberEncodingException("Invalid int encoding");
            }
          }
        }
      }
    }
    pos += len;
    if (pos > limit) {
      throw new EOFException();
    }
    return (n >>> 1) ^ -(n & 1); // back to two's-complement
  }

  @Override
  public long readLong() throws IOException {
    ensureBounds(10);
    int b = buf[pos++] & 0xff;
    int n = b & 0x7f;
    long l;
    if (b > 0x7f) {
      b = buf[pos++] & 0xff;
      n ^= (b & 0x7f) << 7;
      if (b > 0x7f) {
        b = buf[pos++] & 0xff;
        n ^= (b & 0x7f) << 14;
        if (b > 0x7f) {
          b = buf[pos++] & 0xff;
          n ^= (b & 0x7f) << 21;
          if (b > 0x7f) {
            // only the low 28 bits can be set, so this won't carry
            // the sign bit to the long
            l = innerLongDecode((long)n);
          } else {
            l = n;
          }
        } else {
          l = n;
        }
      } else {
        l = n;
      }
    } else {
      l = n;
    }
    if (pos > limit) {
      throw new EOFException();
    }
    return (l >>> 1) ^ -(l & 1); // back to two's-complement
  }

  // splitting readLong up makes it faster because of the JVM does more
  // optimizations on small methods
  private long innerLongDecode(long l) throws IOException {
    int len = 1;
    int b = buf[pos] & 0xff;
    l ^= (b & 0x7fL) << 28;
    if (b > 0x7f) {
      b = buf[pos + len++] & 0xff;
      l ^= (b & 0x7fL) << 35;
      if (b > 0x7f) {
        b = buf[pos + len++] & 0xff;
        l ^= (b & 0x7fL) << 42;
        if (b > 0x7f) {
          b = buf[pos + len++] & 0xff;
          l ^= (b & 0x7fL) << 49;
          if (b > 0x7f) {
            b = buf[pos + len++] & 0xff;
            l ^= (b & 0x7fL) << 56;
            if (b > 0x7f) {
              b = buf[pos + len++] & 0xff;
              l ^= (b & 0x7fL) << 63;
              if (b > 0x7f) {
                throw new InvalidNumberEncodingException("Invalid long encoding");
              }
            }
          }
        }
      }
    }
    pos += len;
    return l;
  }

  @Override
  public float readFloat() throws IOException {
    ensureBounds(4);
    int len = 1;
    int n = (buf[pos] & 0xff) | ((buf[pos + len++] & 0xff) << 8)
        | ((buf[pos + len++] & 0xff) << 16) | ((buf[pos + len++] & 0xff) << 24);
    if ((pos + 4) > limit) {
      throw new EOFException();
    }
    pos += 4;
    return Float.intBitsToFloat(n);
  }

  @Override
  public double readDouble() throws IOException {
    ensureBounds(8);
    int len = 1;
    int n1 = (buf[pos] & 0xff) | ((buf[pos + len++] & 0xff) << 8)
        | ((buf[pos + len++] & 0xff) << 16) | ((buf[pos + len++] & 0xff) << 24);
    int n2 = (buf[pos + len++] & 0xff) | ((buf[pos + len++] & 0xff) << 8)
        | ((buf[pos + len++] & 0xff) << 16) | ((buf[pos + len++] & 0xff) << 24);
    if ((pos + 8) > limit) {
      throw new EOFException();
    }
    pos += 8;
    return Double.longBitsToDouble((((long) n1) & 0xffffffffL)
        | (((long) n2) << 32));
  }

  @Override
  public Utf8 readString(Utf8 old) throws IOException {
    long length = readLong();
    if (length > MAX_ARRAY_SIZE) {
      throw new UnsupportedOperationException("Cannot read strings longer than " + MAX_ARRAY_SIZE + " bytes");
    }
    if (length < 0L) {
      throw new AvroRuntimeException("Malformed data. Length is negative: " + length);
    }
    Utf8 result = (old != null ? old : new Utf8());
    result.setByteLength((int) length);
    if (0L != length) {
      doReadBytes(result.getBytes(), 0, (int) length);
    }
    return result;
  }

  private final Utf8 scratchUtf8 = new Utf8();

  @Override
  public String readString() throws IOException {
    return readString(scratchUtf8).toString();
  }

  @Override
  public void skipString() throws IOException {
    doSkipBytes(readLong());
  }

  @Override
  public ByteBuffer readBytes(ByteBuffer old) throws IOException {
    int length = readInt();
    ByteBuffer result;
    if (old != null && length <= old.capacity()) {
      result = old;
      result.clear();
    } else {
      result = ByteBuffer.allocate(length);
    }
    doReadBytes(result.array(), result.position(), length);
    result.limit(length);
    return result;
  }

  @Override
  public void skipBytes() throws IOException {
    doSkipBytes(readLong());
  }

  @Override
  public void readFixed(byte[] bytes, int start, int length) throws IOException {
    doReadBytes(bytes, start, length);
  }

  @Override
  public void skipFixed(int length) throws IOException {
    doSkipBytes(length);
  }

  @Override
  public int readEnum() throws IOException {
    return readInt();
  }

  protected void doSkipBytes(long length) throws IOException {
    int remaining = limit - pos;
    if (length <= remaining) {
      pos += length;
    } else {
      limit = pos = 0;
      length -= remaining;
      source.skipSourceBytes(length);
    }
  }

  /**
   * Reads <tt>length</tt> bytes into <tt>bytes</tt> starting at <tt>start</tt>.
   *
   * @throws EOFException
   *           If there are not enough number of bytes in the source.
   * @throws IOException
   */
  protected void doReadBytes(byte[] bytes, int start, int length)
      throws IOException {
    if (length < 0)
      throw new AvroRuntimeException("Malformed data. Length is negative: "
                                     + length);
    int remaining = limit - pos;
    if (length <= remaining) {
      System.arraycopy(buf, pos, bytes, start, length);
      pos += length;
    } else {
      // read the rest of the buffer
      System.arraycopy(buf, pos, bytes, start, remaining);
      start += remaining;
      length -= remaining;
      pos = limit;
      // finish from the byte source
      source.readRaw(bytes, start, length);
    }
  }

  /**
   * Returns the number of items to follow in the current array or map. Returns
   * 0 if there are no more items in the current array and the array/map has
   * ended. Arrays are encoded as a series of blocks. Each block consists of a
   * long count value, followed by that many array items. A block with count
   * zero indicates the end of the array. If a block's count is negative, its
   * absolute value is used, and the count is followed immediately by a long
   * block size indicating the number of bytes in the block.
   *
   * @throws IOException If the first byte cannot be read for any reason other
   *           than the end of the file, if the input stream has been closed, or
   *           if some other I/O error occurs.
   */
  protected long doReadItemCount() throws IOException {
    long result = readLong();
    if (result < 0L) {
      // Consume byte-count if present
      readLong();
      result = -result;
    }
    return result;
  }

  /**
   * Reads the count of items in the current array or map and skip those items,
   * if possible. If it could skip the items, keep repeating until there are no
   * more items left in the array or map. Arrays are encoded as a series of
   * blocks. Each block consists of a long count value, followed by that many
   * array items. A block with count zero indicates the end of the array. If a
   * block's count is negative, its absolute value is used, and the count is
   * followed immediately by a long block size indicating the number of bytes in
   * the block. If block size is missing, this method return
   * the count of the items found. The client needs to skip the items
   * individually.
   *
   * @return Zero if there are no more items to skip and end of array/map is
   *         reached. Positive number if some items are found that cannot be
   *         skipped and the client needs to skip them individually.
   *
   * @throws IOException If the first byte cannot be read for any reason other
   *           than the end of the file, if the input stream has been closed, or
   *           if some other I/O error occurs.
   */
  private long doSkipItems() throws IOException {
    long result = readLong();
    while (result < 0L) {
      final long bytecount = readLong();
      doSkipBytes(bytecount);
      result = readLong();
    }
    return result;
  }

  @Override
  public long readArrayStart() throws IOException {
    return doReadItemCount();
  }

  @Override
  public long arrayNext() throws IOException {
    return doReadItemCount();
  }

  @Override
  public long skipArray() throws IOException {
    return doSkipItems();
  }

  @Override
  public long readMapStart() throws IOException {
    return doReadItemCount();
  }

{'fraction_non_alphanumeric': 0.11285796655775514, 'fraction_numerical': 0.01437632135306554, 'mean_word_length': 2.385296031229668, 'pattern_counts': {'":': 0, '<': 127, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4940128', 'n_tokens_mistral': 8218, 'n_tokens_neox': 7823, 'n_words': 2901}