<commit_before>part of archive;

class InputBuffer {
  final List<int> buffer;
  int position = 0;
  final int byteOrder;

  /**
   * Create a InputBuffer for reading from a List<int>
   */
  InputBuffer(this.buffer, {this.byteOrder: LITTLE_ENDIAN}) :
    position = 0;

  /**
   * How many bytes in the buffer.
   */
  int get length => buffer.length;

  /**
   * How many bytes are left in the buffer from the current position?
   */
  int get remainder => length - position;

  /**
   * Is the current position at the end of the buffer?
   */
  bool get isEOF => position >= buffer.length;

  /**
   * Return a InputBuffer to read a subset of this buffer.
   * If [position] is not specified, the current read position is
   * used. If [length] is not specified, the remainder of this buffer is used.
   */
  InputBuffer subset([int position, int length]) {
    if (position == null || position < 0) {
      position = this.position;
    }
    if (length == null || length < 0) {
      length = this.length - position;
    }

    int end = position + length;
    if (end > buffer.length) {
      end = buffer.length;
    }

    return new InputBuffer(buffer.sublist(position, end),
                           byteOrder: byteOrder);
  }

  /**
   * Read [count] bytes from an [offset] of the current read position, without
   * moving the read position.
   */
  List<int> peekBytes(int count, [int offset = 0]) {
    List<int> bytes = buffer.sublist(position + offset,
                                     position + offset + count);
    return bytes;
  }

  /**
   * Move the read position by [count] bytes.
   */
  void skip(int count) {
    position += count;
  }

  /**
   * Read a single byte.
   */
  int readByte() {
    _bitBufferLen = 0;
    return buffer[position++];
  }

  /**
   * Read [count] bytes from the buffer.
   */
  List<int> readBytes(int count) {
    _bitBufferLen = 0;
    List<int> bytes = buffer.sublist(position, position + count);
    position += bytes.length;
    return bytes;
  }

  /**
   * Read a null-terminated string.
   */
  String readString() {
    List<int> codes = [];
    while (!isEOF) {
      int c = readByte();
      if (c == 0) {
        return new String.fromCharCodes(codes);
      }
      codes.add(c);
    }
    throw new Exception('EOF reached without finding string terminator');
  }

  /**
   * Read a 16-bit word from the buffer.
   */
  int readUint16() {
    _bitBufferLen = 0;
    int b1 = buffer[position++] & 0xff;
    int b2 = buffer[position++] & 0xff;
    if (byteOrder == BIG_ENDIAN) {
      return (b1 << 8) | b2;
    }
    return (b2 << 8) | b1;<commit_msg>update inputbuffer<commit_after>part of archive;

class InputBuffer {
  final List<int> buffer;
  int position = 0;
  final int byteOrder;

  /**
   * Create a InputBuffer for reading from a List<int>
   */
  InputBuffer(this.buffer, {this.byteOrder: LITTLE_ENDIAN}) :
    position = 0;

  /**
   * How many bytes in the buffer.
   */
  int get length => buffer.length;

  /**
   * How many bytes are left in the buffer from the current position?
   */
  int get remainder => length - position;

  /**
   * Is the current position at the end of the buffer?
   */
  bool get isEOF => position >= buffer.length;

  /**
   * Return a InputBuffer to read a subset of this buffer.
   * If [position] is not specified, the current read position is
   * used. If [length] is not specified, the remainder of this buffer is used.
   */
  InputBuffer subset([int position, int length]) {
    if (position == null || position < 0) {
      position = this.position;
    }
    if (length == null || length < 0) {
      length = this.length - position;
    }

    int end = position + length;
    if (end > buffer.length) {
      end = buffer.length;
    }

    return new InputBuffer(buffer.sublist(position, end),
                           byteOrder: byteOrder);
  }

  /**
   * Read [count] bytes from an [offset] of the current read position, without
   * moving the read position.
   */
  List<int> peekBytes(int count, [int offset = 0]) {
    List<int> bytes = buffer.sublist(position + offset,
                                     position + offset + count);
    return bytes;
  }

  /**
   * Move the read position by [count] bytes.
   */
  void skip(int count) {
    position += count;
  }

  /**
   * Read a single byte.
   */
  int readByte() {
    _bitBufferLen = 0;
    return buffer[position++];
  }

  /**
   * Read [count] bytes from the buffer.
   */
  List<int> readBytes(int count) {
    _bitBufferLen = 0;
    List<int> bytes = buffer.sublist(position, position + count);
    position += bytes.length;
    return bytes;
  }

  /**
   * Read a null-terminated string.
   */
  String readString([int len]) {
    if (len == null) {
      List<int> codes = [];
      while (!isEOF) {
        int c = readByte();
        if (c == 0) {
          return new String.fromCharCodes(codes);
        }
        codes.add(c);
      }
      throw new Exception('EOF reached without finding string terminator');
    }

    return new String.fromCharCodes(readBytes(len));
  }

  /**
   * Read a 16-bit word from the buffer.
   */
  int readUint16() {
    _bitBufferLen = 0;
    int b1 = buffer[position++] & 0xff;
    int b2 = buffer[position++] & 0xff;
    if (byteOrder == BIG_ENDIAN) {
      return (b1 << 8) | b2;
    }
    return (b2 << 8) | b1;
{'fraction_non_alphanumeric': 0.12409167132476244, 'fraction_numerical': 0.008570896217626234, 'mean_word_length': 2.233734939759036, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6694414', 'n_tokens_mistral': 1722, 'n_tokens_neox': 1658, 'n_words': 543}