<commit_before>require 'awesome_print'

require_relative '../shared/opcodes.rb'

def errn(str, *args)
  if args.count > 0
    str = sprintf(str, *args)
  end
  STDERR.print(str)
end

def err(str, *args)
  errn(str.to_s + "\n", *args)
end

def errap(arg)
  STDERR.puts arg.ai
end

class DabInputStream
  def initialize(stream = nil)
    @stream = stream || STDIN
    @function_cache = ''
  end

  def read_preamble
    {
      dab_mark: _read(3),
      compiler_version: read_uint64,
      vm_version: read_uint64,
      code_length: read_uint64,
      code_crc: read_uint64,
    }
  end

  def _read(n)
    return nil if @stream.eof?
    ret = @stream.read(n)
    @function_cache += ret
    ret
  end

  def read_uint8
    _read(1).unpack('C')[0]
  rescue
    nil
  end

  def read_uint16
    _read(2).unpack('S<')[0]
  rescue
    nil
  end

  def read_uint64
    _read(8).unpack('Q<')[0]
  rescue
    nil
  end

  def read_vlc
    len = read_uint8
    if len == 255<commit_msg>Removed unnecessary code<commit_after>require 'awesome_print'

require_relative '../shared/opcodes.rb'

def errn(str, *args)
  if args.count > 0
    str = sprintf(str, *args)
  end
  STDERR.print(str)
end

def err(str, *args)
  errn(str.to_s + "\n", *args)
end

def errap(arg)
  STDERR.puts arg.ai
end

class DabInputStream
  def initialize(stream = nil)
    @stream = stream || STDIN
  end

  def read_preamble
    {
      dab_mark: _read(3),
      compiler_version: read_uint64,
      vm_version: read_uint64,
      code_length: read_uint64,
      code_crc: read_uint64,
    }
  end

  def _read(n)
    return nil if @stream.eof?
    @stream.read(n)
  end

  def read_uint8
    _read(1).unpack('C')[0]
  rescue
    nil
  end

  def read_uint16
    _read(2).unpack('S<')[0]
  rescue
    nil
  end

  def read_uint64
    _read(8).unpack('Q<')[0]
  rescue
    nil
  end

  def read_vlc
    len = read_uint8
    if len == 255
{'fraction_non_alphanumeric': 0.11467648605996844, 'fraction_numerical': 0.026301946344029457, 'mean_word_length': 2.671814671814672, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '51064', 'n_tokens_mistral': 801, 'n_tokens_neox': 772, 'n_words': 172}