<commit_before># encoding: utf-8

class F4VIO < StringIO

  def byte
    self.read(1).unpack('C').first
  end

  def int16
    self.read(2).unpack('n').first
  end

  def int24
    "\x00#{self.read(3)}".unpack('N').first
  end

  def int32
    self.read(4).unpack('N').first
  end

  def int64
    hi = int_32
    lo = int_32
    (hi * 4294967296) + lo
  end

  def fourCC
    self.read(4).unpack('A4').first
  end

  def string
    o, p = self.pos, 0
    p += 1 while (self.read(1) != "\x00")

    self.pos = o
    str = self.read(p)
    self.pos += 1

    str
  end

end<commit_msg>fixing 64 bit integer access and adding decoding of array of bytes<commit_after># encoding: utf-8

class F4VIO < StringIO

  def byte
    self.read(1).unpack('C').first
  end

  def int16
    self.read(2).unpack('n').first
  end

  def int24
    "\x00#{self.read(3)}".unpack('N').first
  end

  def int32
    self.read(4).unpack('N').first
  end

  def int64
    hi = int32
    lo = int32
    (hi * 4294967296) + lo
  end

  def fourCC
    self.read(4).unpack('A4').first
  end

  def string
    o, p = self.pos, 0
    p += 1 while (self.read(1) != "\x00")

    self.pos = o
    str = self.read(p)
    self.pos += 1

    str
  end

  def byte_ar
    ar = []
    byte.times { ar << byte }
    ar
  end
end
{'fraction_non_alphanumeric': 0.14440993788819875, 'fraction_numerical': 0.06055900621118013, 'mean_word_length': 2.1439024390243904, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '63970', 'n_tokens_mistral': 573, 'n_tokens_neox': 516, 'n_words': 130}