<commit_before>module Seaboar
  class Encoder
    FLOAT_WIDTHS = {
      :half => FLOAT_HALF
    }

    DEFAULTS = {
      :float_width => :half
    }

    def initialize(input, options={})
      @input = input
      @output = ""
      @output.force_encoding('ASCII-8BIT')
      @options = DEFAULTS.merge(options)
    end

    def encode_fixnum(n)
      type = n >= 0 ? MAJ_TYPE_UINT : MAJ_TYPE_NEG_INT
      value = type == MAJ_TYPE_UINT ? n : -1 - n

      if value <= UINT_MAX_INLINE
        put_type(type, value)
      else
        put_numeric_bytes(type, value)
      end
    end

    def put(byte)
      @output << byte.chr
    end

    def put_type(type, extra)
      put(((type << 5) & 0b11100000) | (extra & 0b00011111))
    end

    def put_numeric_bytes(type, n)
      type_bits = type << 5
      case
      when n <= UINT_MAX_INLINE
        put(type_bits | n)
      when n <= 0xff
        put(type_bits | SUBTYPE_UINT8)
        put(n)
      when n <= 0xffff
        put(type_bits | SUBTYPE_UINT16)
        put(n & 0xff)
        put((n >> 8) & 0xff)
      when n <= 0xffffffff
        put(type_bits | SUBTYPE_UINT32)
        put(n & 0xff)
        put((n >> 8) & 0xff)
        put((n >> 16) & 0xff)
        put((n >> 24) & 0xff)
      when n <= 0xffffffffffffffff
        put(type_bits | SUBTYPE_UINT64)
        put(n & 0xff)
        put((n >> 8) & 0xff)
        put((n >> 16) & 0xff)
        put((n >> 24) & 0xff)
        put((n >> 32) & 0xff)
        put((n >> 40) & 0xff)
        put((n >> 48) & 0xff)
        put((n >> 56) & 0xff)
      end<commit_msg>Fix endianness of encoding for unsigned integers<commit_after>module Seaboar
  class Encoder
    FLOAT_WIDTHS = {
      :half => FLOAT_HALF
    }

    DEFAULTS = {
      :float_width => :half
    }

    def initialize(input, options={})
      @input = input
      @output = ""
      @output.force_encoding('ASCII-8BIT')
      @options = DEFAULTS.merge(options)
    end

    def encode_fixnum(n)
      type = n >= 0 ? MAJ_TYPE_UINT : MAJ_TYPE_NEG_INT
      value = type == MAJ_TYPE_UINT ? n : -1 - n

      if value <= UINT_MAX_INLINE
        put_type(type, value)
      else
        put_numeric_bytes(type, value)
      end
    end

    def put(byte)
      @output << byte.chr
    end

    def put_bytes(byte_string)
      @output << byte_string
    end

    def put_type(type, extra)
      put(((type << 5) & 0b11100000) | (extra & 0b00011111))
    end

    def put_numeric_bytes(type, n)
      type_bits = type << 5
      case
      when n <= UINT_MAX_INLINE
        put(type_bits | n)
      when n <= 0xff
        put(type_bits | SUBTYPE_UINT8)
        put(n)
      when n <= 0xffff
        put(type_bits | SUBTYPE_UINT16)
        put_bytes([n].pack("S>"))
      when n <= 0xffffffff
        put(type_bits | SUBTYPE_UINT32)
        put_bytes([n].pack("L>"))
      when n <= 0xffffffffffffffff
        put(type_bits | SUBTYPE_UINT64)
        put_bytes([n].pack("Q>"))
      end
{'fraction_non_alphanumeric': 0.12054329371816638, 'fraction_numerical': 0.03429541595925297, 'mean_word_length': 1.9284294234592445, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '623321', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1026, 'n_words': 234}