<commit_before>module Fraggel

  module Decoder

    class Poisioned < StandardError ; end

    def receive_data(data)
      @buf ||= ""
      @buf << data

      if ! @cs
        read_type
      else
        @cs.call
      end
    end

    def cs(&blk)
      @cs = blk
      @cs.call
    end

    def read_type
      cs do
        case @buf.slice!(0)
        when nil
          # Wait for next byte
        when ?\r
          finish do
            read_type
          end
        when ?:
          read_integer do |i|
            receive_event(:value, i)
            read_type
          end
        when ?$
          read_string do |s|
            receive_event(:value, s)
            read_type
          end
        when ?+
          read_line do |msg|
            receive_event(:status, msg)
            read_type
          end
        when ?-
          read_line do |msg|
            receive_event(:error, msg)
            read_type
          end
        when ?*
          read_integer do |count|
            receive_event(:array, count)
            read_type
          end
        else
          raise Poisioned
        end
      end
    end

    def finish(&blk)
      cs do
        c = @buf.slice!(0)
        case c
        when nil
          # Wait for next byte
        when ?\n
          blk.call
        else
          raise Poisioned
        end
      end
    end

    def read_integer(&blk)
      @int = ""
      cs do
        while c = @buf.slice!(0)
          case c
          when ?0..?9
            @int << c.chr
          when ?\r
            finish do
              blk.call(Integer(@int))
            end
          else
            raise Poisioned
          end
        end
      end
    end

    def read_string(&blk)
      read_integer do |count|
        cs do
          if @buf.length >= count
            string = @buf.slice!(0, count)
            cs do
              case @buf.slice!(0)
              when nil
                # Wait for next byte
              when ?\r
                finish do
                  blk.call(string)
                end
              else
                raise Poisioned
              end
            end
          end
        end
      end
    end

    def read_line(&blk)
      cs do
        if line = @buf.slice!(/.*\r/)
          finish do
            blk.call(line.chomp)
          end
        end
      end
    end

  end

end
<commit_msg>namespace state var in decoder<commit_after>module Fraggel

  module Decoder

    class Poisioned < StandardError ; end

    def receive_data(data)
      @buf ||= ""
      @buf << data

      if ! @dcs
        read_type
      else
        @dcs.call
      end
    end

    def dcs(&blk)
      @dcs = blk
      @dcs.call
    end

    def read_type
      dcs do
        case @buf.slice!(0)
        when nil
          # Wait for next byte
        when ?\r
          finish do
            read_type
          end
        when ?:
          read_integer do |i|
            receive_event(:value, i)
            read_type
          end
        when ?$
          read_string do |s|
            receive_event(:value, s)
            read_type
          end
        when ?+
          read_line do |msg|
            receive_event(:status, msg)
            read_type
          end
        when ?-
          read_line do |msg|
            receive_event(:error, msg)
            read_type
          end
        when ?*
          read_integer do |count|
            receive_event(:array, count)
            read_type
          end
        else
          raise Poisioned
        end
      end
    end

    def finish(&blk)
      dcs do
        c = @buf.slice!(0)
        case c
        when nil
          # Wait for next byte
        when ?\n
          blk.call
        else
          raise Poisioned
        end
      end
    end

    def read_integer(&blk)
      @int = ""
      dcs do
        while c = @buf.slice!(0)
          case c
          when ?0..?9
            @int << c.chr
          when ?\r
            finish do
              blk.call(Integer(@int))
            end
          else
            raise Poisioned
          end
        end
      end
    end

    def read_string(&blk)
      read_integer do |count|
        dcs do
          if @buf.length >= count
            string = @buf.slice!(0, count)
            dcs do
              case @buf.slice!(0)
              when nil
                # Wait for next byte
              when ?\r
                finish do
                  blk.call(string)
                end
              else
                raise Poisioned
              end
            end
          end
        end
      end
    end

    def read_line(&blk)
      dcs do
        if line = @buf.slice!(/.*\r/)
          finish do
            blk.call(line.chomp)
          end
        end
      end
    end

  end

end

{'fraction_non_alphanumeric': 0.0670807453416149, 'fraction_numerical': 0.002898550724637681, 'mean_word_length': 0.8937671501372011, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4588582', 'n_tokens_mistral': 1389, 'n_tokens_neox': 1347, 'n_words': 392}