<commit_before>require 'channel9/instruction'
require 'channel9/builder'
require 'json'

module Channel9
  class Stream # a stream of bytecode with label and local tables
    attr :instructions
    attr :labels
    attr :locals
    attr :line_info
    attr :framevars

    def initialize()
      @instructions = []
      @labels = {}
      @locals = {}
      @line_info = []
      @framevars = {}
      @pos = 0
    end

    def from_json(json)
      o = JSON.parse(json)
      builder = Builder.new(self)
      code = o["code"]
      code.each do |instruction|
        instruction.collect! {|i|
          if (i.is_a?(Hash))
            if (i == {'undef'=>'undef'})
              Channel9::Primitive::Undef
            else
              raise "Unknown complex type in json bytecode stream: #{i.inspect}"
            end
          else
            i
          end
        }
        builder.send(*instruction)
      end
      self
    end
    def self.from_json(json)
      new.from_json(json)
    end

    def to_json(*a)
      labels = @labels.invert
      instructions = []
      i = 0
      while (i < @instructions.length)
        instructions << ["set_label", labels[i]] if (labels[i])
        instructions << ["line", *line_info[i]] if (line_info[i])
        instructions << @instructions[i]
        i += 1
      end

      JSON.pretty_generate(
        "code" => instructions
      )
    end

    def build
      yield Builder.new(self)
    end
    def self.build(&block)
      Stream.new.build(&block)
    end

    def set_label(name)
      add_label(name)
      @labels[name] = @pos
    end
    def label(name_or_ip)
      @labels[name_or_ip] || name_or_ip
    end
<commit_msg>Handle multiple labels on the same instruction.<commit_after>require 'channel9/instruction'
require 'channel9/builder'
require 'json'

module Channel9
  class Stream # a stream of bytecode with label and local tables
    attr :instructions
    attr :labels
    attr :locals
    attr :line_info
    attr :framevars

    def initialize()
      @instructions = []
      @labels = {}
      @locals = {}
      @line_info = []
      @framevars = {}
      @pos = 0
    end

    def from_json(json)
      o = JSON.parse(json)
      builder = Builder.new(self)
      code = o["code"]
      code.each do |instruction|
        instruction.collect! {|i|
          if (i.is_a?(Hash))
            if (i == {'undef'=>'undef'})
              Channel9::Primitive::Undef
            else
              raise "Unknown complex type in json bytecode stream: #{i.inspect}"
            end
          else
            i
          end
        }
        builder.send(*instruction)
      end
      self
    end
    def self.from_json(json)
      new.from_json(json)
    end

    def to_json(*a)
      ilabels = Hash.new {|h,k| h[k] = []}
      @labels.each {|label, line| ilabels[line].push label}
      instructions = []
      i = 0
      while (i < @instructions.length)
        ilabels[i].each do |label|
          instructions << ["set_label", label]
        end
        instructions << ["line", *line_info[i]] if (line_info[i])
        instructions << @instructions[i]
        i += 1
      end

      JSON.pretty_generate(
        "code" => instructions
      )
    end

    def build
      yield Builder.new(self)
    end
    def self.build(&block)
      Stream.new.build(&block)
    end

    def set_label(name)
      add_label(name)
      @labels[name] = @pos
    end
    def label(name_or_ip)
      @labels[name_or_ip] || name_or_ip
    end

{'fraction_non_alphanumeric': 0.11256768310059846, 'fraction_numerical': 0.003989740666856654, 'mean_word_length': 1.8629690048939642, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3477179', 'n_tokens_mistral': 1103, 'n_tokens_neox': 1090, 'n_words': 273}