<commit_before>require 'irb/ruby-lex'
require 'stringio'

class MimickIRB < RubyLex
  attr_accessor :started

  class Continue < StandardError; end
  class Empty < StandardError; end

  def initialize
    super
    set_input(StringIO.new)
  end

  def run(str,line_no=nil)
    obj = nil
    @io << str
    @io.rewind
    unless l = lex
      raise Empty if @line == ''
    else
      case l.strip
      when "reset"
        @line = ""
      when "time"
        @line = "puts %{You started \#{IRBalike.started.since} ago.}"
      else
        @line << l << "\n"
        if @ltype or @continue or @indent > 0
          raise Continue
        end
      end
    end
    unless @line.empty?
      obj = eval @line, TOPLEVEL_BINDING
    end
    @line = ''
    #@line_no = line_no if line_no
    @exp_line_no = line_no || @line_no

    @indent = 0
    @indent_stack = []

    $stdout.rewind
    output = $stdout.read
    $stdout.truncate(0)
    $stdout.rewind
    [output, obj]
  rescue Object => e
    case e when Empty, Continue
    else @line = ""
    end
    raise e
  ensure
    set_input(StringIO.new)
  end

end

# TO DO: output of blocks is gathered and shown at the end of the block


class ExampleExpander
  def initialize(sep=" -> ", align=52)
    @sep = sep
    @align = align
    @irb = MimickIRB.new
    @output = ""
  end
  def add_line(line,line_num=nil)
    line = line.chomp
    line = $1 if /(.+)#{@sep}.*/.match line
    $stdout = StringIO.new
    begin
      out,obj = @irb.run(line, line_num)
      @output << line_output(line,out)
    rescue MimickIRB::Empty
      @output << line_output(line)
    rescue MimickIRB::Continue
      @output << line_output(line)<commit_msg>Comment for future modification of the documentation examples expander.<commit_after>require 'irb/ruby-lex'
require 'stringio'

class MimickIRB < RubyLex
  attr_accessor :started

  class Continue < StandardError; end
  class Empty < StandardError; end

  def initialize
    super
    set_input(StringIO.new)
  end

  def run(str,line_no=nil)
    obj = nil
    @io << str
    @io.rewind
    unless l = lex
      raise Empty if @line == ''
    else
      case l.strip
      when "reset"
        @line = ""
      when "time"
        @line = "puts %{You started \#{IRBalike.started.since} ago.}"
      else
        @line << l << "\n"
        if @ltype or @continue or @indent > 0
          raise Continue
        end
      end
    end
    unless @line.empty?
      obj = eval @line, TOPLEVEL_BINDING
    end
    @line = ''
    #@line_no = line_no if line_no
    @exp_line_no = line_no || @line_no

    @indent = 0
    @indent_stack = []

    $stdout.rewind
    output = $stdout.read
    $stdout.truncate(0)
    $stdout.rewind
    [output, obj]
  rescue Object => e
    case e when Empty, Continue
    else @line = ""
    end
    raise e
  ensure
    set_input(StringIO.new)
  end

end

# TO DO: output of blocks is gathered and shown at the end of the block
# when lines in a block have ->, they should be rememberd, and when
# output is generated when closing the block, each output line should
# be appended to the in-block -> lines before showing lines after the block.


class ExampleExpander
  def initialize(sep=" -> ", align=52)
    @sep = sep
    @align = align
    @irb = MimickIRB.new
    @output = ""
  end
  def add_line(line,line_num=nil)
    line = line.chomp
    line = $1 if /(.+)#{@sep}.*/.match line
    $stdout = StringIO.new
    begin
      out,obj = @irb.run(line, line_num)
      @output << line_output(line,out)
    rescue MimickIRB::Empty
      @output << line_output(line)
    rescue MimickIRB::Continue
      @output << line_output(line)
{'fraction_non_alphanumeric': 0.10775980257746093, 'fraction_numerical': 0.003290375651220181, 'mean_word_length': 2.3467889908256883, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4453777', 'n_tokens_mistral': 1273, 'n_tokens_neox': 1226, 'n_words': 364}