<commit_before>class StringIO

  include Enumerable

  DEFAULT_RECORD_SEPARATOR = "\n" unless defined?(::DEFAULT_RECORD_SEPARATOR)

  def self.open(*args)
    io = new(*args)
    return io unless block_given?

    begin
      yield io
    ensure
      io.send(:finalize)
    end
  end

  attr_reader :string, :pos
  attr_accessor :lineno

  def initialize(string = "", mode = nil)
    @string = Type.coerce_to string, String, :to_str
    @pos = 0
    @lineno = 0

    if mode
      if mode.is_a?(Integer)
        mode_from_integer(mode)
      else
        mode = StringValue(mode)
        mode_from_string(mode)
      end
    else
      mode_from_string("r+")
    end

    self
  end

  def initialize_copy(from)
    from = Type.coerce_to(from, StringIO, :to_strio)

    taint if from.tainted?

    @string = from.instance_variable_get(:@string).dup
    @append = from.instance_variable_get(:@append)
    @readable = from.instance_variable_get(:@readable)
    @writable = from.instance_variable_get(:@writable)

    @pos = from.instance_variable_get(:@pos)
    @lineno = from.instance_variable_get(:@lineno)

    self
  end

  def chars
    to_enum :each_char
  end

  def each_byte
    return to_enum :each_byte unless block_given?
    raise IOError, "not opened for reading" unless @readable
    if @pos < @string.length
      @string[@pos..-1].each_byte { |b| @pos += 1; yield b}
    end
    self
  end

  alias_method :bytes, :each_byte

  def each_char
    return to_enum :each_char unless block_given?
    if $KCODE == "UTF8"
      lookup = 7.downto(4)
      while c = read(1) do
        n = c[0]
        leftmost_zero_bit = lookup.find{|i| n[i].zero? }
        case leftmost_zero_bit
        when 7 # ASCII
          yield c
        when 6 # UTF 8 complementary characters
          next # Encoding error, ignore
        else
          more = read(6-leftmost_zero_bit)
          break unless more
          yield c+more
        end
      end
    else
      while s = read(1)
        yield s
      end
    end

    self
  end<commit_msg>StringIO#chars is an alias for StringIO#each_char<commit_after>class StringIO

  include Enumerable

  DEFAULT_RECORD_SEPARATOR = "\n" unless defined?(::DEFAULT_RECORD_SEPARATOR)

  def self.open(*args)
    io = new(*args)
    return io unless block_given?

    begin
      yield io
    ensure
      io.send(:finalize)
    end
  end

  attr_reader :string, :pos
  attr_accessor :lineno

  def initialize(string = "", mode = nil)
    @string = Type.coerce_to string, String, :to_str
    @pos = 0
    @lineno = 0

    if mode
      if mode.is_a?(Integer)
        mode_from_integer(mode)
      else
        mode = StringValue(mode)
        mode_from_string(mode)
      end
    else
      mode_from_string("r+")
    end

    self
  end

  def initialize_copy(from)
    from = Type.coerce_to(from, StringIO, :to_strio)

    taint if from.tainted?

    @string = from.instance_variable_get(:@string).dup
    @append = from.instance_variable_get(:@append)
    @readable = from.instance_variable_get(:@readable)
    @writable = from.instance_variable_get(:@writable)

    @pos = from.instance_variable_get(:@pos)
    @lineno = from.instance_variable_get(:@lineno)

    self
  end

  def each_byte
    return to_enum :each_byte unless block_given?
    raise IOError, "not opened for reading" unless @readable
    if @pos < @string.length
      @string[@pos..-1].each_byte { |b| @pos += 1; yield b}
    end
    self
  end

  alias_method :bytes, :each_byte

  def each_char
    return to_enum :each_char unless block_given?
    if $KCODE == "UTF8"
      lookup = 7.downto(4)
      while c = read(1) do
        n = c[0]
        leftmost_zero_bit = lookup.find{|i| n[i].zero? }
        case leftmost_zero_bit
        when 7 # ASCII
          yield c
        when 6 # UTF 8 complementary characters
          next # Encoding error, ignore
        else
          more = read(6-leftmost_zero_bit)
          break unless more
          yield c+more
        end
      end
    else
      while s = read(1)
        yield s
      end
    end

    self
  end

  alias_method :chars, :each_char
{'fraction_non_alphanumeric': 0.09088699878493317, 'fraction_numerical': 0.006804374240583232, 'mean_word_length': 2.3572593800978794, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '714163', 'n_tokens_mistral': 1431, 'n_tokens_neox': 1351, 'n_words': 346}