<commit_before>
require 'stringio'

require 'parslet/source/line_cache'

# Wraps the input IO to parslet. The interface defined by this class is 
# smaller than what IO offers, but enhances it with a #column and #line 
# method for the current position. 
#
class Parslet::Source
  def initialize(io)
    if io.respond_to? :to_str
      io = StringIO.new(io)
    end
    
    @io = io
    @virtual_position = @io.pos
    @eof_position = nil
    
    @line_cache = LineCache.new
    
    # Stores an array of <offset, buffer> tuples. 
    @slices = []
  end
  
  # Reads n chars from the input and returns a Range instance. 
  #
  def read(n)
    slice = read_from_cache(@virtual_position, n)
    @virtual_position += slice.size
    
    slice
  end
  
  def eof?
    @eof_position && @virtual_position >= @eof_position
  end
  def pos
    @virtual_position
  end
  def pos=(new_pos)
    @virtual_position = new_pos
  end

  # Returns a <line, column> tuple for the given position. If no position is
  # given, line/column information is returned for the current position given
  # by #pos. 
  #
  def line_and_column(position=nil)
    @line_cache.line_and_column(position || self.pos)
  end
  
private
  # Minimal size of a single read
  MIN_READ_SIZE = 500
  # Number of slices to keep 
  BUFFER_CACHE_SIZE = 3
  <commit_msg>. tuning<commit_after>
require 'stringio'

require 'parslet/source/line_cache'

# Wraps the input IO to parslet. The interface defined by this class is 
# smaller than what IO offers, but enhances it with a #column and #line 
# method for the current position. 
#
class Parslet::Source
  def initialize(io)
    if io.respond_to? :to_str
      io = StringIO.new(io)
    end
    
    @io = io
    @virtual_position = @io.pos
    @eof_position = nil
    
    @line_cache = LineCache.new
    
    # Stores an array of <offset, buffer> tuples. 
    @slices = []
  end
  
  # Reads n chars from the input and returns a Range instance. 
  #
  def read(n)
    slice = read_from_cache(@virtual_position, n)
    @virtual_position += slice.size
    
    slice
  end
  
  def eof?
    @eof_position && @virtual_position >= @eof_position
  end
  def pos
    @virtual_position
  end
  def pos=(new_pos)
    @virtual_position = new_pos
  end

  # Returns a <line, column> tuple for the given position. If no position is
  # given, line/column information is returned for the current position given
  # by #pos. 
  #
  def line_and_column(position=nil)
    @line_cache.line_and_column(position || self.pos)
  end
  
private
  # Minimal size of a single read
  MIN_READ_SIZE = 10 * 1024
  # Number of slices to keep 
  BUFFER_CACHE_SIZE = 10
  
{'fraction_non_alphanumeric': 0.0784313725490196, 'fraction_numerical': 0.004524886877828055, 'mean_word_length': 2.884333821376281, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6573513', 'n_tokens_mistral': 910, 'n_tokens_neox': 874, 'n_words': 290}