<commit_before>class ScanError < StandardError; end

class StringScanner
  Id = "bite me $Id".freeze
  Version = "1.0.0".freeze

  attr_accessor :pos
  attr_reader :match, :string

  alias :pointer :pos
  alias :pointer= :pos=

  def [] n
    match.to_a[n]
  end

  def bol?
    pos == 0 or string[pos-1..pos-1] == "\n"
  end
  alias :beginning_of_line? :bol?

  def check pattern
    _scan pattern, false, true, true
  end

  def check_until pattern
    _scan pattern, false, true, false
  end

  def clear
    warn "StringScanner#clear is obsolete; use #terminate instead"
    terminate
  end

  def concat str
    self.string << str
    self
  end
  alias :<< :concat # TODO: reverse

  def empty?
    warn "StringScanner#empty? is obsolete; use #eos? instead?"
    eos?
  end

  def eos?
    _lame_guard

    self.pos >= self.string.size
  end

  def exist? pattern
    _scan pattern, false, false, false
  end

  def get_byte
    scan(/./mn)
  end

  def getbyte
    warn "StringScanner#getbyte is obsolete; use #get_byte instead"
    get_byte
  end

  def getch
    scan(/./m)
  end

  def initialize string, dup = false
    self.string = string
    self.reset
  end

  def inspect
    if defined? @string then
      rest = string.size > 5 ? string[pos..pos+4] + "..." : string
      r = if eos? then
            "#<StringScanner fin>"
          elsif pos > 0 then
            prev = string[0...pos].inspect
            "#<StringScanner #{pos}/#{string.size} #{prev} @ #{rest.inspect}>"
          else
            "#<StringScanner #{pos}/#{string.size} @ #{rest.inspect}>"
          end
      r.taint if self.string.tainted?
      r
    else
      "#<StringScanner (uninitialized)>"
    end
  end

  def match? pattern
    _scan pattern, false, false, true
  end

  def matched
    match.to_s if matched?
  end

  def matched?
    not match.nil?
  end

  def matched_size
    match.to_s.size if matched?
  end

  def matchedsize
    warn "StringScanner#matchedsize is obsolete; use #matched_size instead"
    matched_size
  end

  def post_match
    match.post_match if matched?
  end

  def pre_match
    string[0, match.begin(0)] if matched?
  end

  def reset
    @prev_pos = self.pos = 0
    @match = nil
    self
  end

  def rest
    string[pos..-1]
  end

  def rest?
    return !eos?
  end

  def rest_size
    self.rest.size
  end

  def restsize
    warn "StringScanner#restsize is obsolete; use #rest_size instead"
    rest_size
  end

  def scan pattern
    _scan pattern, true, true, true
  end

  def scan_until pattern
    _scan pattern, true, true, false
  end

  def scan_full pattern, succptr, getstr
    _scan pattern, succptr, getstr, true
  end

  def search_full pattern, succptr, getstr
    _scan pattern, succptr, getstr, false
  end

  def self.must_C_version
    self
  end

  def skip pattern
    _scan pattern, true, false, true
  end

  def skip_until pattern
    _scan pattern, true, false, false
  end

  def string= s
    reset
    @string = s
  end

  def terminate
    @match = nil
    self.pos = string.size
    self
  end

  def unscan
    raise ScanError if @match.nil?
    self.pos = @prev_pos
    @prev_pos = nil
    @match = nil
    self
  end

  def peek len
    raise ArgumentError if len < 0
    return "" if len.zero?
    return string.substring(pos, len)
  end

  def peep len
    warn "StringScanner#peep is obsolete; use #peek instead"
    peek len
  end

  def _scan pattern, succptr, getstr, headonly
    _lame_guard

    @match = nil

    return nil if (string.size - pos) < 0 # TODO: make more elegant

    @match = if headonly then
               pattern.match_start(string, pos)
             else
               pattern.search_region(string, pos, string.size, true)
             end

    return nil if match.nil?

    m = string[pos...match.end(0)]

    if succptr then
      @prev_pos = pos
      self.pos += m.size
    end

    if getstr then
      m
    else
      m.size
    end
  end
  private :_scan

  def _lame_guard
    raise ArgumentError unless instance_variable_defined?(:@string)
  end
  private :_lame_guard
end
<commit_msg>Commented out everything until I was 100% failure instead of 80% error... whoever wrote this code used a lot of rubinius primitives instead of pure MRI<commit_after>class ScanError < StandardError; end

p "loaded pure ruby StringScanner"

class StringScanner
  Id = "bite me $Id".freeze
  Version = "1.0.0".freeze

  attr_accessor :pos
  attr_reader :match, :string

  alias :pointer :pos
  alias :pointer= :pos=

  def [] n
    match.to_a[n]
  end

  def bol?
    pos == 0 or string[pos-1..pos-1] == "\n"
  end
  alias :beginning_of_line? :bol?

  def check pattern
    _scan pattern, false, true, true
  end

  def check_until pattern
    _scan pattern, false, true, false
  end

  def clear
    warn "StringScanner#clear is obsolete; use #terminate instead"
    terminate
  end

  def concat str
    self.string << str
    self
  end
  alias :<< :concat # TODO: reverse

  def empty?
    warn "StringScanner#empty? is obsolete; use #eos? instead?"
    eos?
  end

  def eos?
    _lame_guard

    self.pos >= self.string.size
  end

  def exist? pattern
    _scan pattern, false, false, false
  end

  def get_byte
    scan(/./mn)
  end

  def getbyte
    warn "StringScanner#getbyte is obsolete; use #get_byte instead"
    get_byte
  end

  def getch
    scan(/./m)
  end

  def initialize string, dup = false
    self.string = string
    self.reset
  end

  def inspect
    if defined? @string then
      rest = string.size > 5 ? string[pos..pos+4] + "..." : string
      r = if eos? then
            "#<StringScanner fin>"
          elsif pos > 0 then
            prev = string[0...pos].inspect
            "#<StringScanner #{pos}/#{string.size} #{prev} @ #{rest.inspect}>"
          else
            "#<StringScanner #{pos}/#{string.size} @ #{rest.inspect}>"
          end
      r.taint if self.string.tainted?
      r
    else
      "#<StringScanner (uninitialized)>"
    end
  end

  def match? pattern
    _scan pattern, false, false, true
  end

  def matched
    match.to_s if matched?
  end

  def matched?
    not match.nil?
  end

  def matched_size
    match.to_s.size if matched?
  end

  def matchedsize
    warn "StringScanner#matchedsize is obsolete; use #matched_size instead"
    matched_size
  end

  def post_match
    match.post_match if matched?
  end

  def pre_match
    string[0, match.begin(0)] if matched?
  end

  def reset
    @prev_pos = self.pos = 0
    @match = nil
    self
  end

  def rest
    string[pos..-1]
  end

  def rest?
    return !eos?
  end

  def rest_size
    self.rest.size
  end

  def restsize
    warn "StringScanner#restsize is obsolete; use #rest_size instead"
    rest_size
  end

  def scan pattern
    _scan pattern, true, true, true
  end

  def scan_until pattern
    _scan pattern, true, true, false
  end

  def scan_full pattern, succptr, getstr
    _scan pattern, succptr, getstr, true
  end

  def search_full pattern, succptr, getstr
    _scan pattern, succptr, getstr, false
  end

  def self.must_C_version
    self
  end

  def skip pattern
    _scan pattern, true, false, true
  end

  def skip_until pattern
    _scan pattern, true, false, false
  end

  def string= s
    reset
    @string = s
  end

  def terminate
    @match = nil
    self.pos = string.size
    self
  end

  def unscan
#     raise ScanError if @match.nil?
#     self.pos = @prev_pos
#     @prev_pos = nil
#     @match = nil
#     self
  end

  def peek len
    raise ArgumentError if len < 0
    return "" if len.zero?
    return string[pos, len]
  end

  def peep len
    warn "StringScanner#peep is obsolete; use #peek instead"
    peek len
  end

  def _scan pattern, succptr, getstr, headonly
#     _lame_guard

#     @match = nil

#     return nil if (string.size - pos) < 0 # TODO: make more elegant

#     @match = if headonly then
#                pattern.match_start(string, pos)
#              else
#                pattern.search_region(string, pos, string.size, true)
#              end

#     return nil if match.nil?

#     m = string[pos...match.end(0)]

#     if succptr then
#       @prev_pos = pos
#       self.pos += m.size
#     end

#     if getstr then
#       m
#     else
#       m.size
#     end
  end
  private :_scan

  def _lame_guard
    raise ArgumentError unless instance_variable_defined?(:@string)
  end
  private :_lame_guard
end

{'fraction_non_alphanumeric': 0.08541517963359505, 'fraction_numerical': 0.0046395431834403995, 'mean_word_length': 2.256003098373354, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5472774', 'n_tokens_mistral': 2918, 'n_tokens_neox': 2796, 'n_words': 910}