<commit_before>require_relative '../scope'

module Tritium::Engines::Reference::Scope
  class Text < Base
    attr :text

    # @private
    def initialize(text, root = nil, parent = nil)
      @text = text
      @root = root || self
      super
    end

    def set(text)
      @text = text
    end

    def replace(matcher, replacement = "", &block)
      if matcher.is_a? String
        matcher = Regexp.new(matcher)
      end

      @text.gsub!(matcher) do |match|
        $~.captures.each_with_index do |arg, index|
          var("#{index + 1}", arg)
        end
        replace = replacement
        (replace = open_text_scope_with(replace, &block)) if block
        replace.gsub(/([\1\2\3\4])/) do |var_match|
          index = $1.getbyte(0).to_s
          var(index)
        end
      end 
    end
  
    def remove(matcher = nil)<commit_msg>Ok, much better and more compatible way to do the advanced replace. It now works with \\1 through \\9 in the replacer.<commit_after>require_relative '../scope'

module Tritium::Engines::Reference::Scope
  class Text < Base
    attr :text

    # @private
    def initialize(text, root = nil, parent = nil)
      @text = text
      @root = root || self
      super
    end

    def set(text)
      @text = text
    end

    def replace(matcher, replacement = "", &block)
      if matcher.is_a? String
        matcher = Regexp.new(matcher)
      end

      @text.gsub!(matcher) do |match|
        $~.captures.each_with_index do |arg, index|
          var("#{index + 1}", arg)
        end
        replace = replacement
        (replace = open_text_scope_with(replace, &block)) if block
        replace.gsub(/\\([\d])/) do |var_match|
          var($1)
        end
      end 
    end
  
    def remove(matcher = nil)
{'fraction_non_alphanumeric': 0.1198182850653038, 'fraction_numerical': 0.006246450880181715, 'mean_word_length': 2.0696864111498257, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2350366', 'n_tokens_mistral': 571, 'n_tokens_neox': 555, 'n_words': 154}