<commit_before>require "rack_encoding_scrubber/version"

class RackEncodingScrubber
  def initialize(app)
    @app = app
  end

  def call(env)
    encode env
    @app.call(env)
  end

  def encode(env)
    request_method = env['REQUEST_METHOD']
    if request_method == 'GET'
      %w[QUERY_STRING REQUEST_PATH PATH_INFO QUERY_STRING REQUEST_URI ORIGINAL_FULLPATH].each do |header|
        if env[header] and env[header]["%u00"]
          env[header].gsub!(/%u00\h{0,2}/, "")
        end
      end
    end
  end
end<commit_msg>latin-1 %E4 auch raus<commit_after>require "rack_encoding_scrubber/version"

class RackEncodingScrubber
  def initialize(app)
    @app = app
  end

  def call(env)
    encode env
    @app.call(env)
  end

  def encode(env)
    request_method = env['REQUEST_METHOD']
    if request_method == 'GET'
      %w[QUERY_STRING REQUEST_PATH PATH_INFO QUERY_STRING REQUEST_URI ORIGINAL_FULLPATH].each do |header|
        if env[header] and env[header]["%u00"]
          env[header].gsub!(/%u00\h{0,2}/, "")
          env[header].gsub!(/%\h{0,2}/,"")
        end
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.12104787714543812, 'fraction_numerical': 0.014453477868112014, 'mean_word_length': 3.0437956204379564, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5874221', 'n_tokens_mistral': 419, 'n_tokens_neox': 402, 'n_words': 83}