<commit_before>#shotgun app.rb -p 9294

require 'sinatra'
require 'json'
require 'newrelic_rpm'
require 'nokogiri'
require 'digest/sha1'
require 'securerandom'

@@random = Random.new

enable :logging

configure do
  set :server, :puma
  set :public_folder, 'tmp'
  enable :static
end

def all_methods(path, opts = {}, &block)
  get(path, opts, &block)
  post(path, opts, &block)
  put(path, opts, &block)
  delete(path, opts, &block)
  patch(path, opts, &block)
  options(path, opts, &block)
  head(path, opts, &block)
end

def get_headers
  env.select {|k, v| k.start_with? 'HTTP_'}
end

def echo_response
  body_hasher = Digest::SHA1.new
  body = ""

  request.body.each do |chunk|
    # Exclude obviously non-printable character(s).
    body << chunk if /[\x00-\x1F]/ !~ chunk
    body_hasher << chunk
  end

  response_args = {
    method: request.request_method,
    path: request.path,
    args: request.query_string,
    body: body,
    headers: get_headers(),
    uuid: SecureRandom.uuid
  }
  response_args.merge!(
    bodySha1: body_hasher.base64digest(),
    bodyLength: body.length
  ) if body.length > 0

  # Prefer JSON if possible, including cases of unrecognised/erroneous types.
  if request.accept?('application/xml') && !request.accept?('application/json')
    content_type 'application/xml'
    build_xml_response(response_args)
  else
    content_type 'application/json'
    JSON.pretty_generate(response_args)
  end
end

def build_xml_response(method:, path:, uuid:, body:, bodySha1: nil,
  bodyLength:0, headers:, args: nil)

  builder = Nokogiri::XML::Builder.new do |xml|
    xml.echoResponse {
      xml.method_ method
      xml.path path
      xml.uuid uuid
      xml.bodySha1 bodySha1 if bodySha1
      xml.bodyLength bodyLength if bodyLength > 0
      xml.body body if bodyLength > 0
      xml.headers { |headers_|
        headers.each_pair do |key, value|
          headers_.header { |header|
            header.key key.split('HTTP_')[1]
            header.value value
          }
        end
      }
      xml.args { |args|
        request.env['rack.request.query_hash'].each_pair do |key, value|
          args.arg { |arg|
            arg.key key
            arg.value value
          }
        end
      }
    }
  end
  builder.to_xml
end

def random_file(name, size)
  path = Pathname.new('tmp').join('size', name)

  return path if path.exist?

  content = @@random.bytes(size)
  path.dirname.mkpath

  path.open('w') do |f|
    f << content
  end

  path
end

get '/size/:size' do |original_size|
  size, unit = original_size.scan(/\d+|\D+/)

  size = Integer(size)
  unit = String(unit).downcase

  multiplier = case unit
                 when 'kb' then 1024
                 when 'mb' then 1024*1024
                 else 1
               end

  send_file random_file(original_size,  * multiplier)
end

get '/status/:code' do |code|
  [code.to_i, echo_response]
end

get '/wait/:seconds' do |seconds|
  duration = Float(seconds)

  Kernel.sleep(duration)
  body "slept #{duration} seconds"
end

get '/favicon.ico' do # Avoid bumping counter on favicon
end

all_methods "/**" do
  echo_response
end
<commit_msg>B64 encode files that appear to be binary.<commit_after>#shotgun app.rb -p 9294

require 'sinatra'
require 'json'
require 'newrelic_rpm'
require 'nokogiri'
require 'digest/sha1'
require 'securerandom'
require 'base64'

@@random = Random.new

enable :logging

configure do
  set :server, :puma
  set :public_folder, 'tmp'
  enable :static
end

def all_methods(path, opts = {}, &block)
  get(path, opts, &block)
  post(path, opts, &block)
  put(path, opts, &block)
  delete(path, opts, &block)
  patch(path, opts, &block)
  options(path, opts, &block)
  head(path, opts, &block)
end

def get_headers
  env.select {|k, v| k.start_with? 'HTTP_'}
end

def echo_response
  body = request.body.read
  hash = Digest::SHA1.base64digest(body)
  # B64 encode if contains obviously non-printable character(s).
  if request.media_type == 'application/octet-stream' ||
      request.media_type == 'multipart/form-data' || body =~ /[^[:print:]]/
    body = Base64.encode64(body)
  end

  response_args = {
    method: request.request_method,
    path: request.path,
    args: request.query_string,
    body: body,
    headers: get_headers(),
    uuid: SecureRandom.uuid
  }
  response_args.merge!(
    bodySha1: hash,
    bodyLength: body.length
  ) if body.length > 0

  # Prefer JSON if possible, including cases of unrecognised/erroneous types.
  if request.accept?('application/xml') && !request.accept?('application/json')
    content_type 'application/xml'
    build_xml_response(response_args)
  else
    content_type 'application/json'
    JSON.pretty_generate(response_args)
  end
end

def build_xml_response(method:, path:, uuid:, body:, bodySha1: nil,
  bodyLength:0, headers:, args: nil)

  builder = Nokogiri::XML::Builder.new do |xml|
    xml.echoResponse {
      xml.method_ method
      xml.path path
      xml.uuid uuid
      xml.bodySha1 bodySha1 if bodySha1
      xml.bodyLength bodyLength if bodyLength > 0
      xml.body body if bodyLength > 0
      xml.headers { |headers_|
        headers.each_pair do |key, value|
          headers_.header { |header|
            header.key key.split('HTTP_')[1]
            header.value value
          }
        end
      }
      xml.args { |args|
        request.env['rack.request.query_hash'].each_pair do |key, value|
          args.arg { |arg|
            arg.key key
            arg.value value
          }
        end
      }
    }
  end
  builder.to_xml
end

def random_file(name, size)
  path = Pathname.new('tmp').join('size', name)

  return path if path.exist?

  content = @@random.bytes(size)
  path.dirname.mkpath

  path.open('w') do |f|
    f << content
  end

  path
end

get '/size/:size' do |original_size|
  size, unit = original_size.scan(/\d+|\D+/)

  size = Integer(size)
  unit = String(unit).downcase

  multiplier = case unit
                 when 'kb' then 1024
                 when 'mb' then 1024*1024
                 else 1
               end

  send_file random_file(original_size,  * multiplier)
end

get '/status/:code' do |code|
  [code.to_i, echo_response]
end

get '/wait/:seconds' do |seconds|
  duration = Float(seconds)

  Kernel.sleep(duration)
  body "slept #{duration} seconds"
end

get '/favicon.ico' do # Avoid bumping counter on favicon
end

all_methods "/**" do
  echo_response
end

{'fraction_non_alphanumeric': 0.11836162591496652, 'fraction_numerical': 0.01168042361002959, 'mean_word_length': 2.836320191158901, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 30}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4570283', 'n_tokens_mistral': 2247, 'n_tokens_neox': 2168, 'n_words': 560}