<commit_before>#!/usr/bin/env ruby
require 'net/http'
require 'net/https'
require 'cgi'
require 'rubygems'
require 'json'
require 'yaml'
require 'nakamura/users'
require 'logger'

# Convert time to nanoseconds
class Time
  def to_nsec
    self.to_f.to_s.gsub(".", "").to_i
  end
end

class String
  def base64_decode
    unpack('m').first
  end
  
  def base64_encode
    [self].pack('m').chop
  end
end


class Hash
  
  def dump
    return keys.collect{|k| "#{k} => #{self[k]}"}.join(", ")
  end
  
end

## Fix array handling
module Net::HTTPHeader
  def set_form_data(params, sep = '&')
    self.body = params.map {|k, v| encode_kvpair(k, v) }.flatten.join(sep)
    self.content_type = 'application/x-www-form-urlencoded'
  end


  def initialize_http_header(initheader)
      @header = { "Referer" => ["/dev/integrationtests"] }
      return unless initheader
      initheader.each do |key, value|
        warn "net/http: warning: duplicated HTTP header: #{key}" if key?(key) and $VERBOSE
        @header[key.downcase] = [value.strip]
      end
  end
  
  def encode_kvpair(k, vs)
    if vs.nil? or vs == '' then
      "#{CGI::escape(k)}="
    else
      Array(vs).map {|v| "#{CGI::escape(k)}=#{CGI::escape(v.to_s)}" }
    end
  end
end

class WrappedCurlResponse
  
  def initialize(response)
    @response = response
  end
  
  def code
    return @response.response_code
  end
  
  def message
    return @response.response_code
  end
  
  def body
    return @response.body_str
  end
  
end

module SlingInterface
  
  class Sling
    
    attr_accessor :log, :trustedauth, :loggedin
    
    def initialize(server="http://localhost:8080/", trustedauth=false)
      @server = server
      if not @server.end_with? '/'
        @server += '/'
      end
      @serveruri = URI.parse(server)
      @user = SlingUsers::User.admin_user()
      @trustedauth = trustedauth
      @cookies = Hash.new()
      @loggedin = false
      @log = Logger.new(STDOUT)
      @log.level = Logger::WARN
    end
    
    def dump_response(response)
      @log.info "Response: #{response.code} #{response.message}"
      @log.debug "#{response.body}"
    end
    
    def switch_user(user)
      @log.info "Switched user to #{user}"
      @cookies = Hash.new()
      @user = user
      if ( @trustedauth ) 
         @loggedin = false
      end
    end
    
    
    def text_to_multipart(key,value)
      return "Content-Disposition: form-data; name=\"#{CGI::escape(key)}\"\r\n" + 
             "\r\n" + 
             "#{value}\r\n"
    end
    
    def file_to_multipart(key,filename,mime_type,content)
      if ( filename != nil ) 
        return "Content-Disposition: form-data; name=\"*\"; filename=\"#{filename}\"\r\n" +
             "Content-Transfer-Encoding: binary\r\n" +
             "Content-Type: #{mime_type}\r\n" + 
             "\r\n" + 
             "#{content}\r\n"
      else 
        return "Content-Disposition: form-data; name=\"jcr:content\"\r\n" +
             "Content-Transfer-Encoding: binary\r\n" +
             "Content-Type: #{mime_type}\r\n" + 
             "\r\n" + 
             "#{content}\r\n"
      end

    end
    
    def execute_file_post(path, fieldname, filename, data, content_type)
      uri = URI.parse(path)
      fileTypeHint = "Content-Disposition: form-data; name=\"*@TypeHint\"\r\n\r\n" +
                     "nt:file\r\n"

      params = [fileTypeHint,file_to_multipart(fieldname, filename, content_type, data)]
      boundary = '349832898984244898448024464570528145'
      post_body = params.collect {|p| '--' + boundary + "\r\n" + p}.join('') + "--" + boundary + "--\r\n"
      req = Net::HTTP::Post.new(uri.path)
      req.set_content_type("multipart/form-data", {"boundary" => boundary})
      req.body = post_body
      res = sendRequest(uri, req)
      dump_response(res)
      return res
    end

    def createHttp(uri)
      http = Net::HTTP.new(uri.host, uri.port)
      if (uri.scheme == 'https')
         http.use_ssl = true
         http.verify_mode = OpenSSL::SSL::VERIFY_NONE
      end
      return http
    end

    def sendRequest(uri, req)
      # Not all browsers take port number into account when setting cookies.
      isSlingReq = uri.host && (uri.host == @serveruri.host)
      if (isSlingReq)
        if ( @trustedauth )
          if ( ! @loggedin )
              do_login()
          end
        else
          @user.do_request_auth(req)
        end
        set_cookies(req)
      else
        @log.info("#{uri.host} does not match expected #{@serverui.host}. Not sending state.")
      end
      res = createHttp(uri).start { |http| http.request(req ) }
      save_cookies(res) if (isSlingReq)
      return res
    end
    
    def delete_file(path) 
      uri = URI.parse(path)
      req = Net::HTTP::Delete.new(uri.path)
      res = sendRequest(uri, req)
      dump_response(res)
      return res
    end
    
    def execute_put_file(path, data)
      @log.debug "URL: #{path}"
      @log.debug("PUTFILE: #{path} (as '#{@user.name}')")
      uri = URI.parse(path)
      req = Net::HTTP::Put.new(uri.path)
      req.body = data
      res = sendRequest(uri, req)
      dump_response(res)
      return res<commit_msg>Fix variable name typo.<commit_after>#!/usr/bin/env ruby
require 'net/http'
require 'net/https'
require 'cgi'
require 'rubygems'
require 'json'
require 'yaml'
require 'nakamura/users'
require 'logger'

# Convert time to nanoseconds
class Time
  def to_nsec
    self.to_f.to_s.gsub(".", "").to_i
  end
end

class String
  def base64_decode
    unpack('m').first
  end
  
  def base64_encode
    [self].pack('m').chop
  end
end


class Hash
  
  def dump
    return keys.collect{|k| "#{k} => #{self[k]}"}.join(", ")
  end
  
end

## Fix array handling
module Net::HTTPHeader
  def set_form_data(params, sep = '&')
    self.body = params.map {|k, v| encode_kvpair(k, v) }.flatten.join(sep)
    self.content_type = 'application/x-www-form-urlencoded'
  end


  def initialize_http_header(initheader)
      @header = { "Referer" => ["/dev/integrationtests"] }
      return unless initheader
      initheader.each do |key, value|
        warn "net/http: warning: duplicated HTTP header: #{key}" if key?(key) and $VERBOSE
        @header[key.downcase] = [value.strip]
      end
  end
  
  def encode_kvpair(k, vs)
    if vs.nil? or vs == '' then
      "#{CGI::escape(k)}="
    else
      Array(vs).map {|v| "#{CGI::escape(k)}=#{CGI::escape(v.to_s)}" }
    end
  end
end

class WrappedCurlResponse
  
  def initialize(response)
    @response = response
  end
  
  def code
    return @response.response_code
  end
  
  def message
    return @response.response_code
  end
  
  def body
    return @response.body_str
  end
  
end

module SlingInterface
  
  class Sling
    
    attr_accessor :log, :trustedauth, :loggedin
    
    def initialize(server="http://localhost:8080/", trustedauth=false)
      @server = server
      if not @server.end_with? '/'
        @server += '/'
      end
      @serveruri = URI.parse(server)
      @user = SlingUsers::User.admin_user()
      @trustedauth = trustedauth
      @cookies = Hash.new()
      @loggedin = false
      @log = Logger.new(STDOUT)
      @log.level = Logger::WARN
    end
    
    def dump_response(response)
      @log.info "Response: #{response.code} #{response.message}"
      @log.debug "#{response.body}"
    end
    
    def switch_user(user)
      @log.info "Switched user to #{user}"
      @cookies = Hash.new()
      @user = user
      if ( @trustedauth ) 
         @loggedin = false
      end
    end
    
    
    def text_to_multipart(key,value)
      return "Content-Disposition: form-data; name=\"#{CGI::escape(key)}\"\r\n" + 
             "\r\n" + 
             "#{value}\r\n"
    end
    
    def file_to_multipart(key,filename,mime_type,content)
      if ( filename != nil ) 
        return "Content-Disposition: form-data; name=\"*\"; filename=\"#{filename}\"\r\n" +
             "Content-Transfer-Encoding: binary\r\n" +
             "Content-Type: #{mime_type}\r\n" + 
             "\r\n" + 
             "#{content}\r\n"
      else 
        return "Content-Disposition: form-data; name=\"jcr:content\"\r\n" +
             "Content-Transfer-Encoding: binary\r\n" +
             "Content-Type: #{mime_type}\r\n" + 
             "\r\n" + 
             "#{content}\r\n"
      end

    end
    
    def execute_file_post(path, fieldname, filename, data, content_type)
      uri = URI.parse(path)
      fileTypeHint = "Content-Disposition: form-data; name=\"*@TypeHint\"\r\n\r\n" +
                     "nt:file\r\n"

      params = [fileTypeHint,file_to_multipart(fieldname, filename, content_type, data)]
      boundary = '349832898984244898448024464570528145'
      post_body = params.collect {|p| '--' + boundary + "\r\n" + p}.join('') + "--" + boundary + "--\r\n"
      req = Net::HTTP::Post.new(uri.path)
      req.set_content_type("multipart/form-data", {"boundary" => boundary})
      req.body = post_body
      res = sendRequest(uri, req)
      dump_response(res)
      return res
    end

    def createHttp(uri)
      http = Net::HTTP.new(uri.host, uri.port)
      if (uri.scheme == 'https')
         http.use_ssl = true
         http.verify_mode = OpenSSL::SSL::VERIFY_NONE
      end
      return http
    end

    def sendRequest(uri, req)
      # Not all browsers take port number into account when setting cookies.
      isSlingReq = uri.host && (uri.host == @serveruri.host)
      if (isSlingReq)
        if ( @trustedauth )
          if ( ! @loggedin )
              do_login()
          end
        else
          @user.do_request_auth(req)
        end
        set_cookies(req)
      else
        @log.info("#{uri.host} does not match expected #{@serveruri.host}. Not sending state.")
      end
      res = createHttp(uri).start { |http| http.request(req ) }
      save_cookies(res) if (isSlingReq)
      return res
    end
    
    def delete_file(path) 
      uri = URI.parse(path)
      req = Net::HTTP::Delete.new(uri.path)
      res = sendRequest(uri, req)
      dump_response(res)
      return res
    end
    
    def execute_put_file(path, data)
      @log.debug "URL: #{path}"
      @log.debug("PUTFILE: #{path} (as '#{@user.name}')")
      uri = URI.parse(path)
      req = Net::HTTP::Put.new(uri.path)
      req.body = data
      res = sendRequest(uri, req)
      dump_response(res)
      return res
{'fraction_non_alphanumeric': 0.14090076188639214, 'fraction_numerical': 0.008486835760439772, 'mean_word_length': 2.272325654780688, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1948624', 'n_tokens_mistral': 3635, 'n_tokens_neox': 3536, 'n_words': 754}