<commit_before>require 'json'
require 'fog'
require 'fileutils'
require './lib/server-commands'
require './lib/server-files'
require 'rack-ssl-enforcer'
include ServerFiles
include ServerCommands

tmp_file = "tmp/last_request.txt"
tmp_results = "tmp/results.txt"
local_repos = ENV['LOCAL_REPOS'] || "/opt/bitnami/apps/projects/"

  def upload_files(results_location)
    artifact_files = Dir.glob("./artifacts/*")
    puts "files uploading: #{artifact_files.inspect}"
    if artifact_files.length > 0
      write_file(results_location+'_artifact_files',artifact_files.map{|f| 'https://s3.amazonaws.com/deferred-server/'+results_location+'_artifact_files_'+f.to_s.gsub('/','_')}.to_json)
      artifact_files.each do |file|
        write_file(results_location+'_artifact_files_'+file.to_s.gsub('/','_'), File.read(file))
      end
    end
  end

  def reset_artifacts_directory
    FileUtils.rm_rf('./artifacts', :secure => false)
    Dir.mkdir('./artifacts') unless File.exists?('./artifacts')
    artifact_files = Dir.glob("./artifacts/*")
    puts "files after clearing: #{artifact_files.inspect}"
  end

# Run me with 'ruby' and I run as a script
if $0 =~ /#{File.basename(__FILE__)}$/
  puts "running as local script"

  #upload_files('batman')
  stop_server

  puts "done"
else
  use Rack::SslEnforcer unless ENV['RACK_ENV']=='test'
  set :public_folder, File.dirname(__FILE__) + '/public'
  set :root, File.dirname(__FILE__)

  helpers do
    def protected!
      unless authorized?
        response['WWW-Authenticate'] = %(Basic realm="Testing HTTP Auth")
        throw(:halt, [401, "Not authorized\n"])
      end
    end

    def authorized?
      @auth ||=  Rack::Auth::Basic::Request.new(request.env)
      @auth.provided? && @auth.basic? && @auth.credentials && @auth.credentials == ['admin', 'responder']
    end
  end

  before { protected! if request.path_info == "/" }

  get '/' do
    if File.exists?(tmp_results)
      @results = File.read(tmp_results)
      @results.gsub(/\"api_token\":\"\w\"/,'"api_token":"*******"')
    end
    @last_push = File.read(tmp_file) if File.exists?(tmp_file)
    erb :index
  end

  get '/last_job' do
    last_job_time = if File.exists?(tmp_file)
      File.mtime(tmp_file)
    else
      Time.now
    end
    {:last_time => last_job_time}.to_json
  end

  def github_hook_commit(push)
    local_repos = ENV['LOCAL_REPOS'] || "/opt/bitnami/apps/projects/"
    repo_url = push['repository']['url'] rescue nil
    repo_name = push['repository']['name'] rescue nil
    user = push['repository']['owner']['name'] rescue nil
    after_commit = push['after']
    project_key  = "#{user}/#{repo_name}"
    commit_key   = "#{project_key}/#{after_commit}"<commit_msg>fix on token stripping<commit_after>require 'json'
require 'fog'
require 'fileutils'
require './lib/server-commands'
require './lib/server-files'
require 'rack-ssl-enforcer'
include ServerFiles
include ServerCommands

tmp_file = "tmp/last_request.txt"
tmp_results = "tmp/results.txt"
local_repos = ENV['LOCAL_REPOS'] || "/opt/bitnami/apps/projects/"

  def upload_files(results_location)
    artifact_files = Dir.glob("./artifacts/*")
    puts "files uploading: #{artifact_files.inspect}"
    if artifact_files.length > 0
      write_file(results_location+'_artifact_files',artifact_files.map{|f| 'https://s3.amazonaws.com/deferred-server/'+results_location+'_artifact_files_'+f.to_s.gsub('/','_')}.to_json)
      artifact_files.each do |file|
        write_file(results_location+'_artifact_files_'+file.to_s.gsub('/','_'), File.read(file))
      end
    end
  end

  def reset_artifacts_directory
    FileUtils.rm_rf('./artifacts', :secure => false)
    Dir.mkdir('./artifacts') unless File.exists?('./artifacts')
    artifact_files = Dir.glob("./artifacts/*")
    puts "files after clearing: #{artifact_files.inspect}"
  end

# Run me with 'ruby' and I run as a script
if $0 =~ /#{File.basename(__FILE__)}$/
  puts "running as local script"

  #upload_files('batman')
  stop_server

  puts "done"
else
  use Rack::SslEnforcer unless ENV['RACK_ENV']=='test'
  set :public_folder, File.dirname(__FILE__) + '/public'
  set :root, File.dirname(__FILE__)

  helpers do
    def protected!
      unless authorized?
        response['WWW-Authenticate'] = %(Basic realm="Testing HTTP Auth")
        throw(:halt, [401, "Not authorized\n"])
      end
    end

    def authorized?
      @auth ||=  Rack::Auth::Basic::Request.new(request.env)
      @auth.provided? && @auth.basic? && @auth.credentials && @auth.credentials == ['admin', 'responder']
    end
  end

  before { protected! if request.path_info == "/" }

  get '/' do
    if File.exists?(tmp_results)
      @results = File.read(tmp_results)
      @results.gsub(/api_token.*:\"\w+\",/,'api_token":"*******",')
    end
    @last_push = File.read(tmp_file) if File.exists?(tmp_file)
    erb :index
  end

  get '/last_job' do
    last_job_time = if File.exists?(tmp_file)
      File.mtime(tmp_file)
    else
      Time.now
    end
    {:last_time => last_job_time}.to_json
  end

  def github_hook_commit(push)
    local_repos = ENV['LOCAL_REPOS'] || "/opt/bitnami/apps/projects/"
    repo_url = push['repository']['url'] rescue nil
    repo_name = push['repository']['name'] rescue nil
    user = push['repository']['owner']['name'] rescue nil
    after_commit = push['after']
    project_key  = "#{user}/#{repo_name}"
    commit_key   = "#{project_key}/#{after_commit}"
{'fraction_non_alphanumeric': 0.1609872904770676, 'fraction_numerical': 0.0022103518143304474, 'mean_word_length': 3.8352626892252895, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '70704', 'n_tokens_mistral': 1876, 'n_tokens_neox': 1834, 'n_words': 358}