<commit_before>module Deployinator
  module Helpers
    RUN_LOG_PATH = "run_logs/"

    # list of URLs that don't require authorization (can take regular expressions)
    # this should be in a config
    AUTH_FREE_URLS = []
    AUTH_FREE_URLS << %w[/last_pushes /last_chef /favicon.ico /web/versions /api/versions]
    AUTH_FREE_URLS << %w[/log.txt /month_stats_lag /web/lb_status.json]
    AUTH_FREE_URLS << %w[/deploys/\w+/\w+/\d+/\d+]

    OK_PATH_REGEX = %r{^(#{AUTH_FREE_URLS.compact.join("|")})$}

    def run_log_path
      RUN_LOG_PATH
    end

    def set_block(&block)
      @block = block
    end

    def init(env)
      @username = (env["HTTP_X_USERNAME"] || ENV["HTTP_X_USERNAME"]) or raise "Must be logged in"
      @groups   = CGI.unescape(env["HTTP_X_GROUPS"] || ENV["HTTP_X_GROUPS"]).split("|")
      @host     = env["HTTP_HOST"]
      @local    = @host.match(/local|dev/)
      @ny4      = @host.match(/ny4/)
      if @username == "nobody" && ! request.path_info.match(OK_PATH_REGEX)
        raise "Must be logged in"
      end
      @stack = form_hash(env, "stack") unless @stack
      @filename = "#{Time.now.to_i}-#{@username}-#{dep_method(env)}.html"
    end

    def dep_method(env)
      return "undefined" unless form_hash(env, "method")
      "#{@stack}-#{form_hash(env, "method")}"
    end

    def form_hash(env, key)
      fh = env["rack.request.form_hash"]
      fh && fh.key?(key) && fh[key]
    end

    def write_file(str, file)
      File.open("#{RUN_LOG_PATH}#{file}", "a") do |f|
        # escape the script tags so we dont get redirects when looking at logs
        str = str.gsub(%r{(<script>.*?</script>)}) {|script| CGI.escapeHTML(script)}
        f.print str
      end
    end

    # output to a file, and the streaming output handler
    def log_and_stream(output)
      write_file output, @filename if @filename
      @block.call(output)
    end

    # Run external command with timing information
    # streams and logs the output of the command as well
    # does not (currently) check exit status codes
    def run_cmd(cmd)
      ret = ""
      start = Time.now.to_i
      log_and_stream "<div class='command'><h4>Running #{cmd}</h4><p class='output'>"
      time = Benchmark.measure do
        Open3.popen3(cmd) do |inn, out, err|
          output = ""
          until out.eof?
            # raise "Timeout" if output.empty? && Time.now.to_i - start > 300
            chr = out.read(1)
            output << chr
            ret << chr
            if chr == "\n" || chr == "\r"
              log_and_stream output + "<br>"
              output = ""
            end
          end
          log_and_stream(output) unless output.empty?
          log_and_stream("<span class='stderr'>STDERR: #{err.read}</span><br>") unless err.eof?
        end
      end
      log_and_stream "</p>"
      log_and_stream "<h5>Time: #{time}</h5></div>"
      return ret
    end

    def nicify_env(env)
      env = "production" if env == "PROD"
      env.downcase
    end

    def log_and_shout(options={})
      options[:stack] ||= @stack

      raise "Must have stack" unless options[:stack]

      # newlines in any of these will corrupt the logfile, so strip them out
      newoptions = {}
      options.each do |key, value|
        if value.is_a? String
          value = value.chomp
        end
        newoptions[key] = value
      end
      options = newoptions

      options[:env] ||= "PROD"
      options[:nice_env] = nicify_env(options[:env])
      options[:user] ||= @username
      options[:start] = @start_time unless options[:start] || ! @start_time

      if (options[:start])
        options[:end] = Time.now unless options.key?(:end)
        options[:duration] = options[:end] - options[:start]

        log_and_stream "Ended at #{options[:end]}<br>Took: #{options[:duration]} seconds<br>"
        timing_log options[:duration], options[:nice_env], options[:stack]<commit_msg>chomp option values in a more Ruby way<commit_after>module Deployinator
  module Helpers
    RUN_LOG_PATH = "run_logs/"

    # list of URLs that don't require authorization (can take regular expressions)
    # this should be in a config
    AUTH_FREE_URLS = []
    AUTH_FREE_URLS << %w[/last_pushes /last_chef /favicon.ico /web/versions /api/versions]
    AUTH_FREE_URLS << %w[/log.txt /month_stats_lag /web/lb_status.json]
    AUTH_FREE_URLS << %w[/deploys/\w+/\w+/\d+/\d+]

    OK_PATH_REGEX = %r{^(#{AUTH_FREE_URLS.compact.join("|")})$}

    def run_log_path
      RUN_LOG_PATH
    end

    def set_block(&block)
      @block = block
    end

    def init(env)
      @username = (env["HTTP_X_USERNAME"] || ENV["HTTP_X_USERNAME"]) or raise "Must be logged in"
      @groups   = CGI.unescape(env["HTTP_X_GROUPS"] || ENV["HTTP_X_GROUPS"]).split("|")
      @host     = env["HTTP_HOST"]
      @local    = @host.match(/local|dev/)
      @ny4      = @host.match(/ny4/)
      if @username == "nobody" && ! request.path_info.match(OK_PATH_REGEX)
        raise "Must be logged in"
      end
      @stack = form_hash(env, "stack") unless @stack
      @filename = "#{Time.now.to_i}-#{@username}-#{dep_method(env)}.html"
    end

    def dep_method(env)
      return "undefined" unless form_hash(env, "method")
      "#{@stack}-#{form_hash(env, "method")}"
    end

    def form_hash(env, key)
      fh = env["rack.request.form_hash"]
      fh && fh.key?(key) && fh[key]
    end

    def write_file(str, file)
      File.open("#{RUN_LOG_PATH}#{file}", "a") do |f|
        # escape the script tags so we dont get redirects when looking at logs
        str = str.gsub(%r{(<script>.*?</script>)}) {|script| CGI.escapeHTML(script)}
        f.print str
      end
    end

    # output to a file, and the streaming output handler
    def log_and_stream(output)
      write_file output, @filename if @filename
      @block.call(output)
    end

    # Run external command with timing information
    # streams and logs the output of the command as well
    # does not (currently) check exit status codes
    def run_cmd(cmd)
      ret = ""
      start = Time.now.to_i
      log_and_stream "<div class='command'><h4>Running #{cmd}</h4><p class='output'>"
      time = Benchmark.measure do
        Open3.popen3(cmd) do |inn, out, err|
          output = ""
          until out.eof?
            # raise "Timeout" if output.empty? && Time.now.to_i - start > 300
            chr = out.read(1)
            output << chr
            ret << chr
            if chr == "\n" || chr == "\r"
              log_and_stream output + "<br>"
              output = ""
            end
          end
          log_and_stream(output) unless output.empty?
          log_and_stream("<span class='stderr'>STDERR: #{err.read}</span><br>") unless err.eof?
        end
      end
      log_and_stream "</p>"
      log_and_stream "<h5>Time: #{time}</h5></div>"
      return ret
    end

    def nicify_env(env)
      env = "production" if env == "PROD"
      env.downcase
    end

    def log_and_shout(options={})
      options[:stack] ||= @stack

      raise "Must have stack" unless options[:stack]

      # newlines in any of these will corrupt the logfile, so strip them out
      newoptions = {}
      options.each do |key, value|
        value.chomp! if value.respond_to?(:"chomp!")
        newoptions[key] = value
      end
      options = newoptions

      options[:env] ||= "PROD"
      options[:nice_env] = nicify_env(options[:env])
      options[:user] ||= @username
      options[:start] = @start_time unless options[:start] || ! @start_time

      if (options[:start])
        options[:end] = Time.now unless options.key?(:end)
        options[:duration] = options[:end] - options[:start]

        log_and_stream "Ended at #{options[:end]}<br>Took: #{options[:duration]} seconds<br>"
        timing_log options[:duration], options[:nice_env], options[:stack]
{'fraction_non_alphanumeric': 0.1412184552638287, 'fraction_numerical': 0.0030588835075197555, 'mean_word_length': 2.503125, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4364542', 'n_tokens_mistral': 2610, 'n_tokens_neox': 2560, 'n_words': 631}