<commit_before>module Marty::Diagnostic; class EnvironmentVariables < Base
  diagnostic_fn do
    env
  end

  def self.env filter=''
    env = ENV.clone

    # obfuscate SECRET_KEY_BASE for comparison
    env['SECRET_KEY_BASE'] = env['SECRET_KEY_BASE'][0,4] if
      env['SECRET_KEY_BASE']

    # remove SCRIPT_URI, SCRIPT_URL as calling node differs
    ['SCRIPT_URI', 'SCRIPT_URL'].each{|k| env.delete(k)}

    to_block = ['PASSWORD', 'DEBUG']
    env.sort.each_with_object({}){|(k,v),h|
      h[k] = v if to_block.all?{|b| !k.include?(b)} && k.include?(filter)}
  end

  # overwritten to only return inconsitent data
  def self.apply_consistency data
    diff = get_difference(data)
    data.each_with_object({}){
      |(node, diagnostic), new_data|
      new_data[node] = diagnostic.each_with_object({}){
        |(test, info), new_diagnostic|
        new_diagnostic[test] = info + {'consistent' => false} if
          diff.include?(test)
      }
    }
  end
end
end
<commit_msg>add configurability to environment variable diag<commit_after>module Marty::Diagnostic; class EnvironmentVariables < Base
  diagnostic_fn do
    env
  end

  def self.env filter=''
    env = ENV.clone

    to_delete = (Marty::Config['DIAG_ENV_BLOCK'] || []).map(&:upcase) + [
      'SCRIPT_URI', 'SCRIPT_URL']

    to_obfus = (Marty::Config['DIAG_ENV_OBFUSCATE'] || []).map(&:upcase) + [
      'SECRET_KEY_BASE']

    to_block = (Marty::Config['DIAG_ENV_BLOCK_IF_INCL'] || []).map(&:upcase) + [
      'PASSWORD', 'DEBUG']

    to_delete.each{|k| env.delete(k)}

    to_obfus.each{|k| env[k] = env[k][0,4] if env[k]}

    env.sort.each_with_object({}){|(k,v),h|
      h[k] = v if to_block.all?{|b| !k.include?(b)} && k.include?(filter)}
  end

  # overwritten to only return inconsitent data
  def self.apply_consistency data
    diff = get_difference(data)
    data.each_with_object({}){
      |(node, diagnostic), new_data|
      new_data[node] = diagnostic.each_with_object({}){
        |(test, info), new_diagnostic|
        new_diagnostic[test] = info + {'consistent' => false} if
          diff.include?(test)
      }
    }
  end
end
end

{'fraction_non_alphanumeric': 0.1550751879699248, 'fraction_numerical': 0.0018796992481203006, 'mean_word_length': 3.4915611814345993, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '406915', 'n_tokens_mistral': 803, 'n_tokens_neox': 775, 'n_words': 149}