<commit_before>require_relative "generic_detector"

module Selinimum
  module Detectors
    # stuff we ignore that should never affect a build
    # TODO: config file, maybe .gitignore style?
    class WhitelistDetector < GenericDetector
      def can_process?(file)
        return false if file =~ %r{\Aspec/fixtures/}
        return true if file =~ %r{\.(txt|md|png|jpg|gif|ico|svg|html|yml)\z}
        return true if file =~ %r{\A(spec/coffeescripts|doc|guard|bin|script|gems/rubocop-canvas\z)/}
        return true if file == "spec/spec.opts"
        return true if file == "Gemfile.d/test.rb" # TODO: this is :totes: temporary, just until https://gerrit.instructure.com/#/c/58088/ lands and we can remove testbot from canvas-lms proper
        false
      end
    end
  end
end<commit_msg>spec: add _core_en.js and selinimum files to whitelist<commit_after>require_relative "generic_detector"

module Selinimum
  module Detectors
    # stuff we ignore that should never affect a build
    # TODO: config file, maybe .gitignore style?
    class WhitelistDetector < GenericDetector
      def can_process?(file)
        return false if file =~ %r{\Aspec/fixtures/}
        return true if file =~ %r{\.(txt|md|png|jpg|gif|ico|svg|html|yml)\z}
        return true if file =~ %r{\A(spec/coffeescripts|doc|guard|bin|script|gems/rubocop-canvas\z)/}
        return true if file =~ %r{\Agems/selinimum/}
        return true if file == "spec/spec.opts"
        return true if file == "public/javascripts/translations/_core_en.js"
        return true if file == "Gemfile.d/test.rb" # TODO: this is :totes: temporary, just until https://gerrit.instructure.com/#/c/58088/ lands and we can remove testbot from canvas-lms proper
        false
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.11028993746446845, 'fraction_numerical': 0.005685048322910745, 'mean_word_length': 3.170616113744076, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4228342', 'n_tokens_mistral': 577, 'n_tokens_neox': 563, 'n_words': 162}