<commit_before>module CMSScanner
  # Not really an interesting file, but will use this class for convenience
  class Headers < InterestingFile
    # @return [ Hash ] The headers
    def entries
      res = Browser.get(url)
      return [] unless res
      res.headers
    end

    # @return [ Array<String> ] The interesting headers detected
    def interesting_entries
      results = []

      entries.each do |header, value|
        next if known_headers.include?(header.downcase)

        results << "#{header}: #{value}"
      end
      results
    end

    # @return [ Array<String> ] Downcased known headers
    def known_headers
      %w(
        age accept-ranges cache-control content-type content-length connection date etag expires
        location last-modified pragma set-cookie vary
        x-cache x-content-type-options x-language x-pingback x-varnish
      )
    end

    def eql?(other)
      super(other) && interesting_entries == other.interesting_entries
    end
  end
end
<commit_msg>Adds more headers to ignore<commit_after>module CMSScanner
  # Not really an interesting file, but will use this class for convenience
  class Headers < InterestingFile
    # @return [ Hash ] The headers
    def entries
      res = Browser.get(url)
      return [] unless res
      res.headers
    end

    # @return [ Array<String> ] The interesting headers detected
    def interesting_entries
      results = []

      entries.each do |header, value|
        next if known_headers.include?(header.downcase)

        results << "#{header}: #{value}"
      end
      results
    end

    # @return [ Array<String> ] Downcased known headers
    def known_headers
      %w(
        age accept-ranges cache-control content-type content-length connection date etag expires
        location last-modified pragma set-cookie strict-transport-security transfer-encoding vary
        x-cache x-content-security-policy x-content-type-options x-frame-options x-language
        x-permitted-cross-domain-policies x-pingback x-varnish x-webkit-csp x-xss-protection
      )
    end

    def eql?(other)
      super(other) && interesting_entries == other.interesting_entries
    end
  end
end

{'fraction_non_alphanumeric': 0.07868252516010979, 'fraction_numerical': 0.0, 'mean_word_length': 2.657190635451505, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1418783', 'n_tokens_mistral': 611, 'n_tokens_neox': 593, 'n_words': 187}