<commit_before>File.dirname(__FILE__).tap do |supermarket|
  Dir[File.join(supermarket, 'import', '*.rb')].map do |file|
    file.split(File::SEPARATOR).last.split('.').first
  end.each do |name|
    require "supermarket/import/#{name}"
  end
end

require 'supermarket/community_site'

module Supermarket
  module Import
    def self.debug
      yield if ENV['SUPERMARKET_DEBUG']
    end

    def self.report(e)
      raven_options = {}

      if e.respond_to?(:record) && e.record.is_a?(::CookbookVersion)
        if e.record.errors[:tarball_content_type]
          raven_options[:extra] = {
            tarball_content_type: e.record.tarball_content_type
          }
        end
      end

      if e.is_a?(CookbookVersionDependencies::UnableToProcessTarball)
        raven_options[:extra] = {
          cookbook_name: e.cookbook_name,
          cookbook_version: e.cookbook_version,
          messages: e.errors.full_messages.join('; ')
        }
      end

      Raven.capture_exception(e, raven_options)

      debug do
        message_header = "#{e.class}: #{e.message}\n  #{raven_options.inspect}"
        message_body = ([message_header] + e.backtrace).join("\n  ")

        yield message_body
      end
    end
  end
end
<commit_msg>Debug messages show scrubbed backtraces<commit_after>File.dirname(__FILE__).tap do |supermarket|
  Dir[File.join(supermarket, 'import', '*.rb')].map do |file|
    file.split(File::SEPARATOR).last.split('.').first
  end.each do |name|
    require "supermarket/import/#{name}"
  end
end

require 'supermarket/community_site'

module Supermarket
  module Import
    def self.debug
      yield if ENV['SUPERMARKET_DEBUG']
    end

    def self.report(e)
      raven_options = {}

      if e.respond_to?(:record) && e.record.is_a?(::CookbookVersion)
        if e.record.errors[:tarball_content_type]
          raven_options[:extra] = {
            tarball_content_type: e.record.tarball_content_type
          }
        end
      end

      if e.is_a?(CookbookVersionDependencies::UnableToProcessTarball)
        raven_options[:extra] = {
          cookbook_name: e.cookbook_name,
          cookbook_version: e.cookbook_version,
          messages: e.errors.full_messages.join('; ')
        }
      end

      Raven.capture_exception(e, raven_options)

      debug do
        message_header = "#{e.class}: #{e.message}\n  #{raven_options.inspect}"

        relevant_backtrace = e.backtrace.select do |line|
          line.include?('supermarket') || line.include?('chef-legacy')
        end

        message_body = ([message_header] + relevant_backtrace).join("\n  ")

        yield message_body
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.12067669172932331, 'fraction_numerical': 0.0, 'mean_word_length': 2.7164804469273744, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6479400', 'n_tokens_mistral': 878, 'n_tokens_neox': 857, 'n_words': 147}