<commit_before>require "openssl"
OpenSSL::SSL::VERIFY_PEER = OpenSSL::SSL::VERIFY_NONE
require "gmail"
require "rubygems/format"

module GemsStatus
  class Mail
    attr_accessor :uid, :subject
  end


  class Utils
    attr_accessor :errors
    EMAIL_TMP_PATH = "tmp/utils/mail"
    @@errors = {}
    @@md5_sums = {}
    @@licenses = {}
    @@keys = {}
    @@gems = {}
    @@known_licenses = {}
    @@emails = nil

    def Utils.get_emails_from_fs
      @@emails = {}
      Dir.glob("#{EMAIL_TMP_PATH}/**/*").each do |filename|
        next unless File.file?(filename)
        open(filename, "r") do |file|
          mail = GemsStatus::Mail.new
          mailing_list = File.dirname(filename).split("/").pop
          mail.uid = File.basename(filename)
          mail.subject = file.read
          
          if @@emails[mailing_list]
            @@emails[mailing_list] << mail
          else
            @@emails[mailing_list] = [mail]
          end
          self.log_debug "email from filesystem #{filename}"
        end
      end
    end

    def Utils.save_email_to_fs(listname, mail)
      dirname = "#{EMAIL_TMP_PATH}/#{listname}"
      filename = "#{dirname}/#{mail.uid}"
      if File.exists?(filename)
        self.log_debug "email had already been downloaded #{mail.uid} - skipping"
        return
      end
      FileUtils::mkdir_p(dirname)  unless File.exists?(dirname)
      open(filename, "w") do |f|
        self.log_debug "writing email to fs #{mail.uid}"
        f.write(mail.subject)
      end
    end

    def Utils.download_emails(email_username, email_password, mailing_lists)
      begin
        self.get_emails_from_fs if @@emails.nil?
      rescue Exception => e
        self.log_error "", "There was a problem getting emails from filesystem #{e.message}"
      end
      if @@emails.nil?
        self.log_error "", "There was a problem getting emails from filesystem"
        @@emails = {}
      end
      self.log_debug "Emails read from filesystem #{@@emails.to_a.flatten.length}"
      Gmail.new(email_username, email_password) do |gmail|
       mailing_lists.each do |mailing_list|
         @@emails[mailing_list] = [] unless @@emails[mailing_list]
         Utils::log_debug "New security email alerts from #{mailing_list} #{gmail.inbox.count( :unread, :to => mailing_list)}"
         gmail.inbox.emails(:unread, :to => mailing_list).each do |email|
           mail = GemsStatus::Mail.new
           mail.uid = email.uid
           mail.subject = email.subject
           Utils::log_debug "Read #{mail.subject}"
           @@emails[mailing_list] << mail
           begin
             self.save_email_to_fs(mailing_list, mail)
             email.read!
           rescue Exception => e
             self.log_error "", "Error saving mail to filesystem #{e.message}"
             email.unread!
           end
         end
       end
      end
      return @@emails
    end

    def Utils.known_licenses=(licenses)
      @@known_licenses = licenses
    end

    def Utils.known_licenses
      return @@known_licenses
    end

    def Utils.errors
      return @@errors
    end

    def Utils.check_parameters(classname, conf, parameters)
      if !conf['classname'] then
        raise "trying to initialize #{classname} when parameter classname does not exists"
      end
      if conf['classname'] != classname then
        raise "trying to initialize #{classname} when parameter classname is #{conf['classname']}"
      end
      parameters.each do |p|
        if !conf[p] then
          raise "parameter #{p} not found for #{classname}"
        end
      end
    end

    def Utils.log_error(name, msg)
      @@errors[name] = "\n" unless @@errors[name]
      @@errors[name]  << msg << "\n"
      $stderr.puts "ERROR: #{msg}"
    end

    def Utils.log_debug(msg)
      $stderr.puts "DEBUG: #{msg}"
    end

    def Utils.download_md5(name, version, gems_url)
      key = "#{name}-#{version}-#{gems_url.gsub("/", "_").gsub(":", "_")}"
      return @@md5_sums[key] if @@md5_sums[key]
      begin
        gem_file_path = self.download_gem(name, version, gems_url)
      rescue Exception => e
        self.log_error(name, e.message)
        return nil
      end
      md5 = Digest::MD5.hexdigest(open(gem_file_path).read)
      @@md5_sums["#{name}-#{version}"] = md5
      return md5
    end

    def Utils.download_license(name, version, gems_url)
      version = version.to_s if version.is_a? Gem::Version
      key = "#{name}-#{version}-#{gems_url.gsub("/", "_").gsub(":", "_")}"
      return @@licenses[key] if @@licenses[key]
      begin
        gem_file_path = self.download_gem(name, version, gems_url)
      rescue Exception => e
        self.log_error(name, e.message)
        return nil
      end
      license = Gem::Format.from_file_by_path(gem_file_path).spec.license
      if !license || license.empty?
        if @@known_licenses[name]
          if @@known_licenses[name][version]
            license = @@known_licenses[name][version]
            Utils::log_debug "get license from known licenses for #{name} #{version}"
          else #@@known_licenses[name] but different version
            Utils::log_debug "I can't find license for #{name} but I have info from another version #{@@known_licenses[name].sort.last}"
          end
        end
      end<commit_msg>replace Gem::Format.from_file_by_path with Gem::Package.new<commit_after>require "openssl"
OpenSSL::SSL::VERIFY_PEER = OpenSSL::SSL::VERIFY_NONE
require "gmail"
require "rubygems/format"

module GemsStatus
  class Mail
    attr_accessor :uid, :subject
  end


  class Utils
    attr_accessor :errors
    EMAIL_TMP_PATH = "tmp/utils/mail"
    @@errors = {}
    @@md5_sums = {}
    @@licenses = {}
    @@keys = {}
    @@gems = {}
    @@known_licenses = {}
    @@emails = nil

    def Utils.get_emails_from_fs
      @@emails = {}
      Dir.glob("#{EMAIL_TMP_PATH}/**/*").each do |filename|
        next unless File.file?(filename)
        open(filename, "r") do |file|
          mail = GemsStatus::Mail.new
          mailing_list = File.dirname(filename).split("/").pop
          mail.uid = File.basename(filename)
          mail.subject = file.read
          
          if @@emails[mailing_list]
            @@emails[mailing_list] << mail
          else
            @@emails[mailing_list] = [mail]
          end
          self.log_debug "email from filesystem #{filename}"
        end
      end
    end

    def Utils.save_email_to_fs(listname, mail)
      dirname = "#{EMAIL_TMP_PATH}/#{listname}"
      filename = "#{dirname}/#{mail.uid}"
      if File.exists?(filename)
        self.log_debug "email had already been downloaded #{mail.uid} - skipping"
        return
      end
      FileUtils::mkdir_p(dirname)  unless File.exists?(dirname)
      open(filename, "w") do |f|
        self.log_debug "writing email to fs #{mail.uid}"
        f.write(mail.subject)
      end
    end

    def Utils.download_emails(email_username, email_password, mailing_lists)
      begin
        self.get_emails_from_fs if @@emails.nil?
      rescue Exception => e
        self.log_error "", "There was a problem getting emails from filesystem #{e.message}"
      end
      if @@emails.nil?
        self.log_error "", "There was a problem getting emails from filesystem"
        @@emails = {}
      end
      self.log_debug "Emails read from filesystem #{@@emails.to_a.flatten.length}"
      Gmail.new(email_username, email_password) do |gmail|
       mailing_lists.each do |mailing_list|
         @@emails[mailing_list] = [] unless @@emails[mailing_list]
         Utils::log_debug "New security email alerts from #{mailing_list} #{gmail.inbox.count( :unread, :to => mailing_list)}"
         gmail.inbox.emails(:unread, :to => mailing_list).each do |email|
           mail = GemsStatus::Mail.new
           mail.uid = email.uid
           mail.subject = email.subject
           Utils::log_debug "Read #{mail.subject}"
           @@emails[mailing_list] << mail
           begin
             self.save_email_to_fs(mailing_list, mail)
             email.read!
           rescue Exception => e
             self.log_error "", "Error saving mail to filesystem #{e.message}"
             email.unread!
           end
         end
       end
      end
      return @@emails
    end

    def Utils.known_licenses=(licenses)
      @@known_licenses = licenses
    end

    def Utils.known_licenses
      return @@known_licenses
    end

    def Utils.errors
      return @@errors
    end

    def Utils.check_parameters(classname, conf, parameters)
      if !conf['classname'] then
        raise "trying to initialize #{classname} when parameter classname does not exists"
      end
      if conf['classname'] != classname then
        raise "trying to initialize #{classname} when parameter classname is #{conf['classname']}"
      end
      parameters.each do |p|
        if !conf[p] then
          raise "parameter #{p} not found for #{classname}"
        end
      end
    end

    def Utils.log_error(name, msg)
      @@errors[name] = "\n" unless @@errors[name]
      @@errors[name]  << msg << "\n"
      $stderr.puts "ERROR: #{msg}"
    end

    def Utils.log_debug(msg)
      $stderr.puts "DEBUG: #{msg}"
    end

    def Utils.download_md5(name, version, gems_url)
      key = "#{name}-#{version}-#{gems_url.gsub("/", "_").gsub(":", "_")}"
      return @@md5_sums[key] if @@md5_sums[key]
      begin
        gem_file_path = self.download_gem(name, version, gems_url)
      rescue Exception => e
        self.log_error(name, e.message)
        return nil
      end
      md5 = Digest::MD5.hexdigest(open(gem_file_path).read)
      @@md5_sums["#{name}-#{version}"] = md5
      return md5
    end

    def Utils.download_license(name, version, gems_url)
      version = version.to_s if version.is_a? Gem::Version
      key = "#{name}-#{version}-#{gems_url.gsub("/", "_").gsub(":", "_")}"
      return @@licenses[key] if @@licenses[key]
      begin
        gem_file_path = self.download_gem(name, version, gems_url)
      rescue Exception => e
        self.log_error(name, e.message)
        return nil
      end
      license = Gem::Package.new(gem_file_path).spec.license
      if !license || license.empty?
        if @@known_licenses[name]
          if @@known_licenses[name][version]
            license = @@known_licenses[name][version]
            Utils::log_debug "get license from known licenses for #{name} #{version}"
          else #@@known_licenses[name] but different version
            Utils::log_debug "I can't find license for #{name} but I have info from another version #{@@known_licenses[name].sort.last}"
          end
        end
      end
{'fraction_non_alphanumeric': 0.12475340535462659, 'fraction_numerical': 0.001690934711131987, 'mean_word_length': 2.416559691912709, 'pattern_counts': {'":': 4, '<': 19, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2609649', 'n_tokens_mistral': 3388, 'n_tokens_neox': 3281, 'n_words': 782}