<commit_before>module PackageManager
  class Base
    COLOR = '#fff'
    BIBLIOTHECARY_SUPPORT = false
    BIBLIOTHECARY_PLANNED = false
    SECURITY_PLANNED = false
    HIDDEN = false

    def self.platforms
      @platforms ||= begin
        PackageManager.constants
          .reject { |platform| platform == :Base }
          .map{|sym| "PackageManager::#{sym}".constantize }
          .reject { |platform| platform::HIDDEN }
          .sort_by(&:name)
      end
    end

    def self.language
      Linguist::Language.all.find{|l| l.color == color }.try(:name)
    end

    def self.format_name(platform)
      return nil if platform.nil?
      platforms.find{|p| p.to_s.demodulize.downcase == platform.downcase }.to_s.demodulize
    end

    def self.color
      self::COLOR
    end

    def self.formatted_name
      self.to_s.demodulize
    end

    def self.package_link(_project, _version = nil)
      nil
    end

    def self.download_url(_name, _version = nil)
      nil
    end

    def self.documentation_url(_name, _version = nil)
      nil
    end

    def self.install_instructions(_project, _version = nil)
      nil
    end

    def self.check_status_url(project)
      package_link(project)
    end

    def self.platform_name(platform)
      if platform.downcase == 'npm'
        return 'npm'
      elsif platform.downcase == 'wordpress'
        return 'WordPress'
      else
        return platform
      end
    end

    def self.dependency_platform(platform_string)
      return platform_string if platform_string.nil?
      case platform_string.downcase
      when 'rubygemslockfile'
        'rubygems'
      when 'cocoapodslockfile'
        'cocoapods'
      when 'nugetlockfile', 'nuspec'
        'nuget'
      when 'packagistlockfile'
        'packagist'
      when 'gemspec'
        'rubygems'
      when 'npmshrinkwrap'
        'npm'
      else
        platform_string.downcase
      end
    end

    def self.save(project)
      return unless project.present?
      mapped_project = mapping(project)
      mapped_project = mapped_project.delete_if { |_key, value| value.blank? } if mapped_project.present?
      return false unless mapped_project.present?
      puts "Saving #{mapped_project[:name]}"
      dbproject = Project.find_or_initialize_by({:name => mapped_project[:name], :platform => self.name.demodulize})
      if dbproject.new_record?
        dbproject.assign_attributes(mapped_project.except(:name, :releases, :versions))
        dbproject.save
      else
        dbproject.update_attributes(mapped_project.except(:name, :releases, :versions))
      end

      if self::HAS_VERSIONS
        versions(project).each do |version|
          dbproject.versions.find_or_create_by(version)
        end
      end

      if self::HAS_DEPENDENCIES
        save_dependencies(mapped_project)
      end
      dbproject.reload
      dbproject.last_synced_at = Time.now
      dbproject.save
      dbproject
    end

    def self.update(name)
      begin
        project = project(name)
        save(project) if project.present?
      rescue SystemExit, Interrupt
        exit 0
      rescue Exception => exception
        if ENV["RACK_ENV"] == "production"
          Bugsnag.notify(exception)
        else
          raise
        end
      end
    end

    def self.import_async
      download_async(project_names)
    end

    def self.import_recent_async
      download_async(recent_names)
    end

    def self.import_new_async
      download_async(new_names)
    end

    def self.import
      project_names.each { |name| update(name) }
    end

    def self.import_recent
      recent_names.each { |name| update(name) }
    end

    def self.import_new
      new_names.each { |name| update(name) }
    end

    def self.new_names
      names = project_names
      existing_names = []
      Project.platform(self.name.demodulize).select(:id, :name).find_each {|project| existing_names << project.name }
      names - existing_names
    end

    def self.save_dependencies(mapped_project)
      name = mapped_project[:name]
      proj = Project.find_by(name: name, platform: self.name.demodulize)
      proj.versions.each do |version|
        deps = dependencies(name, version.number, mapped_project)
        next unless deps && deps.any? && version.dependencies.empty?
        deps.each do |dep|
          unless version.dependencies.find_by_project_name dep[:project_name]
            named_project = Project.platform(self.name.demodulize).where('lower(name) = ?', dep[:project_name].downcase).first.try(:id)
            version.dependencies.create(dep.merge(project_id: named_project.try(:strip)))
          end
        end
      end
    end

    def self.dependencies(_name, _version, _project)
      []
    end<commit_msg>Eager loading in package manager syncing<commit_after>module PackageManager
  class Base
    COLOR = '#fff'
    BIBLIOTHECARY_SUPPORT = false
    BIBLIOTHECARY_PLANNED = false
    SECURITY_PLANNED = false
    HIDDEN = false

    def self.platforms
      @platforms ||= begin
        PackageManager.constants
          .reject { |platform| platform == :Base }
          .map{|sym| "PackageManager::#{sym}".constantize }
          .reject { |platform| platform::HIDDEN }
          .sort_by(&:name)
      end
    end

    def self.language
      Linguist::Language.all.find{|l| l.color == color }.try(:name)
    end

    def self.format_name(platform)
      return nil if platform.nil?
      platforms.find{|p| p.to_s.demodulize.downcase == platform.downcase }.to_s.demodulize
    end

    def self.color
      self::COLOR
    end

    def self.formatted_name
      self.to_s.demodulize
    end

    def self.package_link(_project, _version = nil)
      nil
    end

    def self.download_url(_name, _version = nil)
      nil
    end

    def self.documentation_url(_name, _version = nil)
      nil
    end

    def self.install_instructions(_project, _version = nil)
      nil
    end

    def self.check_status_url(project)
      package_link(project)
    end

    def self.platform_name(platform)
      if platform.downcase == 'npm'
        return 'npm'
      elsif platform.downcase == 'wordpress'
        return 'WordPress'
      else
        return platform
      end
    end

    def self.dependency_platform(platform_string)
      return platform_string if platform_string.nil?
      case platform_string.downcase
      when 'rubygemslockfile'
        'rubygems'
      when 'cocoapodslockfile'
        'cocoapods'
      when 'nugetlockfile', 'nuspec'
        'nuget'
      when 'packagistlockfile'
        'packagist'
      when 'gemspec'
        'rubygems'
      when 'npmshrinkwrap'
        'npm'
      else
        platform_string.downcase
      end
    end

    def self.save(project)
      return unless project.present?
      mapped_project = mapping(project)
      mapped_project = mapped_project.delete_if { |_key, value| value.blank? } if mapped_project.present?
      return false unless mapped_project.present?
      puts "Saving #{mapped_project[:name]}"
      dbproject = Project.find_or_initialize_by({:name => mapped_project[:name], :platform => self.name.demodulize})
      if dbproject.new_record?
        dbproject.assign_attributes(mapped_project.except(:name, :releases, :versions))
        dbproject.save
      else
        dbproject.update_attributes(mapped_project.except(:name, :releases, :versions))
      end

      if self::HAS_VERSIONS
        versions(project).each do |version|
          unless dbproject.versions.find {|v| v.number == version[:number] }
            dbproject.versions.create(version)
          end
        end
      end

      if self::HAS_DEPENDENCIES
        save_dependencies(mapped_project)
      end
      dbproject.reload
      dbproject.last_synced_at = Time.now
      dbproject.save
      dbproject
    end

    def self.update(name)
      begin
        project = project(name)
        save(project) if project.present?
      rescue SystemExit, Interrupt
        exit 0
      rescue Exception => exception
        if ENV["RACK_ENV"] == "production"
          Bugsnag.notify(exception)
        else
          raise
        end
      end
    end

    def self.import_async
      download_async(project_names)
    end

    def self.import_recent_async
      download_async(recent_names)
    end

    def self.import_new_async
      download_async(new_names)
    end

    def self.import
      project_names.each { |name| update(name) }
    end

    def self.import_recent
      recent_names.each { |name| update(name) }
    end

    def self.import_new
      new_names.each { |name| update(name) }
    end

    def self.new_names
      names = project_names
      existing_names = []
      Project.platform(self.name.demodulize).select(:id, :name).find_each {|project| existing_names << project.name }
      names - existing_names
    end

    def self.save_dependencies(mapped_project)
      name = mapped_project[:name]
      proj = Project.find_by(name: name, platform: self.name.demodulize)
      proj.versions.includes(:dependencies).each do |version|
        deps = dependencies(name, version.number, mapped_project)
        next unless deps && deps.any? && version.dependencies.empty?
        deps.each do |dep|
          unless version.dependencies.find_by_project_name dep[:project_name]
            named_project = Project.platform(self.name.demodulize).where('lower(name) = ?', dep[:project_name].downcase).first.try(:id)
            version.dependencies.create(dep.merge(project_id: named_project.try(:strip)))
          end
        end
      end
    end

    def self.dependencies(_name, _version, _project)
      []
    end
{'fraction_non_alphanumeric': 0.09009009009009009, 'fraction_numerical': 0.00020710365537951746, 'mean_word_length': 2.4370106761565835, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4719303', 'n_tokens_mistral': 3012, 'n_tokens_neox': 2910, 'n_words': 604}