<commit_before>require 'tmpdir'
require 'fileutils'
require 'open-uri'

module YARD
  module CLI
    # CLI command to return the objects that were added/removed from 2 versions
    # of a project (library, gem, working copy).
    # @since 0.6.0
    class Diff < Command
      def initialize
        super
        @list_all = false
        log.show_backtraces = true
      end
      
      def description
        'Returns the object diff of two gems or .yardoc files'
      end
      
      def run(*args)
        registry = optparse(*args).map do |gemfile|
          if load_gem_data(gemfile)
            log.info "Found #{gemfile}"
            Registry.all.map {|o| o.path }
          else
            log.error "Cannot find gem #{gemfile}"
            nil
          end
        end.compact
        
        return if registry.size != 2

        [   ["Added objects", registry[1] - registry[0]],
            ["Removed objects", registry[0] - registry[1]]].each do |name, objects|
          next if objects.empty?
          last_object = nil
          all_objects_notice = false
          puts name + ":"
          objects.sort.each do |object|
            if !@list_all && last_object && object =~ /#{Regexp.quote last_object}(::|\.|#)/
              print " (...)" unless all_objects_notice
              all_objects_notice = true
              next
            else
              puts
            end
            all_objects_notice = false
            print "  " + object
            last_object = object
          end
          puts
          puts
        end
      end
      
      private
      
      def load_gem_data(gemfile)
        Registry.clear
        
        # First check for argument as .yardoc file
        [File.join(gemfile, '.yardoc'), gemfile].each do |yardoc|
          log.info "Searching for .yardoc db at #{yardoc}"
          if File.directory?(yardoc)
            Registry.load_yardoc(yardoc)
            Registry.load_all
            return true
          end
        end
        
        # Next check installed RubyGems
        gemfile_without_ext = gemfile.sub(/\.gem$/, '')
        log.info "Searching for installed gem #{gemfile_without_ext}"
        Gem.source_index.find_name('').find do |spec| 
          if spec.full_name == gemfile_without_ext
            if yardoc = Registry.yardoc_file_for_gem(spec.name, "= #{spec.version}")
              Registry.load_yardoc(yardoc)
              Registry.load_all
            else
              log.enter_level(Logger::ERROR) do
                olddir = Dir.pwd
                Gems.run(spec.name, spec.version.to_s)
                Dir.chdir(olddir)
              end
            end
            return true
          end
        end
        
        # Look for local .gem file
        gemfile += '.gem' unless gemfile =~ /\.gem$/
        log.info "Searching for local gem file #{gemfile}"
        if File.exist?(gemfile)
          File.open(gemfile, 'rb') do |io|
            expand_and_parse(gemfile, io)
          end
          return true
        end
        
        # Remote gemfile from rubygems.org
        url = "http://rubygems.org/downloads/#{gemfile}"
        log.info "Searching for remote gem file #{url}"
        begin
          open(url) {|io| expand_and_parse(gemfile, io) }
          return true
        rescue OpenURI::HTTPError
        end
        false
      end
      
      def expand_and_parse(gemfile, io)
        dir = expand_gem(gemfile, io)
        generate_yardoc(dir)
        cleanup(gemfile)
      end
      
      def generate_yardoc(dir)
        olddir = Dir.pwd
        Dir.chdir(dir)
        log.enter_level(Logger::ERROR) { Yardoc.run('-n', '--no-save') }
        Dir.chdir(olddir)
      end
      
      def expand_gem(gemfile, io)
        tmpdir = File.join(Dir.tmpdir, gemfile)
        log.info "Expanding #{gemfile} to #{tmpdir}..."
        FileUtils.mkdir_p(tmpdir)
        Gem::Package.open(io) do |pkg|
          pkg.each do |entry|
            pkg.extract_entry(tmpdir, entry)
          end
        end
        tmpdir
      end
      
      def cleanup(gemfile)
        dir = File.join(Dir.tmpdir, gemfile)
        log.info "Cleaning up #{dir}..."
        FileUtils.rm_rf(dir)
      end
      
      def optparse(*args)
        opts = OptionParser.new
        opts.banner = "Usage: yard diff [options] oldgem newgem"
        opts.separator ""
        opts.separator "Example: yard diff yard-0.5.6 yard-0.5.8"
        opts.separator ""
        opts.separator "If the files don't exist locally, they will be grabbed using the `gem fetch`"
        opts.separator "command. If the gem is a .yardoc directory, it will be used. Finally, if the"
        opts.separator "gem name matches an installed gem (full name-version syntax), that gem will be used."
        
        opts.on('-a', '--all', 'List all objects, even if they are inside added/removed module/class') do
          @list_all = true
        end
        common_options(opts)
        parse_options(opts, args)
        unless args.size == 2
          puts opts.banner
          exit(0)
        end
          
        args
      end
    end
  end
end<commit_msg>Make sure rubygems/package is loaded before running Diff utility<commit_after>require 'tmpdir'
require 'fileutils'
require 'open-uri'

module YARD
  module CLI
    # CLI command to return the objects that were added/removed from 2 versions
    # of a project (library, gem, working copy).
    # @since 0.6.0
    class Diff < Command
      def initialize
        require_rubygems
        super
        @list_all = false
        log.show_backtraces = true
      end
      
      def description
        'Returns the object diff of two gems or .yardoc files'
      end
      
      def run(*args)
        registry = optparse(*args).map do |gemfile|
          if load_gem_data(gemfile)
            log.info "Found #{gemfile}"
            Registry.all.map {|o| o.path }
          else
            log.error "Cannot find gem #{gemfile}"
            nil
          end
        end.compact
        
        return if registry.size != 2

        [   ["Added objects", registry[1] - registry[0]],
            ["Removed objects", registry[0] - registry[1]]].each do |name, objects|
          next if objects.empty?
          last_object = nil
          all_objects_notice = false
          puts name + ":"
          objects.sort.each do |object|
            if !@list_all && last_object && object =~ /#{Regexp.quote last_object}(::|\.|#)/
              print " (...)" unless all_objects_notice
              all_objects_notice = true
              next
            else
              puts
            end
            all_objects_notice = false
            print "  " + object
            last_object = object
          end
          puts
          puts
        end
      end
      
      private
      
      def load_gem_data(gemfile)
        Registry.clear
        
        # First check for argument as .yardoc file
        [File.join(gemfile, '.yardoc'), gemfile].each do |yardoc|
          log.info "Searching for .yardoc db at #{yardoc}"
          if File.directory?(yardoc)
            Registry.load_yardoc(yardoc)
            Registry.load_all
            return true
          end
        end
        
        # Next check installed RubyGems
        gemfile_without_ext = gemfile.sub(/\.gem$/, '')
        log.info "Searching for installed gem #{gemfile_without_ext}"
        Gem.source_index.find_name('').find do |spec| 
          if spec.full_name == gemfile_without_ext
            if yardoc = Registry.yardoc_file_for_gem(spec.name, "= #{spec.version}")
              Registry.load_yardoc(yardoc)
              Registry.load_all
            else
              log.enter_level(Logger::ERROR) do
                olddir = Dir.pwd
                Gems.run(spec.name, spec.version.to_s)
                Dir.chdir(olddir)
              end
            end
            return true
          end
        end
        
        # Look for local .gem file
        gemfile += '.gem' unless gemfile =~ /\.gem$/
        log.info "Searching for local gem file #{gemfile}"
        if File.exist?(gemfile)
          File.open(gemfile, 'rb') do |io|
            expand_and_parse(gemfile, io)
          end
          return true
        end
        
        # Remote gemfile from rubygems.org
        url = "http://rubygems.org/downloads/#{gemfile}"
        log.info "Searching for remote gem file #{url}"
        begin
          open(url) {|io| expand_and_parse(gemfile, io) }
          return true
        rescue OpenURI::HTTPError
        end
        false
      end
      
      def expand_and_parse(gemfile, io)
        dir = expand_gem(gemfile, io)
        generate_yardoc(dir)
        cleanup(gemfile)
      end
      
      def generate_yardoc(dir)
        olddir = Dir.pwd
        Dir.chdir(dir)
        log.enter_level(Logger::ERROR) { Yardoc.run('-n', '--no-save') }
        Dir.chdir(olddir)
      end
      
      def expand_gem(gemfile, io)
        tmpdir = File.join(Dir.tmpdir, gemfile)
        log.info "Expanding #{gemfile} to #{tmpdir}..."
        FileUtils.mkdir_p(tmpdir)
        Gem::Package.open(io) do |pkg|
          pkg.each do |entry|
            pkg.extract_entry(tmpdir, entry)
          end
        end
        tmpdir
      end

      def require_rubygems
        require 'rubygems'
        require 'rubygems/package'
      rescue LoadError => e
        log.error "Missing RubyGems, cannot run this command."
        raise(e)
      end
      
      def cleanup(gemfile)
        dir = File.join(Dir.tmpdir, gemfile)
        log.info "Cleaning up #{dir}..."
        FileUtils.rm_rf(dir)
      end
      
      def optparse(*args)
        opts = OptionParser.new
        opts.banner = "Usage: yard diff [options] oldgem newgem"
        opts.separator ""
        opts.separator "Example: yard diff yard-0.5.6 yard-0.5.8"
        opts.separator ""
        opts.separator "If the files don't exist locally, they will be grabbed using the `gem fetch`"
        opts.separator "command. If the gem is a .yardoc directory, it will be used. Finally, if the"
        opts.separator "gem name matches an installed gem (full name-version syntax), that gem will be used."
        
        opts.on('-a', '--all', 'List all objects, even if they are inside added/removed module/class') do
          @list_all = true
        end
        common_options(opts)
        parse_options(opts, args)
        unless args.size == 2
          puts opts.banner
          exit(0)
        end
          
        args
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.09286255206361227, 'fraction_numerical': 0.003218477849299508, 'mean_word_length': 1.574317738791423, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6396919', 'n_tokens_mistral': 3081, 'n_tokens_neox': 3042, 'n_words': 837}