<commit_before>require 'base64'
require 'logger'
require 'zip'
require 'bundler'
require 'rest_client'
require 'json'

require_relative 'api'

module IronWorker

  class Service < IronWorker::Api::Client

    attr_accessor :config

    def initialize(token, options={})
      if options[:config]
        self.config = options[:config]
      else
        c = IronWorker::Config.new unless self.config
        c.token = token
        self.config = c
      end
      options[:version] = IronWorker.api_version
      options[:logger] = IronWorker.logger
      super("worker-aws-us-east-1.iron.io", token, options)
      self.host = self.config.host if self.config && self.config.host
      # automerge simple worker gem and dependenices
      self.config.merge_gem('zip')
      self.config.merge_gem('rest-client')
      self.config.merge_gem('rest')
      self.config.merge_gem('iron_worker')
      IronWorker.logger.info 'IronWorker initialized.'
      IronWorker.logger.debug ' host = ' + self.host.inspect
    end

    # Options:
    #    - :callback_url
    #    - :merge => array of files to merge in with this file
    def upload(filename, class_name, options={})
      name = options[:name] || class_name
      project_id = get_project_id(options)
      tmp = Dir.tmpdir()
      md5file = "iron_worker_#{class_name.gsub("::", ".")}_#{token[0, 8]}.md5"
      existing_md5 = nil
      md5_f = File.join(tmp, md5file)
      if File.exists?(md5_f)
        existing_md5 = IO.read(md5_f)
      end


      begin

        zip_filename = build_merged_file(filename, options[:merge], options[:unmerge], options[:merged_gems], options[:unmerged_gems], options[:merged_mailers], options[:merged_folders])

        # Check for code changes.
        zipfile = Zip::ZipFile.open(zip_filename, Zip::ZipFile::CREATE)
        crc = zipfile.entries.collect { |x| x.crc }.inject(:+)
        new_code = false
        if self.config.force_upload || crc.to_s != existing_md5
          IronWorker.logger.info "Uploading #{class_name}, code modified."
          File.open(md5_f, 'w') { |f| f.write(crc) }
          new_code = true
          # todo: delete md5 file if error occurs during upload process
        else
#        puts "#{class_name}: same code, not uploading"
          return
        end

        if new_code
          upload_code(name, zip_filename, 'runner.rb', :runtime=>'ruby')
        end

      rescue Exception => ex
        # if it errors, let's delete md5 since it wouldn't have uploaded.
        File.delete(md5_f) if File.exists?(md5_f)
        raise ex
      end
    end

    def get_server_gems
      hash = get("gems/list")
      JSON.parse(hash["gems"])
    end

    def logger
      IronWorker.logger
    end

    def self.get_gem_path(gem_info)
#      gem_name =(gem_info[:require] || gem_info[:name].match(/^[a-zA-Z0-9\-_]+/)[0])
      gem_name =(gem_info[:name].match(/^[a-zA-Z0-9\-_]+/)[0])
      #puts "Searching for #{gem_name}..."
      gems= Gem::Specification.respond_to?(:each) ? Gem::Specification.find_all_by_name(gem_name) : Gem::GemPathSearcher.new.find_all(gem_name)
      if (!Gem::GemPathSearcher.private_instance_methods.include?(:_deprecated_initialize)) && (!gems || gems.empty?)
        gems = Gem::GemPathSearcher.new.init_gemspecs.select { |gem| gem.name==gem_name }
      end
      IronWorker.logger.debug 'gems found=' + gems.inspect
      gems = gems.select { |g| g.version.version==gem_info[:version] } if gem_info[:version]
      if !gems.empty?
        gem = gems.last
        return gem, gem.full_gem_path
      else
        return nil, nil
      end
    end


    def gem_dependencies(list_of_gems)
      IronWorker.logger.debug "Getting gem_dependencies.."
      deps = []
      dependendent_gems ={}
      return unless Gem::Specification.respond_to? :all
      list_of_gems.each_value do |v|
        IronWorker.logger.debug "Dependencies for #{v}"
        @deps = v[:gemspec].dependencies
        @deps.each do |d|
          IronWorker.logger.debug "dep: #{d}"
          deps << Bundler::DepProxy.new(d, 'ruby')
        end
      end
      filtered_deps = deps.select { |d| d.type != :development }
      IronWorker.logger.debug "filtered_deps=#{filtered_deps}"
      index = Bundler::Index.new
      Gem::Specification.all.collect { |s| index<<s }
      list = Bundler::Resolver.resolve(filtered_deps, index)
      list.each do |gemspec|
        next if list_of_gems.keys.include?(gemspec.name)
        gi = IronWorker::MergeHelper.create_gem_info(gemspec.name, gemspec.version.version)
        dependendent_gems[gemspec.name] = gi
      end
      dependendent_gems<commit_msg>bugfix for deps automerge feature on heroku<commit_after>require 'base64'
require 'logger'
require 'zip'
require 'bundler'
require 'rest_client'
require 'json'

require_relative 'api'

module IronWorker

  class Service < IronWorker::Api::Client

    attr_accessor :config

    def initialize(token, options={})
      if options[:config]
        self.config = options[:config]
      else
        c = IronWorker::Config.new unless self.config
        c.token = token
        self.config = c
      end
      options[:version] = IronWorker.api_version
      options[:logger] = IronWorker.logger
      super("worker-aws-us-east-1.iron.io", token, options)
      self.host = self.config.host if self.config && self.config.host
      # automerge simple worker gem and dependenices
      self.config.merge_gem('zip')
      self.config.merge_gem('rest-client')
      self.config.merge_gem('rest')
      self.config.merge_gem('iron_worker')
      IronWorker.logger.info 'IronWorker initialized.'
      IronWorker.logger.debug ' host = ' + self.host.inspect
    end

    # Options:
    #    - :callback_url
    #    - :merge => array of files to merge in with this file
    def upload(filename, class_name, options={})
      name = options[:name] || class_name
      project_id = get_project_id(options)
      tmp = Dir.tmpdir()
      md5file = "iron_worker_#{class_name.gsub("::", ".")}_#{token[0, 8]}.md5"
      existing_md5 = nil
      md5_f = File.join(tmp, md5file)
      if File.exists?(md5_f)
        existing_md5 = IO.read(md5_f)
      end


      begin

        zip_filename = build_merged_file(filename, options[:merge], options[:unmerge], options[:merged_gems], options[:unmerged_gems], options[:merged_mailers], options[:merged_folders])

        # Check for code changes.
        zipfile = Zip::ZipFile.open(zip_filename, Zip::ZipFile::CREATE)
        crc = zipfile.entries.collect { |x| x.crc }.inject(:+)
        new_code = false
        if self.config.force_upload || crc.to_s != existing_md5
          IronWorker.logger.info "Uploading #{class_name}, code modified."
          File.open(md5_f, 'w') { |f| f.write(crc) }
          new_code = true
          # todo: delete md5 file if error occurs during upload process
        else
#        puts "#{class_name}: same code, not uploading"
          return
        end

        if new_code
          upload_code(name, zip_filename, 'runner.rb', :runtime=>'ruby')
        end

      rescue Exception => ex
        # if it errors, let's delete md5 since it wouldn't have uploaded.
        File.delete(md5_f) if File.exists?(md5_f)
        raise ex
      end
    end

    def get_server_gems
      hash = get("gems/list")
      JSON.parse(hash["gems"])
    end

    def logger
      IronWorker.logger
    end

    def self.get_gem_path(gem_info)
#      gem_name =(gem_info[:require] || gem_info[:name].match(/^[a-zA-Z0-9\-_]+/)[0])
      gem_name =(gem_info[:name].match(/^[a-zA-Z0-9\-_]+/)[0])
      #puts "Searching for #{gem_name}..."
      gems= Gem::Specification.respond_to?(:each) ? Gem::Specification.find_all_by_name(gem_name) : Gem::GemPathSearcher.new.find_all(gem_name)
      if (!Gem::GemPathSearcher.private_instance_methods.include?(:_deprecated_initialize)) && (!gems || gems.empty?)
        gems = Gem::GemPathSearcher.new.init_gemspecs.select { |gem| gem.name==gem_name }
      end
      IronWorker.logger.debug 'gems found=' + gems.inspect
      gems = gems.select { |g| g.version.version==gem_info[:version] } if gem_info[:version]
      if !gems.empty?
        gem = gems.last
        return gem, gem.full_gem_path
      else
        return nil, nil
      end
    end


    def gem_dependencies(list_of_gems)
      return {} unless Gem::Specification.respond_to? :all
      IronWorker.logger.debug "Getting gem_dependencies.."
      deps = []
      dependendent_gems ={}
      list_of_gems.each_value do |v|
        IronWorker.logger.debug "Dependencies for #{v}"
        @deps = v[:gemspec].dependencies
        @deps.each do |d|
          IronWorker.logger.debug "dep: #{d}"
          deps << Bundler::DepProxy.new(d, 'ruby')
        end
      end
      filtered_deps = deps.select { |d| d.type != :development }
      IronWorker.logger.debug "filtered_deps=#{filtered_deps}"
      index = Bundler::Index.new
      Gem::Specification.all.collect { |s| index<<s }
      list = Bundler::Resolver.resolve(filtered_deps, index)
      list.each do |gemspec|
        next if list_of_gems.keys.include?(gemspec.name)
        gi = IronWorker::MergeHelper.create_gem_info(gemspec.name, gemspec.version.version)
        dependendent_gems[gemspec.name] = gi
      end
      dependendent_gems
{'fraction_non_alphanumeric': 0.1263611859838275, 'fraction_numerical': 0.005390835579514825, 'mean_word_length': 2.7493936944219888, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1176284', 'n_tokens_mistral': 3073, 'n_tokens_neox': 2953, 'n_words': 615}