<commit_before>require 'tempfile'
require 'fileutils'

module Spider

    module AppManager
        
        def self.install_or_update(apps, options={})
            require 'spiderfw/home'
            home = Spider::Home.new(Dir.pwd)
            installed = {}
            Spider.init_base
            active = Spider.config.get('apps')
            Spider.home.apps.each do |app, info|
                installed[app] = {
                    :active => active.include?(app)
                }
                if spec = info[:spec]
                    installed[app].merge!({
                        :version => spec.version
                    })                        
                end
            end
            to_inst = apps.select{ |a| !installed[a] }
            to_upd = apps.select{ |a| installed[a] }
            install_apps(to_inst, options)
            update_apps(to_upd, options)
            return {:installed => to_inst, :updated => to_upd}
        end
        
        def self.install_apps(apps, options={})
            return if apps.empty?
            require 'spiderfw/setup/app_server_client'
            use_git = false
            unless options[:no_git]
                begin
                    require 'git'
                    use_git = true
                rescue => exc
                    puts exc.message
                    puts "git gem not available; install git gem for Git support"
                end
            end
            
            existent = []
            apps.each do |app|
                if File.exist?("apps/#{app}")
                    puts _("%s already exists, skipping") % app
                    existent << app
                end
            end
            require 'spiderfw/setup/app_manager'
            specs = []
            url = options[:url]
            unless url
                require 'spiderfw/spider'
                Spider.init_base
                url = Spider.config.get('app_server.url')
            end
            client = Spider::AppServerClient.new(url)
            if options[:no_deps]
                specs = client.get_specs(apps)
            else
                specs = client.get_deps(apps, :no_optional => options[:no_optional])
            end
            deps = specs.map{ |s| s.app_id }
            unless (deps - apps).empty?
                puts _("The following apps will be installed as a dependency:")
                puts (deps - apps).inspect
            end
            i_options = {
                :use_git => use_git, 
                :no_gems => options[:no_gems],
                :no_optional_gems => options[:no_optional_gems]
            }
            i_options[:ssh_user] = options[:ssh_user] if options[:ssh_user]
            inst_specs = specs.reject{ |s| existent.include? s.app_id }
            Spider::AppManager.install(inst_specs, Dir.pwd, i_options)
            unless options[:no_activate]
                require 'spiderfw/spider'
                specs_hash = {}
                specs.each{ |s| specs_hash[s.app_id] = s }
                Spider.activate_apps(deps, specs_hash)
            end
            
        end
        
        def self.update_apps(apps, options={})
            require 'spiderfw/spider'
            require 'spiderfw/setup/app_server_client'
            Spider.init_base
            url = options[:url] || Spider.conf.get('app_server.url')
            use_git = false
            unless options[:no_git]
                begin
                    require 'git'
                    use_git = true
                rescue
                    puts "git gem not available; install git gem for Git support"
                end
            end
            if options[:all]
                require 'spiderfw/home'
                home = Spider::Home.new(Dir.pwd)
                apps = home.list_apps
            end
            if apps.empty?
                puts _("No app to update")
                exit
            end
            require 'spiderfw/setup/app_manager'
            specs = []
            client = Spider::AppServerClient.new(url)
            if options[:no_deps]
                specs = client.get_specs(apps)
            else
                specs = client.get_deps(apps, :no_optional => options[:no_optional])
            end
            deps = specs.map{ |s| s.app_id }
            unless (deps - apps).empty?
                puts _("The following apps will be updated as a dependency:")
                puts (deps - apps).inspect
            end
            Spider::AppManager.update(specs, Dir.pwd, {
                :use_git => use_git, 
                :no_gems => options[:no_gems],
                :no_optional_gems => options[:no_optional_gems]
            })
        end

        def self.install(specs, home_path, options)
            options[:use_git] = true unless options[:use_git] == false
            options[:home_path] = home_path
            specs = [specs] if specs && !specs.is_a?(Array)
            specs ||= []
            pre_setup(specs, options)
            specs.each do |spec|
                if spec.git_repo && options[:use_git]
                    git_install(spec, home_path, options)
                else
                    pack_install(spec, home_path, options)
                end
            end
            post_setup(specs, options)
        end

        def self.git_install(spec, home_path, options={})
            require 'git'
            if ::File.exist?("apps/#{spec.id}")
                puts _("%s already installed, skipping") % spec.id
                return
            end
            repo = Git.open(home_path)
            puts _("Fetching %s from %s") % [spec.app_id, spec.git_repo]
            repo_url = spec.git_repo
            if options[:ssh_user] && repo_url =~ /ssh:\/\/([^@]+@)?(.+)/
                repo_url = "ssh://#{options[:ssh_user]}@#{$2}"
            end
            
            Dir.chdir(home_path) do
                `git submodule add #{repo_url} apps/#{spec.id}`
                `git submodule init`
                `git submodule update`
            end
            repo.add(['.gitmodules', "apps/#{spec.id}"])
            repo.commit(_("Added app %s") % spec.id) 
        end

        def self.pack_install(spec, home_path, options={})
            require 'rubygems/package'
            client = AppServerClient.new(spec.app_server)
            print _("Fetching %s from server... ") % spec.app_id
            tmp_path = client.fetch_app(spec.app_id)
            puts _("Fetched.")
            dest = File.join(home_path, "apps/#{spec.app_id}")
            FileUtils.mkdir_p(dest)
            open tmp_path, Gem.binary_mode do |io|
                Gem::Package::TarReader.new(io) do |reader|
                    reader.each do |entry|
                        dest_path = File.join(dest, entry.full_name)
                        if entry.directory?
                            FileUtils.mkdir(dest_path)<commit_msg>Avoid GIT_WORK_TREE env variable messing with git submodule commands<commit_after>require 'tempfile'
require 'fileutils'

module Spider

    module AppManager
        
        def self.install_or_update(apps, options={})
            require 'spiderfw/home'
            home = Spider::Home.new(Dir.pwd)
            installed = {}
            Spider.init_base
            active = Spider.config.get('apps')
            Spider.home.apps.each do |app, info|
                installed[app] = {
                    :active => active.include?(app)
                }
                if spec = info[:spec]
                    installed[app].merge!({
                        :version => spec.version
                    })                        
                end
            end
            to_inst = apps.select{ |a| !installed[a] }
            to_upd = apps.select{ |a| installed[a] }
            install_apps(to_inst, options)
            update_apps(to_upd, options)
            return {:installed => to_inst, :updated => to_upd}
        end
        
        def self.install_apps(apps, options={})
            return if apps.empty?
            require 'spiderfw/setup/app_server_client'
            use_git = false
            unless options[:no_git]
                begin
                    require 'git'
                    use_git = true
                rescue => exc
                    puts exc.message
                    puts "git gem not available; install git gem for Git support"
                end
            end
            
            existent = []
            apps.each do |app|
                if File.exist?("apps/#{app}")
                    puts _("%s already exists, skipping") % app
                    existent << app
                end
            end
            require 'spiderfw/setup/app_manager'
            specs = []
            url = options[:url]
            unless url
                require 'spiderfw/spider'
                Spider.init_base
                url = Spider.config.get('app_server.url')
            end
            client = Spider::AppServerClient.new(url)
            if options[:no_deps]
                specs = client.get_specs(apps)
            else
                specs = client.get_deps(apps, :no_optional => options[:no_optional])
            end
            deps = specs.map{ |s| s.app_id }
            unless (deps - apps).empty?
                puts _("The following apps will be installed as a dependency:")
                puts (deps - apps).inspect
            end
            i_options = {
                :use_git => use_git, 
                :no_gems => options[:no_gems],
                :no_optional_gems => options[:no_optional_gems]
            }
            i_options[:ssh_user] = options[:ssh_user] if options[:ssh_user]
            inst_specs = specs.reject{ |s| existent.include? s.app_id }
            Spider::AppManager.install(inst_specs, Dir.pwd, i_options)
            unless options[:no_activate]
                require 'spiderfw/spider'
                specs_hash = {}
                specs.each{ |s| specs_hash[s.app_id] = s }
                Spider.activate_apps(deps, specs_hash)
            end
            
        end
        
        def self.update_apps(apps, options={})
            require 'spiderfw/spider'
            require 'spiderfw/setup/app_server_client'
            Spider.init_base
            url = options[:url] || Spider.conf.get('app_server.url')
            use_git = false
            unless options[:no_git]
                begin
                    require 'git'
                    use_git = true
                rescue
                    puts "git gem not available; install git gem for Git support"
                end
            end
            if options[:all]
                require 'spiderfw/home'
                home = Spider::Home.new(Dir.pwd)
                apps = home.list_apps
            end
            if apps.empty?
                puts _("No app to update")
                exit
            end
            require 'spiderfw/setup/app_manager'
            specs = []
            client = Spider::AppServerClient.new(url)
            if options[:no_deps]
                specs = client.get_specs(apps)
            else
                specs = client.get_deps(apps, :no_optional => options[:no_optional])
            end
            deps = specs.map{ |s| s.app_id }
            unless (deps - apps).empty?
                puts _("The following apps will be updated as a dependency:")
                puts (deps - apps).inspect
            end
            Spider::AppManager.update(specs, Dir.pwd, {
                :use_git => use_git, 
                :no_gems => options[:no_gems],
                :no_optional_gems => options[:no_optional_gems]
            })
        end

        def self.install(specs, home_path, options)
            options[:use_git] = true unless options[:use_git] == false
            options[:home_path] = home_path
            specs = [specs] if specs && !specs.is_a?(Array)
            specs ||= []
            pre_setup(specs, options)
            specs.each do |spec|
                if spec.git_repo && options[:use_git]
                    git_install(spec, home_path, options)
                else
                    pack_install(spec, home_path, options)
                end
            end
            post_setup(specs, options)
        end

        def self.git_install(spec, home_path, options={})
            require 'git'
            if ::File.exist?("apps/#{spec.id}")
                puts _("%s already installed, skipping") % spec.id
                return
            end
            repo = Git.open(home_path)
            puts _("Fetching %s from %s") % [spec.app_id, spec.git_repo]
            repo_url = spec.git_repo
            if options[:ssh_user] && repo_url =~ /ssh:\/\/([^@]+@)?(.+)/
                repo_url = "ssh://#{options[:ssh_user]}@#{$2}"
            end
            
            ENV['GIT_WORK_TREE'] = nil
            Dir.chdir(home_path) do
                `git submodule add #{repo_url} apps/#{spec.id}`
                `git submodule init`
                `git submodule update`
            end
            repo.add(['.gitmodules', "apps/#{spec.id}"])
            repo.commit(_("Added app %s") % spec.id) 
        end

        def self.pack_install(spec, home_path, options={})
            require 'rubygems/package'
            client = AppServerClient.new(spec.app_server)
            print _("Fetching %s from server... ") % spec.app_id
            tmp_path = client.fetch_app(spec.app_id)
            puts _("Fetched.")
            dest = File.join(home_path, "apps/#{spec.app_id}")
            FileUtils.mkdir_p(dest)
            open tmp_path, Gem.binary_mode do |io|
                Gem::Package::TarReader.new(io) do |reader|
                    reader.each do |entry|
                        dest_path = File.join(dest, entry.full_name)
                        if entry.directory?
                            FileUtils.mkdir(dest_path)
{'fraction_non_alphanumeric': 0.09997141632128055, 'fraction_numerical': 0.00014291839359725596, 'mean_word_length': 1.2837793733681462, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2470495', 'n_tokens_mistral': 3937, 'n_tokens_neox': 3750, 'n_words': 825}