<commit_before>module Spider; module HTTP
    
    class Server
        
        @supports = {
            :chunked_request => false,
            :ssl => false
        }

        def self.supports?(capability)
            @supports[capability]
        end

        
        def options(opts={})
            defaults = {
                :host => '0.0.0.0',
                :port => 8080
            }
            return defaults.merge(opts)
        end
        
        def start(opts={})
            @options = opts
            Spider.logger.info{ "Routes: \n"+Spider::HomeController.print_app_routes }
            start_server(opts)
        end
        
        def shutdown
            Spider.logger.info("Webserver shutdown");
            shutdown_server            
        end
        
        def request_received
        end
        
        def self.start(server_name, port, options={})
            servers = {
                'webrick'   => :WEBrick,
                'mongrel'   => :Mongrel,
                'thin'      => :Thin
            }
            
            start = lambda{
                $SPIDER_WEB_SERVER = true
                require 'spiderfw'
                require 'spiderfw/controller/http_controller'
                
                port ||= Spider.conf.get('webserver.port')
                server_name ||= Spider.conf.get('http.server')
                pid_file = File.join(Spider.paths[:var], 'run/server.pid')
                puts _("Using webserver %s") % server_name if options[:verbose]
                puts _("Listening on port %s") % port if options[:verbose]
                server = Spider::HTTP.const_get(servers[server_name]).new
                ssl_server = nil
                Spider.startup
                if Spider.conf.get('devel.trace.extended')
                    require 'ruby-debug'
                    require 'spiderfw/utils/monkey/debugger'
                    Debugger.start
                    Debugger.post_mortem
                end
                
                thread = Thread.new do
                    server.start(:port => port, :cgi => options[:cgi])
                end
                $stdout << "Spider server running on port #{port}\n"
                if options[:ssl]
                    options[:ssl_cert] ||= Spider.conf.get('orgs.default.cert')
                    options[:ssl_key] ||= Spider.conf.get('orgs.default.private_key')
                    raise "SSL Certificate not set" unless options[:ssl_cert]
                    raise "SSL Key not set" unless options[:ssl_key]
                    raise "SSL Certificate (#{options[:ssl_cert]}) not found" unless File.file?(options[:ssl_cert])
                    raise "SSL Key (#{options[:ssl_key]}) not found" unless File.file?(options[:ssl_key])
                    ssl_thread = Thread.new do
                        ssl_server = Spider::HTTP.const_get(servers[server_name]).new
                        ssl_server.start(:port => options[:ssl], :ssl => true, 
                            :ssl_cert => options[:ssl_cert], :ssl_private_key => options[:ssl_key])
                    end
                end
                do_shutdown = lambda{
                    Debugger.post_mortem = false
                    # debugger
                    server.shutdown
                    ssl_server.shutdown if ssl_server
                    pid_file = File.join(Spider.paths[:var], 'run/server.pid')
                    begin
                        File.unlink(pid_file)
                    rescue Errno::ENOENT
                    end
                }
                Spider.on_shutdown(&do_shutdown)
                
                thread.join
                ssl_thread.join if ssl_thread
            }
            if options[:daemonize]
                require 'spiderfw'
                require 'spiderfw/utils/fork'
                pid_file = File.join(Spider.paths[:var], 'run/server.pid')
                process_name = (options[:daemonize] == true) ? 'spider-server' : options[:daemonize]
                forked = Spider.fork do
                    File.open(pid_file, 'w') do |f|
                        f.write(Process.pid)
                    end
                    $SPIDER_SCRIPT ||= $0
                    $0 = process_name
                    STDIN.reopen "/dev/null"       # Free file descriptors and
                    STDOUT.reopen "/dev/null", "a" # point them somewhere sensible
                    STDERR.reopen STDOUT           # STDOUT/STDERR should go to a logfile
                    start.call
                end
                Process.detach(forked)
            else
                Spider.init_base
                spawner_started = false
                if Spider.conf.get('webserver.respawn_on_change')
                    Spider.start_loggers
                    begin
                        begin
                            Bundler.require :default, Spider.runmode.to_sym
                        rescue
                        end
                        spawner = Spawner.new({'spawn' => start})
                        spawner.run('spawn')
                        spawner_started = true
                    rescue LoadError => exc
                        raise unless exc.message =~ /fssm/
                        Spider.logger.error("Install 'fssm' gem to enable respawning")
                    end
                end
                unless spawner_started
                    Spider.main_process_startup
                    Spider.startup
                    start.call 
                end
            end
        end
        
    end
    
    class Spawner
        attr_reader :child_pid
        
        def initialize(actions)
            @actions = actions
        end
        
        def run(action=nil)
            @monitor_thread = monitor_fs
            spawn(action)
            @monitor_thread.join
        end
        
        def spawn(action)
            rd, wr = IO.pipe
            if pid = fork
                # Spawner
                Spider.logger.debug("Spawner forked")
                @child_pid = pid

                unless @already_forked
                    Spider.main_process_startup
                    exit_spawner = lambda{ 
                        Spider.logger.debug "Spawner exiting" 
                        Process.kill 'KILL', @monitor_thread[:spawner_child_pid]
                    }
                    Spider.on_main_process_shutdown(&exit_spawner)
                    Spider.main_process_startup
                    $SPIDER_SCRIPT ||= $0
                    $0 = 'spider-spawner'
                end
                @already_forked = true

                wr.close
                @monitor_thread[:spawner_child_pid] = pid
                # TODO
                # msg = rd.read(5)
                # rd.close
                # spawn(msg) unless msg.blank?

            else
                # Child
                $SPIDER_SPAWNED = true
                trap('TERM'){ }
                trap('INT'){ }
                rd.close
                Spider.spawner = wr
                return unless @actions[action]
                @actions[action].call
            end
        end
        
        def monitor_fs
            require 'fssm'
            spawner = self
            action = 'spawn'
            return Thread.new do
                fsm_exclude = ['var', 'tmp']<commit_msg>Removed double main_process_startup<commit_after>module Spider; module HTTP
    
    class Server
        
        @supports = {
            :chunked_request => false,
            :ssl => false
        }

        def self.supports?(capability)
            @supports[capability]
        end

        
        def options(opts={})
            defaults = {
                :host => '0.0.0.0',
                :port => 8080
            }
            return defaults.merge(opts)
        end
        
        def start(opts={})
            @options = opts
            Spider.logger.info{ "Routes: \n"+Spider::HomeController.print_app_routes }
            start_server(opts)
        end
        
        def shutdown
            Spider.logger.info("Webserver shutdown");
            shutdown_server            
        end
        
        def request_received
        end
        
        def self.start(server_name, port, options={})
            servers = {
                'webrick'   => :WEBrick,
                'mongrel'   => :Mongrel,
                'thin'      => :Thin
            }
            
            start = lambda{
                $SPIDER_WEB_SERVER = true
                require 'spiderfw'
                require 'spiderfw/controller/http_controller'
                
                port ||= Spider.conf.get('webserver.port')
                server_name ||= Spider.conf.get('http.server')
                pid_file = File.join(Spider.paths[:var], 'run/server.pid')
                puts _("Using webserver %s") % server_name if options[:verbose]
                puts _("Listening on port %s") % port if options[:verbose]
                server = Spider::HTTP.const_get(servers[server_name]).new
                ssl_server = nil
                Spider.startup
                if Spider.conf.get('devel.trace.extended')
                    require 'ruby-debug'
                    require 'spiderfw/utils/monkey/debugger'
                    Debugger.start
                    Debugger.post_mortem
                end
                
                thread = Thread.new do
                    server.start(:port => port, :cgi => options[:cgi])
                end
                $stdout << "Spider server running on port #{port}\n"
                if options[:ssl]
                    options[:ssl_cert] ||= Spider.conf.get('orgs.default.cert')
                    options[:ssl_key] ||= Spider.conf.get('orgs.default.private_key')
                    raise "SSL Certificate not set" unless options[:ssl_cert]
                    raise "SSL Key not set" unless options[:ssl_key]
                    raise "SSL Certificate (#{options[:ssl_cert]}) not found" unless File.file?(options[:ssl_cert])
                    raise "SSL Key (#{options[:ssl_key]}) not found" unless File.file?(options[:ssl_key])
                    ssl_thread = Thread.new do
                        ssl_server = Spider::HTTP.const_get(servers[server_name]).new
                        ssl_server.start(:port => options[:ssl], :ssl => true, 
                            :ssl_cert => options[:ssl_cert], :ssl_private_key => options[:ssl_key])
                    end
                end
                do_shutdown = lambda{
                    Debugger.post_mortem = false
                    # debugger
                    server.shutdown
                    ssl_server.shutdown if ssl_server
                    pid_file = File.join(Spider.paths[:var], 'run/server.pid')
                    begin
                        File.unlink(pid_file)
                    rescue Errno::ENOENT
                    end
                }
                Spider.on_shutdown(&do_shutdown)
                
                thread.join
                ssl_thread.join if ssl_thread
            }
            if options[:daemonize]
                require 'spiderfw'
                require 'spiderfw/utils/fork'
                pid_file = File.join(Spider.paths[:var], 'run/server.pid')
                process_name = (options[:daemonize] == true) ? 'spider-server' : options[:daemonize]
                forked = Spider.fork do
                    File.open(pid_file, 'w') do |f|
                        f.write(Process.pid)
                    end
                    $SPIDER_SCRIPT ||= $0
                    $0 = process_name
                    STDIN.reopen "/dev/null"       # Free file descriptors and
                    STDOUT.reopen "/dev/null", "a" # point them somewhere sensible
                    STDERR.reopen STDOUT           # STDOUT/STDERR should go to a logfile
                    start.call
                end
                Process.detach(forked)
            else
                Spider.init_base
                spawner_started = false
                if Spider.conf.get('webserver.respawn_on_change')
                    Spider.start_loggers
                    begin
                        begin
                            Bundler.require :default, Spider.runmode.to_sym
                        rescue
                        end
                        spawner = Spawner.new({'spawn' => start})
                        spawner.run('spawn')
                        spawner_started = true
                    rescue LoadError => exc
                        raise unless exc.message =~ /fssm/
                        Spider.logger.error("Install 'fssm' gem to enable respawning")
                    end
                end
                unless spawner_started
                    Spider.main_process_startup
                    Spider.startup
                    start.call 
                end
            end
        end
        
    end
    
    class Spawner
        attr_reader :child_pid
        
        def initialize(actions)
            @actions = actions
        end
        
        def run(action=nil)
            @monitor_thread = monitor_fs
            spawn(action)
            @monitor_thread.join
        end
        
        def spawn(action)
            rd, wr = IO.pipe
            if pid = fork
                # Spawner
                Spider.logger.debug("Spawner forked")
                @child_pid = pid

                unless @already_forked
                    Spider.main_process_startup
                    exit_spawner = lambda{ 
                        Spider.logger.debug "Spawner exiting" 
                        Process.kill 'KILL', @monitor_thread[:spawner_child_pid]
                    }
                    Spider.on_main_process_shutdown(&exit_spawner)
                    $SPIDER_SCRIPT ||= $0
                    $0 = 'spider-spawner'
                end
                @already_forked = true

                wr.close
                @monitor_thread[:spawner_child_pid] = pid
                # TODO
                # msg = rd.read(5)
                # rd.close
                # spawn(msg) unless msg.blank?

            else
                # Child
                $SPIDER_SPAWNED = true
                trap('TERM'){ }
                trap('INT'){ }
                rd.close
                Spider.spawner = wr
                return unless @actions[action]
                @actions[action].call
            end
        end
        
        def monitor_fs
            require 'fssm'
            spawner = self
            action = 'spawn'
            return Thread.new do
                fsm_exclude = ['var', 'tmp']
{'fraction_non_alphanumeric': 0.08301707779886149, 'fraction_numerical': 0.0017619951206288968, 'mean_word_length': 1.0613214136052522, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2199561', 'n_tokens_mistral': 3945, 'n_tokens_neox': 3615, 'n_words': 738}