<commit_before>module CloudCrowd
  
  # A Node is a Sinatra/Thin application that runs a single instance per-machine
  # It registers with the central server, receives WorkUnits, and forks off 
  # Workers to process them. The actions are:
  #
  # [get /heartbeat] Returns 200 OK to let monitoring tools know the server's up.
  # [post /work] The central server hits <tt>/work</tt> to dispatch a WorkUnit to this Node.
  class Node < Sinatra::Default
    
    # A Node's default port. You only run a single node per machine, so they
    # can all use the same port without any problems.
    DEFAULT_PORT        = 9063
    
    # A list of regex scrapers, which let us extract the one-minute load 
    # average and the amount of free memory on different flavors of UNIX.
    
    SCRAPE_UPTIME       = /\d+\.\d+/
    SCRAPE_LINUX_MEMORY = /MemFree:\s+(\d+) kB/
    SCRAPE_MAC_MEMORY   = /Pages free:\s+(\d+)./   
    SCRAPE_MAC_PAGE     = /page size of (\d+) bytes/
    
    # The interval at which the node monitors the machine's load and memory use
    # (if configured to do so in config.yml).
    MONITOR_INTERVAL    = 3
    
    # The response sent back when this node is overloaded.
    OVERLOADED_MESSAGE  = 'Node Overloaded'
    
    attr_reader :enabled_actions, :host, :port, :central
            
    set :root, ROOT
    set :authorization_realm, "CloudCrowd"
    
    helpers Helpers
    
    # methodoverride allows the _method param.
    enable :methodoverride
    
    # Enabling HTTP Authentication turns it on for all requests.
    # This works the same way as in the central CloudCrowd::Server.
    before do
      login_required if CloudCrowd.config[:http_authentication]
    end
    
    # To monitor a Node with Monit, God, Nagios, or another tool, you can hit 
    # /heartbeat to make sure its still online.
    get '/heartbeat' do
      "buh-bump"
    end
    
    # Posts a WorkUnit to this Node. Forks a Worker and returns the process id.
    # Returns a 503 if this Node is overloaded.
    post '/work' do
      throw :halt, [503, OVERLOADED_MESSAGE] if @overloaded
      pid = fork { Worker.new(self, JSON.parse(params[:work_unit])).run }
      Process.detach(pid)
      json :pid => pid
    end
    
    # When creating a node, specify the port it should run on.
    def initialize(port=nil, daemon=false)
      require 'json'
      @central          = CloudCrowd.central_server
      @host             = Socket.gethostname
      @enabled_actions  = CloudCrowd.actions.keys
      @port             = port || DEFAULT_PORT
      @daemon           = daemon
      @overloaded       = false
      @max_load         = CloudCrowd.config[:max_load]
      @min_memory       = CloudCrowd.config[:min_free_memory]
      start unless test?
    end
    
    # Starting up a Node registers with the central server and begins to listen
    # for incoming WorkUnits.
    def start
      @server          = Thin::Server.new('0.0.0.0', @port, self, :signals => false)
      @server.tag      = 'cloud-crowd-node'
      @server.pid_file = CloudCrowd.pid_path('node.pid')
      @server.log_file = CloudCrowd.log_path('node.log')
      @server.daemonize if @daemon
      trap_signals
      FileUtils.mkdir_p(CloudCrowd.log_path) if @daemon && !File.exists?(CloudCrowd.log_path)
      asset_store
      @server_thread   = Thread.new { @server.start }
      check_in(true)
      monitor_system if @max_load || @min_memory
      @server_thread.join
    end
    
    # Checking in with the central server informs it of the location and 
    # configuration of this Node. If it can't check-in, there's no point in 
    # starting.
    def check_in(critical=false)
      @central["/node/#{@host}"].put(
        :port             => @port,
        :busy             => @overloaded,
        :max_workers      => CloudCrowd.config[:max_workers],
        :enabled_actions  => @enabled_actions.join(',')
      )
    rescue Errno::ECONNREFUSED
      puts "Failed to connect to the central server (#{@central.to_s})."
      raise SystemExit if critical
    end
    
    # Before exiting, the Node checks out with the central server, releasing all
    # of its WorkUnits for other Nodes to handle
    def check_out
      @central["/node/#{@host}"].delete
    end
    <commit_msg>moving log_path creation up (thanks Ant)<commit_after>module CloudCrowd
  
  # A Node is a Sinatra/Thin application that runs a single instance per-machine
  # It registers with the central server, receives WorkUnits, and forks off 
  # Workers to process them. The actions are:
  #
  # [get /heartbeat] Returns 200 OK to let monitoring tools know the server's up.
  # [post /work] The central server hits <tt>/work</tt> to dispatch a WorkUnit to this Node.
  class Node < Sinatra::Default
    
    # A Node's default port. You only run a single node per machine, so they
    # can all use the same port without any problems.
    DEFAULT_PORT        = 9063
    
    # A list of regex scrapers, which let us extract the one-minute load 
    # average and the amount of free memory on different flavors of UNIX.
    
    SCRAPE_UPTIME       = /\d+\.\d+/
    SCRAPE_LINUX_MEMORY = /MemFree:\s+(\d+) kB/
    SCRAPE_MAC_MEMORY   = /Pages free:\s+(\d+)./   
    SCRAPE_MAC_PAGE     = /page size of (\d+) bytes/
    
    # The interval at which the node monitors the machine's load and memory use
    # (if configured to do so in config.yml).
    MONITOR_INTERVAL    = 3
    
    # The response sent back when this node is overloaded.
    OVERLOADED_MESSAGE  = 'Node Overloaded'
    
    attr_reader :enabled_actions, :host, :port, :central
            
    set :root, ROOT
    set :authorization_realm, "CloudCrowd"
    
    helpers Helpers
    
    # methodoverride allows the _method param.
    enable :methodoverride
    
    # Enabling HTTP Authentication turns it on for all requests.
    # This works the same way as in the central CloudCrowd::Server.
    before do
      login_required if CloudCrowd.config[:http_authentication]
    end
    
    # To monitor a Node with Monit, God, Nagios, or another tool, you can hit 
    # /heartbeat to make sure its still online.
    get '/heartbeat' do
      "buh-bump"
    end
    
    # Posts a WorkUnit to this Node. Forks a Worker and returns the process id.
    # Returns a 503 if this Node is overloaded.
    post '/work' do
      throw :halt, [503, OVERLOADED_MESSAGE] if @overloaded
      pid = fork { Worker.new(self, JSON.parse(params[:work_unit])).run }
      Process.detach(pid)
      json :pid => pid
    end
    
    # When creating a node, specify the port it should run on.
    def initialize(port=nil, daemon=false)
      require 'json'
      @central          = CloudCrowd.central_server
      @host             = Socket.gethostname
      @enabled_actions  = CloudCrowd.actions.keys
      @port             = port || DEFAULT_PORT
      @daemon           = daemon
      @overloaded       = false
      @max_load         = CloudCrowd.config[:max_load]
      @min_memory       = CloudCrowd.config[:min_free_memory]
      start unless test?
    end
    
    # Starting up a Node registers with the central server and begins to listen
    # for incoming WorkUnits.
    def start
      FileUtils.mkdir_p(CloudCrowd.log_path) if @daemon && !File.exists?(CloudCrowd.log_path)
      @server          = Thin::Server.new('0.0.0.0', @port, self, :signals => false)
      @server.tag      = 'cloud-crowd-node'
      @server.pid_file = CloudCrowd.pid_path('node.pid')
      @server.log_file = CloudCrowd.log_path('node.log')
      @server.daemonize if @daemon
      trap_signals
      asset_store
      @server_thread   = Thread.new { @server.start }
      check_in(true)
      monitor_system if @max_load || @min_memory
      @server_thread.join
    end
    
    # Checking in with the central server informs it of the location and 
    # configuration of this Node. If it can't check-in, there's no point in 
    # starting.
    def check_in(critical=false)
      @central["/node/#{@host}"].put(
        :port             => @port,
        :busy             => @overloaded,
        :max_workers      => CloudCrowd.config[:max_workers],
        :enabled_actions  => @enabled_actions.join(',')
      )
    rescue Errno::ECONNREFUSED
      puts "Failed to connect to the central server (#{@central.to_s})."
      raise SystemExit if critical
    end
    
    # Before exiting, the Node checks out with the central server, releasing all
    # of its WorkUnits for other Nodes to handle
    def check_out
      @central["/node/#{@host}"].delete
    end
    
{'fraction_non_alphanumeric': 0.08657099521642748, 'fraction_numerical': 0.004200210010500525, 'mean_word_length': 2.382794001578532, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6165164', 'n_tokens_mistral': 2680, 'n_tokens_neox': 2602, 'n_words': 869}