<commit_before>module DCell
  # Manage nodes we're connected to
  class NodeManager
    include Celluloid::ZMQ
    include Enumerable

    finalizer :clear_nodes

    attr_reader :heartbeat_rate, :heartbeat_timeout

    def initialize
      @nodes = {}

      @heartbeat_rate    = 5  # How often to send heartbeats in seconds
      @heartbeat_timeout = 10 # How soon until a lost heartbeat triggers a node partition
    end

    # Return all available nodes in the cluster
    def all
      Directory.all.map do |node_id|
        find node_id
      end
    end

    # Iterate across all available nodes
    def each
      Directory.all.each do |node_id|
        yield find node_id
      end
    end

    # Find a node by its node ID
    def find(id)
      node = @nodes[id]
      return node if node

      addr = Directory[id]
      return unless addr

      if id == DCell.id
        node = DCell.me
      else
        node = Node.new(id, addr)
      end

      @nodes[id] ||= node
      @nodes[id]
    end
    alias_method :[], :find

    def update(id)
      addr = Directory[id]
      return unless addr
      if ( node = @nodes[id] ) and node.alive?
        node.update_address( addr )
      else
        @nodes[id] = Node.new( id, addr )
      end
    end

    def remove(id)
      if @nodes[id]
        @nodes[id].terminate if @nodes[id].alive?
        @nodes.delete(id)  
      end
    end


    def clear_nodes
      Directory.clear
    end

  end
end
<commit_msg>all of this does not prevent non-reachable nodes from crashing. This is an idea around making the node manager aware of the nodes which have crashed and eventually restarting them accordingly (supervising involved\?)<commit_after>module DCell
  # Manage nodes we're connected to
  class NodeManager
    include Celluloid::ZMQ
    include Enumerable

    trap_exit :node_died

    finalizer :clear_nodes

    attr_reader :heartbeat_rate, :heartbeat_timeout

    def initialize
      @nodes = {}

      @heartbeat_rate    = 5  # How often to send heartbeats in seconds
      @heartbeat_timeout = 10 # How soon until a lost heartbeat triggers a node partition
    end

    # Return all available nodes in the cluster
    def all
      Directory.all.map do |node_id|
        find node_id
      end
    end

    # Iterate across all available nodes
    def each
      Directory.all.each do |node_id|
        yield find node_id
      end
    end

    # Find a node by its node ID
    def find(id)
      node = @nodes[id]
      return node if node

      addr = Directory[id]
      return unless addr

      if id == DCell.id
        node = DCell.me
      else
        node = Node.new(id, addr)
        self.link node
      end

      @nodes[id] ||= node
      @nodes[id]
    end
    alias_method :[], :find

    def node_died(node, reason)
      if reason.nil? # wtf?
        # this wtf error seems to come from node socket writes
        # when the socket is not reachable anymore
        Celluloid::logger.debug "wtf?"
        return
      end
      # Handle dead node???
    end

    def update(id)
      addr = Directory[id]
      return unless addr
      if ( node = @nodes[id] ) and node.alive?
        node.update_address( addr )
      else
        @nodes[id] = Node.new( id, addr )
      end
    end

    def remove(id)
      if @nodes[id]
        @nodes[id].terminate if @nodes[id].alive?
        @nodes.delete(id)  
      end
    end


    def clear_nodes
      Directory.clear
    end

  end
end

{'fraction_non_alphanumeric': 0.06992805755395684, 'fraction_numerical': 0.0017266187050359713, 'mean_word_length': 1.9811320754716981, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4860215', 'n_tokens_mistral': 1090, 'n_tokens_neox': 1030, 'n_words': 361}