<commit_before>module Turbine
  module Traversal
    # Provides the means for traversing through the graph.
    #
    # Traversal classes do not themselves provide the methods commonly used
    # for iterating through collections (each, map, etc), but act as a
    # generator for the values in an Enumerator.
    #
    #   enumerator = DepthFirst.new(node, :in).to_enum
    #   # => #<Enumerator: Node, Node, ...>
    #
    #   enumerator.each { |node| ... }
    #   enumerator.map  { |node| ... }
    #   # etc ...
    #
    # The Base class should not be used directly, but instead you should use
    # DepthFirst or BreadthFirst which define strategies for the order in
    # which items are traversed.
    #
    # Each unique item is traversed a maximum of once (loops are not
    # repeatedly followed).
    #
    # Traversals are normally used to iterate through nodes, however you may
    # also use them to traverse edges by providing a +fetcher+ argument which
    # tells the traversal how to reach the next set of adjacent items:
    #
    #   DepthFirst.new(node, :in_edges, [], :out).to_enum
    #   # => #<Enumerator: Edge, Edge, ...>
    #
    # As an end-user, you should rarely have to instantiate a traversal class
    # yourself; Node#ancestors and Node#descendants provide a more convenient
    # short-cut.
    class Base
      # Used to match exceptions raise when trying to resume a dead fiber.
      DEAD_FIBER_MESSAGE = 'dead fiber called'.freeze

      # Creates a new graph traversal.
      #
      # start   - The node from which to start traversing.
      # method  - The method to be used to fetch the adjacent nodes (typically
      #           +in+ or +out+).
      # args    - Additional arguments to be used when calling +method+.
      # fetcher - An optional method name to be called on each adjacent item
      #           in order to fetch *its* adjacent items. Useful if traversing
      #           edges instead of nodes.
      #
      # Returns a new traversal.
      def initialize(start, method, args = nil, fetcher = nil)
        @start   = start
        @method  = method
        @args    = args
        @fetcher = fetcher
      end

      # Public: A human-readable version of the traversal.
      #
      # Returns a string.
      def inspect
        "#<#{ self.class.name } start=#{ @start.inspect } " \
          "method=#{ @method.inspect }" \
          "#{ @fetcher ? " fetcher=#{ @fetcher.inspect }" : '' }>"
      end

      # Public: The next node in the traversal.
      #
      # Raises a StopIteration if all reachable nodes have been visited.
      #
      # For example
      #
      #   traversal.next # => #<Turbine::Node key=:one>
      #   traversal.next # => #<Turbine::Node key=:two>
      #   traversal.next # => ! StopIteration
      #
      # Returns a Node.
      def next
        @fiber.resume
      rescue FiberError => ex
        raise (ex.message == DEAD_FIBER_MESSAGE) ? StopIteration : ex
      end

      # Public: The traversal as an enumerator. This is the main way to
      # traverse since the enumerator implements +each+, +map+, +with_index+,
      # etc.
      #
      # Returns an Enumerator.
      def to_enum
        Enumerator.new do |control|
          rewind
          loop { control.yield(self.next) }
        end
      end
<commit_msg>We no longer need to worry about dead fibers.<commit_after>module Turbine
  module Traversal
    # Provides the means for traversing through the graph.
    #
    # Traversal classes do not themselves provide the methods commonly used
    # for iterating through collections (each, map, etc), but act as a
    # generator for the values in an Enumerator.
    #
    #   enumerator = DepthFirst.new(node, :in).to_enum
    #   # => #<Enumerator: Node, Node, ...>
    #
    #   enumerator.each { |node| ... }
    #   enumerator.map  { |node| ... }
    #   # etc ...
    #
    # The Base class should not be used directly, but instead you should use
    # DepthFirst or BreadthFirst which define strategies for the order in
    # which items are traversed.
    #
    # Each unique item is traversed a maximum of once (loops are not
    # repeatedly followed).
    #
    # Traversals are normally used to iterate through nodes, however you may
    # also use them to traverse edges by providing a +fetcher+ argument which
    # tells the traversal how to reach the next set of adjacent items:
    #
    #   DepthFirst.new(node, :in_edges, [], :out).to_enum
    #   # => #<Enumerator: Edge, Edge, ...>
    #
    # As an end-user, you should rarely have to instantiate a traversal class
    # yourself; Node#ancestors and Node#descendants provide a more convenient
    # short-cut.
    class Base
      # Creates a new graph traversal.
      #
      # start   - The node from which to start traversing.
      # method  - The method to be used to fetch the adjacent nodes (typically
      #           +in+ or +out+).
      # args    - Additional arguments to be used when calling +method+.
      # fetcher - An optional method name to be called on each adjacent item
      #           in order to fetch *its* adjacent items. Useful if traversing
      #           edges instead of nodes.
      #
      # Returns a new traversal.
      def initialize(start, method, args = nil, fetcher = nil)
        @start   = start
        @method  = method
        @args    = args
        @fetcher = fetcher
      end

      # Public: A human-readable version of the traversal.
      #
      # Returns a string.
      def inspect
        "#<#{ self.class.name } start=#{ @start.inspect } " \
          "method=#{ @method.inspect }" \
          "#{ @fetcher ? " fetcher=#{ @fetcher.inspect }" : '' }>"
      end

      # Public: The next node in the traversal.
      #
      # Raises a StopIteration if all reachable nodes have been visited.
      #
      # For example
      #
      #   traversal.next # => #<Turbine::Node key=:one>
      #   traversal.next # => #<Turbine::Node key=:two>
      #   traversal.next # => ! StopIteration
      #
      # Returns a Node.
      def next
        @fiber.resume
      end

      # Public: The traversal as an enumerator. This is the main way to
      # traverse since the enumerator implements +each+, +map+, +with_index+,
      # etc.
      #
      # Returns an Enumerator.
      def to_enum
        Enumerator.new do |control|
          rewind
          loop { control.yield(self.next) }
        end
      end

{'fraction_non_alphanumeric': 0.09135955404149891, 'fraction_numerical': 0.0, 'mean_word_length': 2.0524574669187143, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4201686', 'n_tokens_mistral': 1876, 'n_tokens_neox': 1855, 'n_words': 688}