<commit_before>module ETSource
  module Util

    # Links ------------------------------------------------------------------

    # Extracts data about a link from the raw link string.
    LINK_RE = /
      (?<child>[\w_]+)-          # Child node key
      \([^)]+\)\s                # Carrier key (ignored)
      (?<reversed><)?            # Arrow indicating a reversed link?
      --\s(?<type>\w)\s-->?\s    # Link type and arrow
      \((?<carrier>[^)]+)\)-     # Carrier key
      (?<parent>[\w_]+)          # Parent node key
    /xi

    # Public: Given a path to a directory, yields an array of links for each
    # "links" file in that directory. Does not recurse into subdirectories.
    #
    # directory - Path to a directory in which there are ".links" files.
    #
    # For example:
    #
    #   ETSource::Util.foreach_link(path) do |links, filename|
    #     links # => [ "...", "..." ]
    #   end
    #
    # Returns nothing.
    def self.foreach_link(directory)
      Dir["#{ directory }/*.links"].map do |filename|
        File.foreach(filename) do |line, *rest|
          yield(line.strip, *rest, filename) if line.match(/\S/)
        end
      end

      nil
    end

    # Public: Given an array of nodes, sets up Turbine edges between them in
    # accordance with the +links+ strings held in the nodes.
    #
    # links - An array containing all the link definitions (as strings).
    # nodes - An array containing zero or more Nodes.
    #
    # Returns nothing.
    def self.establish_links!(nodes, links)
      nodes    = Collection.new(nodes)
      carriers = Collection.new(Carrier.all)
      links.each { |link| establish_link(link, nodes, carriers) }

      nil
    end

    # Internal: Given a string defining a link, sets up a Turbine::Edge
    # between the nodes.
    #
    # link     - The raw link definition string.
    # nodes    - All the nodes defined in the graph in a Collection.
    # carriers - A list of all carriers in the dataset, in a Collection
    #<commit_msg>Trying to debug the Semaphore failures.<commit_after>module ETSource
  module Util

    # Links ------------------------------------------------------------------

    # Extracts data about a link from the raw link string.
    LINK_RE = /
      (?<child>[\w_]+)-          # Child node key
      \([^)]+\)\s                # Carrier key (ignored)
      (?<reversed><)?            # Arrow indicating a reversed link?
      --\s(?<type>\w)\s-->?\s    # Link type and arrow
      \((?<carrier>[^)]+)\)-     # Carrier key
      (?<parent>[\w_]+)          # Parent node key
    /xi

    # Public: Given a path to a directory, yields an array of links for each
    # "links" file in that directory. Does not recurse into subdirectories.
    #
    # directory - Path to a directory in which there are ".links" files.
    #
    # For example:
    #
    #   ETSource::Util.foreach_link(path) do |links, filename|
    #     links # => [ "...", "..." ]
    #   end
    #
    # Returns nothing.
    def self.foreach_link(directory)
      Dir["#{ directory }/*.links"].map do |filename|
        File.foreach(filename) do |line, *rest|
          yield(line.strip, *rest, filename) if line.match(/\S/)
        end
      end

      nil
    end

    # Public: Given an array of nodes, sets up Turbine edges between them in
    # accordance with the +links+ strings held in the nodes.
    #
    # links - An array containing all the link definitions (as strings).
    # nodes - An array containing zero or more Nodes.
    #
    # Returns nothing.
    def self.establish_links!(nodes, links)
      nodes    = Collection.new(nodes)
      carriers = Collection.new(Carrier.all)

      # Debugging failures on Semaphore, depite it passing locally...
      if first_carrier = carriers.first
        puts first_carrier.file_path
      else
        puts "No carriers found"
      end

      links.each { |link| establish_link(link, nodes, carriers) }

      nil
    end

    # Internal: Given a string defining a link, sets up a Turbine::Edge
    # between the nodes.
    #
    # link     - The raw link definition string.
    # nodes    - All the nodes defined in the graph in a Collection.
    # carriers - A list of all carriers in the dataset, in a Collection
    #
{'fraction_non_alphanumeric': 0.14070941977918722, 'fraction_numerical': 0.0, 'mean_word_length': 1.87508440243079, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6470558', 'n_tokens_mistral': 1250, 'n_tokens_neox': 1199, 'n_words': 438}