<commit_before>namespace :import do
  OUT_SLOT_RE   = /^\((?<carrier>[\w_]+)\)-(?<node>[\w_]+)(?:$|: )/
  IN_SLOT_RE    = /^(?<node>[\w_]+)-\((?<carrier>[\w_]+)\)(?:$|: )/
  SLOT_DEFAULTS = Tome::Slot.new(key: 'a-@e').attributes

  # Internal: Given a slot string, returns two values: the key of the slot as
  # a symbol, and the data for the slot.
  def parse_slot(string)
    key, data = string.split(': ', 2)
    data = (data.nil? ? {} : eval(data)).with_indifferent_access 

    data[:key] = if match = IN_SLOT_RE.match(key)
      Tome::Slot.key(match[:node], :in, match[:carrier])
    elsif match = OUT_SLOT_RE.match(key)
      Tome::Slot.key(match[:node], :out, match[:carrier])
    end

    data[:path] = "#{ data[:key].to_s.split('_', 2).first }/#{ data[:key] }"

    return [key, data]
  end

  # Internal: Reads the old NL dataset, extracting slots and slot conversions
  # to be imported as new Slot documents. Any slot whose only attribute is
  # "conversion=1.0" will be omitted since Tome/Refinery will assume this as
  # the default anyway.
  #
  # Regional data looks like:
  #   (useable_heat)-agriculture_burner_crude_oil: {...}  # Hash key and value
  #
  # Global data looks like:
  #   "(loss)-agriculture_burner_crude_oil: {...}"  # String
  # 
  # Returns a hash of slot data.
  def nl_slots
    slots = {}.with_indifferent_access

    # Start by reading the Dutch dataset, extracting information about the
    # slots there.

    Dir.glob($from_dir.join('datasets/nl/graph/**.yml')).each do |file|
      YAML.load_file(file).each do |key, properties|
        if key.match(IN_SLOT_RE) || key.match(OUT_SLOT_RE)
          slots[key] = properties
        end
      end
    end

    # Now we parse the node data, extracting the global information about
    # slots.

    nodes = YAML.load_file($from_dir.join('topology/export.graph'))

    nodes.map { |_, node| node['slots'] || [] }.flatten.each do |string|
      key, data = parse_slot(string)

      slot = slots[key]

      # Merge in the NL regional data.
      slot.merge!(data)

      # Change the old "conversion" attribute to "share".
      slot[:share] = slot.delete(:conversion) if slot.key?(:conversion)
    end

    slots
  end

  # Internal: Checks if the attributes for a slot are identical to the
  # defaults, in which case we don't need to add an explicit slot.
  def all_slot_defaults?(attributes)
    without_path = attributes.dup
    without_path.delete(:path)
    without_path.delete(:key)

    without_path.all? do |key, value|
      value.nil? || SLOT_DEFAULTS[key.to_sym] == value
    end
  end

  # --------------------------------------------------------------------------

  desc <<-DESC
    Import slots from the old format to ActiveDocument.

    This starts by *deleting* everything in data/nodes on the assumption that
    there are no hand-made changes. Then the global slot data (from
    export/topology) is merged with data from the NL dataset to complete the
    slot information.
  DESC
  task :slots, [:from, :to] => [:setup] do |_, args|
    include Tome

    # runner = ImportRun.new('slots')
    slots  = nl_slots.values

    reporter = Tome::Term::Reporter.new(
      'Importing slots', imported: :green, skipped: :yellow)

    # We need to select all the slots whose values differ from the defaults.
    inputs, outputs = slots.partition { |data| data[:key].to_s.include?('+') }
    skip, use       = outputs.partition { |data| all_slot_defaults?(data) }

    skip.push(*inputs)

    # Now we make a list of all nodes which had a slot added, so that we can
    # add *all* of the slots for the side (in or out) of that node.
    used_sides = use.map do |data|
      data[:key].to_s.split('@').first
    end.uniq

    skip.each do |data|
      if used_sides.include?(data[:key].to_s.split('@').first)
        use.push(data)
      end
    end

    grouped = use.group_by do |data|
      data[:key].to_s.split(/[-+]@/, 2)[0].to_sym
    end

    reporter.report do |reporter|
      grouped.each do |node_key, slots|
        node  = Node.find(node_key)
        slots = slots.sort_by { |s| s[:key] }

        node_outputs = slots.each_with_object(Hash.new) do |data, collection|
          collection[data[:key].to_s.split('@').last] =
            data[:type] == :loss ? :elastic : data[:share]
        end

        if node_outputs.one? && node_outputs.first.last == 1.0
          # If the node has only a single output slot, and it has the default
          # share of 1.0, don't bother saving anything.
          puts ""
          puts "SKIP: #{ node_outputs.inspect }"
          node.output = {}
          reporter.inc(:skipped)
        else
          node_outputs.length.times { reporter.inc(:imported) }
        end

        node.save(false)
      end

      # Show how many were skipped.
      (slots.length - use.length).times { reporter.inc(:skipped) }
    end

    puts
  end # slots
end # import<commit_msg>SlotImporter: Specify some input slots for import<commit_after>namespace :import do
  OUT_SLOT_RE   = /^\((?<carrier>[\w_]+)\)-(?<node>[\w_]+)(?:$|: )/
  IN_SLOT_RE    = /^(?<node>[\w_]+)-\((?<carrier>[\w_]+)\)(?:$|: )/
  SLOT_DEFAULTS = Tome::Slot.new(key: 'a-@e').attributes

  # Internal: Given a slot string, returns two values: the key of the slot as
  # a symbol, and the data for the slot.
  def parse_slot(string)
    key, data = string.split(': ', 2)
    data = (data.nil? ? {} : eval(data)).with_indifferent_access 

    data[:key] = if match = IN_SLOT_RE.match(key)
      Tome::Slot.key(match[:node], :in, match[:carrier])
    elsif match = OUT_SLOT_RE.match(key)
      Tome::Slot.key(match[:node], :out, match[:carrier])
    end

    data[:path] = "#{ data[:key].to_s.split('_', 2).first }/#{ data[:key] }"

    return [key, data]
  end

  # Internal: Reads the old NL dataset, extracting slots and slot conversions
  # to be imported as new Slot documents. Any slot whose only attribute is
  # "conversion=1.0" will be omitted since Tome/Refinery will assume this as
  # the default anyway.
  #
  # Regional data looks like:
  #   (useable_heat)-agriculture_burner_crude_oil: {...}  # Hash key and value
  #
  # Global data looks like:
  #   "(loss)-agriculture_burner_crude_oil: {...}"  # String
  # 
  # Returns a hash of slot data.
  def nl_slots
    slots = {}.with_indifferent_access

    # Start by reading the Dutch dataset, extracting information about the
    # slots there.

    Dir.glob($from_dir.join('datasets/nl/graph/**.yml')).each do |file|
      YAML.load_file(file).each do |key, properties|
        if key.match(IN_SLOT_RE) || key.match(OUT_SLOT_RE)
          slots[key] = properties
        end
      end
    end

    # Now we parse the node data, extracting the global information about
    # slots.

    nodes = YAML.load_file($from_dir.join('topology/export.graph'))

    nodes.map { |_, node| node['slots'] || [] }.flatten.each do |string|
      key, data = parse_slot(string)

      slot = slots[key]

      # Merge in the NL regional data.
      slot.merge!(data)

      # Change the old "conversion" attribute to "share".
      slot[:share] = slot.delete(:conversion) if slot.key?(:conversion)
    end

    slots
  end

  # Internal: Checks if the attributes for a slot are identical to the
  # defaults, in which case we don't need to add an explicit slot.
  def all_slot_defaults?(attributes)
    without_path = attributes.dup
    without_path.delete(:path)
    without_path.delete(:key)

    without_path.all? do |key, value|
      # The value matches the default, or ...
      (value.nil? || SLOT_DEFAULTS[key.to_sym] == value) ||
        # ... it's a share attribute, and has the default of 1.0.
        (key.to_sym == :share && value == 1.0)
    end
  end

  # --------------------------------------------------------------------------

  desc <<-DESC
    Import slots from the old format to ActiveDocument.

    This starts by *deleting* everything in data/nodes on the assumption that
    there are no hand-made changes. Then the global slot data (from
    export/topology) is merged with data from the NL dataset to complete the
    slot information.
  DESC
  task :slots, [:from, :to] => [:setup] do |_, args|
    include Tome

    slots    = nl_slots.values
    required = YAML.load_file(Tome.data_dir.join('import/required_slots.yml'))

    # We start by building two lists; one containing the slots which we
    # definitely need to import, and one containing those we thing can be
    # skipped.

    use, skip = slots.partition do |data|
      direction = data[:key].to_s.include?('-') ? :output : :input

      # We import *all* output slots whose share is not the default 1.0.
      (direction == :output && ! all_slot_defaults?(data)) ||
        # ... or any slot (including input slots) named in the ETSource
        # data/import/required_inputs_slots.yml file.
        required[direction].include?(data[:key].to_s.split(/[+-]/).first)
    end

    # Next, we create a list of all the nodes which need to have one or more
    # slots defined so that we can add all the input/output slots for that
    # node.

    used_sides = use.map do |data|
      data[:key].to_s.split('@').first
    end.uniq

    skip.each do |data|
      use.push(data) if used_sides.include?(data[:key].to_s.split('@').first)
    end

    # Before proceeding with the import, group all the slots which need to be
    # added by the node key and slot direction so that we can save them all
    # at once.

    grouped = use.group_by do |data|
      data[:key].to_s.split('@', 2).first.to_sym
    end

    Tome::Term::Reporter.new(
      'Importing slots', imported: :green, skipped: :yellow
    ).report do |reporter|
    # reporter.report do |reporter|
      grouped.each do |token, slots|
        # The token is the node key, followed by a + or -
        node_key = token[0..-2]
        setter   = token[-1] == ?- ? :output= : :input=
        node     = Node.find(node_key)

        # Add the slots in a predictable order each time.
        slots = slots.sort_by { |s| s[:key] }

        # Create a hash containing the data to be saved into the node. Loss
        # slots are ignored and set to elastic.
        slot_data = slots.each_with_object({}) do |data, collection|
          collection[data[:key].to_s.split('@').last] =
            data[:type] == :loss ? :elastic : data[:share]
        end

        node.public_send(setter, slot_data)
        node.save(false)

        slot_data.length.times { reporter.inc(:imported) }
      end

      # Show many many were skipped.
      (slots.length - use.length).times { reporter.inc(:skipped) }
    end

    puts
  end # slots
end # import
{'fraction_non_alphanumeric': 0.1419045830202855, 'fraction_numerical': 0.002253944402704733, 'mean_word_length': 2.583109017496635, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6686574', 'n_tokens_mistral': 3501, 'n_tokens_neox': 3388, 'n_words': 1020}