<commit_before>

module Etsource


  class Topology

    def initialize(etsource = Etsource::Base.instance)
      @etsource = etsource
    end

    def graph(country = 'nl')
      @graph ||= import
      @graph.dataset = Etsource::Dataset.new(country).import
      @graph
    end

    def import
      converters = {}
      converter_groups = Hash.new { |hash, key| hash[key] = [] }

      # Load the converter groups from the topology/groups/*.yml
      # creating a in this format:
      # {converter_1: [group_1, group_2], ...}
      Dir.glob("#{base_dir}/groups/*.yml").each do |file|
        group_name = File.basename(file, '.yml').to_sym
        items = YAML::load_file(file)
        items.each do |key|
          converter_groups[key.to_sym] << group_name
        end
      end

      # Loads converters and puts into converters hash. Attaches
      # the previously loaded groups.
      topology_hash.each_pair do |key, attrs|
        # Initialize all converters first, before we map slots and links to them.
        converter_key = key.to_sym

        converter = Qernel::Converter.new({
          key:                  converter_key,
          sector_id:            attrs['sector'].try(:to_sym),
          use_id:               attrs['use'].try(:to_sym),
          energy_balance_group: attrs['energy_balance_group'].try(:to_sym),
          groups:               converter_groups[converter_key]
        })
        converters[converter_key] = converter
      end

      # Connect converters with
      graph = Qernel::Graph.new(converters.values)

      # The new export.graph uses yaml. The old format was parsed line by line,
      # now we must parse the entire structure. A slot object can be built from
      # a link and a slot line, so we merge them, remove duplicates and create
      # the slots as needed
      slot_tokens = []
      topology_hash.each_pair do |c_key, values|
        slot_lines = ((values['slots'] || []) + (values['links'] || []))
        slot_tokens << slot_lines.map{|line| SlotToken.find(line)}.flatten
      end

      slot_tokens.flatten.uniq_by{|t| t.key.strip}.each do |token|
        converter = converters[token.converter_key.to_sym]
        slot = Qernel::Slot.new(token.key, converter, carrier(token), token.direction)
        converter.add_slot(slot) # DEBT: after removing of Blueprint::Models we can simplify this
      end

      topology_hash.each_pair do |converter_key, values|
        (values['links'] || []).each do |line|
          link = LinkToken.find(line)
          next unless link.is_a?(LinkToken)
          link = Qernel::Link.new(link.key,
                                  converters[link.input_key],
                                  converters[link.output_key],
                                  carrier(link),
                                  link.link_type )
        end
      end

      graph.assign_graph_to_qernel_objects
      graph
    end

    def carrier(obj)
      key = obj.respond_to?(:carrier_key) ? obj.carrier_key : obj
      @carriers ||= {}
      @carriers[key] ||= Qernel::Carrier.new(key: key)
    end

  protected

    def topology_hash
      @topology_hash ||= YAML::load(File.read(topology_file))
    end

    def topology_file
      "#{base_dir}/export.graph"
    end

    # working copy
    def base_dir
      "#{@etsource.export_dir}/topology"
    end

    # export, read-only dir
    def export_dir
      "#{@etsource.export_dir}/topology"
    end
  end

  # Extract keys from a Slot Topology String
  #    Token.new("FOO-(HW) -- s --> (HW)-BAR")
  #    => <Token carrier_key:HW, output_key:BAR, input_key:FOO, link_type: :share>
  #
  class LinkToken
    attr_reader :input_key, :carrier_key, :output_key, :key, :link_type

    LINK_TYPES = {
      's' => :share,
      'f' => :flexible,
      'i' => :inversed_flexible,
      'd' => :dependent,
      'c' => :constant
    }

    def initialize(line)
      line.gsub!(/#.+/, '')
      line.strip!
      line.gsub!(/\s+/,'')
      @key = line

      input, output = SlotToken.find(line)

      Rails.logger.warn("No Slots '#{line}'") if input.nil? or output.nil?
      Rails.logger.warn("Carriers do not match in '#{line}'") if input.carrier_key != output.carrier_key
      @carrier_key = input.carrier_key
      @input_key   = input.converter_key
      @output_key  = output.converter_key

      @link_type   = LINK_TYPES[line.gsub(/\s+/,'').scan(/--(\w)-->/).flatten.first]
    end

    def self.find(line)
      if line.gsub(/\s/,'') =~ /\)--\w-->\(/
        new(line)
      else
        []
      end
    end
  end

  # Extract keys from a Slot Topology String
  #    Token.new("(HW)-FOO")
  #    t.converter_key # => :FOO
  #    t.carrier_key # => :HW<commit_msg>Etsource::Topology includes reversed links again.<commit_after>

module Etsource


  class Topology

    def initialize(etsource = Etsource::Base.instance)
      @etsource = etsource
    end

    def graph(country = 'nl')
      @graph ||= import
      @graph.dataset = Etsource::Dataset.new(country).import
      @graph
    end

    def import
      converters = {}
      converter_groups = Hash.new { |hash, key| hash[key] = [] }

      # Load the converter groups from the topology/groups/*.yml
      # creating a in this format:
      # {converter_1: [group_1, group_2], ...}
      Dir.glob("#{base_dir}/groups/*.yml").each do |file|
        group_name = File.basename(file, '.yml').to_sym
        items = YAML::load_file(file)
        items.each do |key|
          converter_groups[key.to_sym] << group_name
        end
      end

      # Loads converters and puts into converters hash. Attaches
      # the previously loaded groups.
      topology_hash.each_pair do |key, attrs|
        # Initialize all converters first, before we map slots and links to them.
        converter_key = key.to_sym

        converter = Qernel::Converter.new({
          key:                  converter_key,
          sector_id:            attrs['sector'].try(:to_sym),
          use_id:               attrs['use'].try(:to_sym),
          energy_balance_group: attrs['energy_balance_group'].try(:to_sym),
          groups:               converter_groups[converter_key]
        })
        converters[converter_key] = converter
      end

      # Connect converters with
      graph = Qernel::Graph.new(converters.values)

      # The new export.graph uses yaml. The old format was parsed line by line,
      # now we must parse the entire structure. A slot object can be built from
      # a link and a slot line, so we merge them, remove duplicates and create
      # the slots as needed
      slot_tokens = []
      topology_hash.each_pair do |c_key, values|
        slot_lines = ((values['slots'] || []) + (values['links'] || []))
        slot_tokens << slot_lines.map{|line| SlotToken.find(line)}.flatten
      end

      slot_tokens.flatten.uniq_by{|t| t.key.strip}.each do |token|
        converter = converters[token.converter_key.to_sym]
        slot = Qernel::Slot.new(token.key, converter, carrier(token), token.direction)
        converter.add_slot(slot) # DEBT: after removing of Blueprint::Models we can simplify this
      end

      topology_hash.each_pair do |converter_key, values|
        (values['links'] || []).each do |line|
          link = LinkToken.find(line)
          next unless link.is_a?(LinkToken)
          link = Qernel::Link.new(link.key,
                                  converters[link.input_key],
                                  converters[link.output_key],
                                  carrier(link),
                                  link.link_type,
                                  link.reversed )
        end
      end

      graph.assign_graph_to_qernel_objects
      graph
    end

    def carrier(obj)
      key = obj.respond_to?(:carrier_key) ? obj.carrier_key : obj
      @carriers ||= {}
      @carriers[key] ||= Qernel::Carrier.new(key: key)
    end

  protected

    def topology_hash
      @topology_hash ||= YAML::load(File.read(topology_file))
    end

    def topology_file
      "#{base_dir}/export.graph"
    end

    # working copy
    def base_dir
      "#{@etsource.export_dir}/topology"
    end

    # export, read-only dir
    def export_dir
      "#{@etsource.export_dir}/topology"
    end
  end

  # Extract keys from a Slot Topology String
  #    Token.new("FOO-(HW) -- s --> (HW)-BAR")
  #    => <Token carrier_key:HW, output_key:BAR, input_key:FOO, link_type: :share>
  #
  class LinkToken
    attr_reader :input_key, :carrier_key, :output_key, :key, :link_type, :reversed

    LINK_TYPES = {
      's' => :share,
      'f' => :flexible,
      'i' => :inversed_flexible,
      'd' => :dependent,
      'c' => :constant
    }

    def initialize(line)
      line.gsub!(/#.+/, '')
      line.strip!
      line.gsub!(/\s+/,'')
      @key = line

      input, output = SlotToken.find(line)

      Rails.logger.warn("No Slots '#{line}'") if input.nil? or output.nil?
      Rails.logger.warn("Carriers do not match in '#{line}'") if input.carrier_key != output.carrier_key
      @carrier_key = input.carrier_key
      @input_key   = input.converter_key
      @output_key  = output.converter_key
      @reversed    = line.include?('<')
      @link_type   = LINK_TYPES[line.gsub(/\s+/,'').scan(/--(\w)--[<>]/).flatten.first]
    end

    def self.find(line)
      if line.gsub(/\s/,'') =~ /\)--\w--[<>]\(/
        new(line)
      else
        []
      end
    end
  end

  # Extract keys from a Slot Topology String
  #    Token.new("(HW)-FOO")
  #    t.converter_key # => :FOO
  #    t.carrier_key # => :HW
{'fraction_non_alphanumeric': 0.1266325357851844, 'fraction_numerical': 0.0006268937415108139, 'mean_word_length': 2.145580019717384, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2112003', 'n_tokens_mistral': 3141, 'n_tokens_neox': 3026, 'n_words': 683}