<commit_before>#
# ActiveFacts Compositions, DataVault Compositor.
#
#       Computes a Data Vault schema.
#
# Copyright (c) 2015 <NAME>. Read the LICENSE file.
#
require "activefacts/compositions/relational"

module ActiveFacts
  module Compositions
    class DataVault < Relational
    public
      def self.options
        {
          reference: ['Boolean', "Emit the reference (static) tables as well. Default is to omit them"],
          datestamp: ['String', "Use this data type for date stamps"],
          id: ['String', "Append this to data vault surrogate keys (default VID)"],
          hubname: ['String', "Suffix or pattern for naming hub tables. Include a + to insert the name. Default 'HUB'"],
          linkname: ['String', "Suffix or pattern for naming link tables. Include a + to insert the name. Default 'LINK'"],
          satname: ['String', "Suffix or pattern for naming satellite tables. Include a + to insert the name. Default 'SAT'"],
          refname: ['String', "Suffix or pattern for naming reference tables. Include a + to insert the name. Default '+'"],
        }
      end

      def initialize constellation, name, options = {}
        # Extract recognised options:
        @option_reference = options.delete('reference')
        @option_datestamp = options.delete('datestamp')
        @option_id = ' ' + (options.delete('id') || 'VID')
        @option_hub_name = options.delete('hubname') || 'HUB'
        @option_hub_name.sub!(/^/,'+ ') unless @option_hub_name =~ /\+/
        @option_link_name = options.delete('linkname') || 'LINK'
        @option_link_name.sub!(/^/,'+ ') unless @option_link_name =~ /\+/
        @option_sat_name = options.delete('satname') || 'SAT'
        @option_sat_name.sub!(/^/,'+ ') unless @option_sat_name =~ /\+/
        @option_ref_name = options.delete('refname') || 'SAT'
        @option_ref_name.sub!(/^/,'+ ') unless @option_ref_name =~ /\+/

        super constellation, name, options

        @option_surrogates = true   # Always inject surrogates regardless of superclass
      end

      # We need to find links that need surrogate keys before we inject the surrogates
      def inject_surrogates
        classify_composites

        super
      end

      def composite_is_reference composite
        object_type = composite.mapping.object_type
        object_type.concept.all_concept_annotation.detect{|ca| ca.mapping_annotation == 'static'} or
          !object_type.is_a?(ActiveFacts::Metamodel::EntityType)
      end

      # Data Vaults need a surrogate key on every Hub and Link.
      # Don't add a surrogate on a Reference table!
      def needs_surrogate(composite)
        return false if composite_is_reference(composite)

        # REVISIT: The following is debatable. If the natural primary key is an ok surrogate, should we inject another?
        return true if @non_reference_composites.include?(composite)

        super
      end

      # Change the default extension from our superclass':
      def inject_surrogate composite, extension = @option_id
        super
      end

      def classify_composites
        detect_reference_tables

        trace :datavault, "Classify non-reference tables into hubs and links" do
          # Make an initial determination, then adjust for foreign keys to links afterwards
          @key_structure = {}
          @link_composites, @hub_composites =
            @non_reference_composites.
            sort_by{|c| c.mapping.name}.
            partition do |composite|
              trace :datavault, "Decide whether #{composite.mapping.name} is a link or a hub" do
                @key_structure[composite] =
                  mapped_to =
                  composite_key_structure composite

                # It's a Link if the preferred identifier includes more than non_reference_composite.
                mapped_to.compact.size > 1
              end
            end

          trace :datavault, "Checking for foreign keys that reference links" do
            # Links may never be the target of a foreign key.
            # Any such links must be defined as hubs instead.
            @links_as_hubs = {}
            fk_dependencies_by_target = {}
            fk_dependencies_by_source = {}
            (@hub_composites+@link_composites).
            each do |composite|
              target_composites = enumerate_foreign_keys composite.mapping
              target_composites.each do |target_composite|
                next if @reference_composites.include?(target_composite)
                (fk_dependencies_by_target[target_composite] ||= []) << composite
                (fk_dependencies_by_source[composite] ||= []) << target_composite
              end
            end

            fk_dependencies_by_target.keys.each do |target_composite|
              if @link_composites.delete(target_composite)
                trace :datavault, "Link #{target_composite.inspect} must be a hub because foreign keys reference it"
                @hub_composites << target_composite
                @links_as_hubs[target_composite] = true
              end
            end

            begin
              converted =
                @link_composites.select do |composite|
                  targets = fk_dependencies_by_source[composite]
                  id_targets = composite_key_structure(composite).compact
                  next if targets.size == id_targets.size
                  trace :datavault, "Link #{composite.mapping.name} must be a hub because it has non-identifying FK references"
                  @link_composites.delete(composite)
                  @hub_composites << composite
                  @links_as_hubs[composite] = true
                end
            end while converted.size > 0

          end

          # Note: We may still have hubs whose identifiers contain foreign keys to one or more other hubs.
          # REVISIT: These foreign keys will be deleted so these hubs stand alone,
          # but have been re-instated as new links to the referenced hubs.
        end

        trace :datavault_classification!, "Data Vault classification of composites:" do
          trace :datavault, "Reference: #{@reference_composites.map(&:mapping).map(&:object_type).map(&:name)*', '}"
          trace :datavault, "Hub: #{@hub_composites.map(&:mapping).map(&:object_type).map(&:name)*', '}"
          trace :datavault, "Link: #{@link_composites.map(&:mapping).map(&:object_type).map(&:name)*', '}"
        end
      end

      def detect_reference_tables
        initial_composites = @composition.all_composite.to_a
        @reference_composites, @non_reference_composites =
          initial_composites.partition { |composite| composite_is_reference(composite) }
      end

      def devolve_all
        delete_reference_table_foreign_keys

        # For each hub and link, move each non-identifying member
        # to a new satellite or promote it to a new link.

        @non_reference_composites.
        each do |composite|
          devolve composite
        end

        rename_parents

        unless @option_reference
          if trace :reference_retraction
            # Add a logger so we can trace the resultant retractions:
            @constellation.loggers << proc do |*args|
              trace :reference_retraction, args.inspect
            end
          end

          @reference_composites.each do |rc|
            trace :reference_retraction, "Retracting #{rc.inspect}" do
              rc.retract
            end
          end
          @reference_composites = []

          @constellation.loggers.pop if trace :reference_retraction
        end
      end

      def delete_reference_table_foreign_keys
        trace :datavault, "Delete foreign keys to reference tables" do
          # Delete all foreign keys to reference tables
          @reference_composites.each do |composite|
            composite.all_foreign_key_as_target_composite.each(&:retract)
          end
        end
      end

      def prefer_natural_key building_natural_key, source_composite, target_composite
        return false if building_natural_key && @link_composites.include?(source_composite)
        building_natural_key && @hub_composites.include?(target_composite)
      end

      def composite_key_structure composite
        # We know that composite.mapping.object_type is an EntityType because all ValueType composites are reference tables

        object_type = composite.mapping.object_type
        mapped_to =
          object_type.preferred_identifier.role_sequence.all_role_ref_in_order.map do |role_ref|
            player = role_ref.role.object_type
            next nil if player == object_type && role_ref.role.fact_type.all_role.size == 1 # Unaries.
            candidate = @candidates[player]
            next nil unless candidate
            # Take care of full absorption
            while candidate.full_absorption
              candidate = candidate.full_absorption.composition
            end
            @non_reference_composites.include?(c = candidate.mapping.composite) ? c : nil
          end

        trace :datavault, "Preferred identifier for #{composite.mapping.name} encloses foreign keys to #{mapped_to.inspect}" unless mapped_to.compact.empty?

        number_of_keys = mapped_to.compact.size
        number_of_values = mapped_to.size-number_of_keys
        trace :datavault_classify,
          if number_of_keys > 1
            # Links have more than one FK to a hub in their key
            "Link #{composite.mapping.name} links #{mapped_to.compact.inspect} with #{number_of_values} values"
          elsif number_of_keys == 1 && number_of_values > 0
            # This is a new hub with a composite key - but we will have to eliminate the foreign key to the base hub
            "Augmented Hub #{composite.mapping.name} has a hub link to #{mapped_to.compact[0].inspect} and #{number_of_values} values"
          elsif number_of_keys == 1
            # This is a new hub with a single-part key that references another hub.
            "Dependent Hub #{composite.mapping.name} is identified by another hub: #{mapped_to.compact[0].inspect}"
          else
            "Hub #{composite.mapping.name} has #{mapped_to.size} parts in its key"
          end

        mapped_to
      end

      # For each member of this composite, decide whether to devolve it to a satellite
      # or to a new link. If it goes to a link that's still part of this natural key,
      # we need to leave that key intact, but remove the foreign key it entails.
      #
      # New links and satellites get new fields for the load date-time and a
      # references to the surrogate(s) on the hub or link, and add an index over
      # those two fields.
      def devolve composite, devolve_links = true
        trace :datavault?, "Devolving non-identifying fields for #{composite.inspect}" do
          # Find the members of this mapping that contain identifying leaves:
          pi = composite.primary_index
          ni = composite.natural_index
          identifiers =
            (Array(pi ? pi.all_index_field : nil) +
             Array(ni ? ni.all_index_field : nil)).
            map{|ixf| ixf.component.path[1]}.
            uniq

          satellites = {}
          is_link = @link_composites.include?(composite) || @links_as_hubs.include?(composite)
          composite.mapping.all_member.to_a.each do |member|

            # Any member that is the absorption of a foreign key to a hub or link
            # (which is all, since we removed FKs to reference tables)
            # must be converted to a Mapping for a new Entity Type that notionally
            # objectifies the absorbed fact type. This Mapping is a new link composite.
            if devolve_links && member.is_a?(MM::Absorption) && member.foreign_key
              next if is_link
              devolve_absorption_to_link member, identifiers.include?(member)
              next
            end

            # If this member is in the natural or surrogate key, leave it there
            # REVISIT: But if it is an FK to another hub, devolve it to a link as well.
            next if identifiers.include?(member)

            # We may absorb a subtype that has no contents. There's no point moving these to a satellite.
            next if is_empty_inheritance member

            satellite_name = name_satellite member
            satellite = satellites[satellite_name]
            unless satellite
              satellite =
              satellites[satellite_name] =
                create_satellite satellite_name, composite
            end

            devolve_member_to_satellite satellite, member
          end
          composite.mapping.re_rank

          # Add the audit and identity fields for the satellite:
          satellites.values.each do |satellite|<commit_msg>Links-as-hubs must not enforce foreign keys to the hubs they should link<commit_after>#
# ActiveFacts Compositions, DataVault Compositor.
#
#       Computes a Data Vault schema.
#
# Copyright (c) 2015 <NAME>. Read the LICENSE file.
#
require "activefacts/compositions/relational"

module ActiveFacts
  module Compositions
    class DataVault < Relational
    public
      def self.options
        {
          reference: ['Boolean', "Emit the reference (static) tables as well. Default is to omit them"],
          datestamp: ['String', "Use this data type for date stamps"],
          id: ['String', "Append this to data vault surrogate keys (default VID)"],
          hubname: ['String', "Suffix or pattern for naming hub tables. Include a + to insert the name. Default 'HUB'"],
          linkname: ['String', "Suffix or pattern for naming link tables. Include a + to insert the name. Default 'LINK'"],
          satname: ['String', "Suffix or pattern for naming satellite tables. Include a + to insert the name. Default 'SAT'"],
          refname: ['String', "Suffix or pattern for naming reference tables. Include a + to insert the name. Default '+'"],
        }
      end

      def initialize constellation, name, options = {}
        # Extract recognised options:
        @option_reference = options.delete('reference')
        @option_datestamp = options.delete('datestamp')
        @option_id = ' ' + (options.delete('id') || 'VID')
        @option_hub_name = options.delete('hubname') || 'HUB'
        @option_hub_name.sub!(/^/,'+ ') unless @option_hub_name =~ /\+/
        @option_link_name = options.delete('linkname') || 'LINK'
        @option_link_name.sub!(/^/,'+ ') unless @option_link_name =~ /\+/
        @option_sat_name = options.delete('satname') || 'SAT'
        @option_sat_name.sub!(/^/,'+ ') unless @option_sat_name =~ /\+/
        @option_ref_name = options.delete('refname') || 'SAT'
        @option_ref_name.sub!(/^/,'+ ') unless @option_ref_name =~ /\+/

        super constellation, name, options

        @option_surrogates = true   # Always inject surrogates regardless of superclass
      end

      # We need to find links that need surrogate keys before we inject the surrogates
      def inject_surrogates
        classify_composites

        super
      end

      def composite_is_reference composite
        object_type = composite.mapping.object_type
        object_type.concept.all_concept_annotation.detect{|ca| ca.mapping_annotation == 'static'} or
          !object_type.is_a?(ActiveFacts::Metamodel::EntityType)
      end

      # Data Vaults need a surrogate key on every Hub and Link.
      # Don't add a surrogate on a Reference table!
      def needs_surrogate(composite)
        return false if composite_is_reference(composite)

        # REVISIT: The following is debatable. If the natural primary key is an ok surrogate, should we inject another?
        return true if @non_reference_composites.include?(composite)

        super
      end

      # Change the default extension from our superclass':
      def inject_surrogate composite, extension = @option_id
        super
      end

      def classify_composites
        detect_reference_tables

        trace :datavault, "Classify non-reference tables into hubs and links" do
          # Make an initial determination, then adjust for foreign keys to links afterwards
          @key_structure = {}
          @link_composites, @hub_composites =
            @non_reference_composites.
            sort_by{|c| c.mapping.name}.
            partition do |composite|
              trace :datavault, "Decide whether #{composite.mapping.name} is a link or a hub" do
                @key_structure[composite] =
                  mapped_to =
                  composite_key_structure composite

                # It's a Link if the preferred identifier includes more than non_reference_composite.
                mapped_to.compact.size > 1
              end
            end

          trace :datavault, "Checking for foreign keys that reference links" do
            # Links may never be the target of a foreign key.
            # Any such links must be defined as hubs instead.
            @links_as_hubs = {}
            fk_dependencies_by_target = {}
            fk_dependencies_by_source = {}
            (@hub_composites+@link_composites).
            each do |composite|
              target_composites = enumerate_foreign_keys composite.mapping
              target_composites.each do |target_composite|
                next if @reference_composites.include?(target_composite)
                (fk_dependencies_by_target[target_composite] ||= []) << composite
                (fk_dependencies_by_source[composite] ||= []) << target_composite
              end
            end

            fk_dependencies_by_target.keys.each do |target_composite|
              if @link_composites.delete(target_composite)
                trace :datavault, "Link #{target_composite.inspect} must be a hub because foreign keys reference it"
                @hub_composites << target_composite
                @links_as_hubs[target_composite] = true
              end
            end

            begin
              converted =
                @link_composites.select do |composite|
                  targets = fk_dependencies_by_source[composite]
                  id_targets = composite_key_structure(composite).compact
                  next if targets.size == id_targets.size
                  trace :datavault, "Link #{composite.mapping.name} must be a hub because it has non-identifying FK references"
                  @link_composites.delete(composite)
                  @hub_composites << composite
                  @links_as_hubs[composite] = true
                end
            end while converted.size > 0

          end

          # Note: We may still have hubs whose identifiers contain foreign keys to one or more other hubs.
          # REVISIT: These foreign keys will be deleted so these hubs stand alone,
          # but have been re-instated as new links to the referenced hubs.
        end

        trace :datavault_classification!, "Data Vault classification of composites:" do
          trace :datavault, "Reference: #{@reference_composites.map(&:mapping).map(&:object_type).map(&:name)*', '}"
          trace :datavault, "Hub: #{@hub_composites.map(&:mapping).map(&:object_type).map(&:name)*', '}"
          trace :datavault, "Link: #{@link_composites.map(&:mapping).map(&:object_type).map(&:name)*', '}"
        end
      end

      def detect_reference_tables
        initial_composites = @composition.all_composite.to_a
        @reference_composites, @non_reference_composites =
          initial_composites.partition { |composite| composite_is_reference(composite) }
      end

      def devolve_all
        delete_reference_table_foreign_keys

        # For each hub and link, move each non-identifying member
        # to a new satellite or promote it to a new link.

        @non_reference_composites.
        each do |composite|
          devolve composite
        end

        rename_parents

        unless @option_reference
          if trace :reference_retraction
            # Add a logger so we can trace the resultant retractions:
            @constellation.loggers << proc do |*args|
              trace :reference_retraction, args.inspect
            end
          end

          @reference_composites.each do |rc|
            trace :reference_retraction, "Retracting #{rc.inspect}" do
              rc.retract
            end
          end
          @reference_composites = []

          @constellation.loggers.pop if trace :reference_retraction
        end
      end

      def delete_reference_table_foreign_keys
        trace :datavault, "Delete foreign keys to reference tables" do
          # Delete all foreign keys to reference tables
          @reference_composites.each do |composite|
            composite.all_foreign_key_as_target_composite.each(&:retract)
          end
        end
      end

      def prefer_natural_key building_natural_key, source_composite, target_composite
        return false if building_natural_key && @link_composites.include?(source_composite)
        building_natural_key && @hub_composites.include?(target_composite)
      end

      def composite_key_structure composite
        # We know that composite.mapping.object_type is an EntityType because all ValueType composites are reference tables

        object_type = composite.mapping.object_type
        mapped_to =
          object_type.preferred_identifier.role_sequence.all_role_ref_in_order.map do |role_ref|
            player = role_ref.role.object_type
            next nil if player == object_type && role_ref.role.fact_type.all_role.size == 1 # Unaries.
            candidate = @candidates[player]
            next nil unless candidate
            # Take care of full absorption
            while candidate.full_absorption
              candidate = candidate.full_absorption.composition
            end
            @non_reference_composites.include?(c = candidate.mapping.composite) ? c : nil
          end

        trace :datavault, "Preferred identifier for #{composite.mapping.name} encloses foreign keys to #{mapped_to.inspect}" unless mapped_to.compact.empty?

        number_of_keys = mapped_to.compact.size
        number_of_values = mapped_to.size-number_of_keys
        trace :datavault_classify,
          if number_of_keys > 1
            # Links have more than one FK to a hub in their key
            "Link #{composite.mapping.name} links #{mapped_to.compact.inspect} with #{number_of_values} values"
          elsif number_of_keys == 1 && number_of_values > 0
            # This is a new hub with a composite key - but we will have to eliminate the foreign key to the base hub
            "Augmented Hub #{composite.mapping.name} has a hub link to #{mapped_to.compact[0].inspect} and #{number_of_values} values"
          elsif number_of_keys == 1
            # This is a new hub with a single-part key that references another hub.
            "Dependent Hub #{composite.mapping.name} is identified by another hub: #{mapped_to.compact[0].inspect}"
          else
            "Hub #{composite.mapping.name} has #{mapped_to.size} parts in its key"
          end

        mapped_to
      end

      # For each member of this composite, decide whether to devolve it to a satellite
      # or to a new link. If it goes to a link that's still part of this natural key,
      # we need to leave that key intact, but remove the foreign key it entails.
      #
      # New links and satellites get new fields for the load date-time and a
      # references to the surrogate(s) on the hub or link, and add an index over
      # those two fields.
      def devolve composite, devolve_links = true
        trace :datavault?, "Devolving non-identifying fields for #{composite.inspect}" do
          # Find the members of this mapping that contain identifying leaves:
          pi = composite.primary_index
          ni = composite.natural_index
          identifiers =
            (Array(pi ? pi.all_index_field : nil) +
             Array(ni ? ni.all_index_field : nil)).
            map{|ixf| ixf.component.path[1]}.
            uniq

          satellites = {}
          is_link = @link_composites.include?(composite) || @links_as_hubs.include?(composite)
          composite.mapping.all_member.to_a.each do |member|

            # Any member that is the absorption of a foreign key to a hub or link
            # (which is all, since we removed FKs to reference tables)
            # must be converted to a Mapping for a new Entity Type that notionally
            # objectifies the absorbed fact type. This Mapping is a new link composite.
            if devolve_links && member.is_a?(MM::Absorption) && member.foreign_key
              next if is_link
              devolve_absorption_to_link member, identifiers.include?(member)
              next
            end

            # If this member is in the natural or surrogate key, leave it there
            # REVISIT: But if it is an FK to another hub, devolve it to a link as well.
            next if identifiers.include?(member)

            # We may absorb a subtype that has no contents. There's no point moving these to a satellite.
            next if is_empty_inheritance member

            satellite_name = name_satellite member
            satellite = satellites[satellite_name]
            unless satellite
              satellite =
              satellites[satellite_name] =
                create_satellite satellite_name, composite
            end

            devolve_member_to_satellite satellite, member
          end
          composite.mapping.re_rank

          if @hub_composites.include?(composite)
            # Links-as-hubs have foreign keys over natural indexes; these must be deleted.
            composite.all_foreign_key_as_source_composite.to_a.each(&:retract)
          end

          # Add the audit and identity fields for the satellite:
          satellites.values.each do |satellite|
{'fraction_non_alphanumeric': 0.07518419935964202, 'fraction_numerical': 0.0010801218994715119, 'mean_word_length': 2.2982188295165393, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3857669', 'n_tokens_mistral': 7142, 'n_tokens_neox': 6796, 'n_words': 2177}