<commit_before>module AwsRefresherSpecCounts
  extend ActiveSupport::Concern

  def expected_ext_management_systems_count
    ::Settings.prototype.amazon.s3 ? 4 : 3
  end

  def base_inventory_counts
    {
      :auth_private_key              => 0,
      :availability_zone             => 0,
      :cloud_network                 => 0,
      :cloud_subnet                  => 0,
      :cloud_volume                  => 0,
      :cloud_volume_backup           => 0,
      :cloud_volume_snapshot         => 0,
      :custom_attribute              => 0,
      :disk                          => 0,
      :ext_management_system         => expected_ext_management_systems_count,
      :firewall_rule                 => 0,
      :flavor                        => 0,
      :floating_ip                   => 0,
      :guest_device                  => 0,
      :hardware                      => 0,
      :miq_template                  => 0,
      :network                       => 0,
      :network_port                  => 0,
      :network_router                => 0,
      :operating_system              => 0,
      :orchestration_stack           => 0,
      :orchestration_stack_output    => 0,
      :orchestration_stack_parameter => 0,
      :orchestration_stack_resource  => 0,
      :orchestration_template        => 0,
      :security_group                => 0,
      :snapshot                      => 0,
      :system_service                => 0,
      :vm                            => 0,
      :vm_or_template                => 0
    }
  end

  def assert_counts(expected_table_counts)
    expected_counts = base_inventory_counts.merge(expected_table_counts)
    assert_table_counts(expected_counts)
    assert_ems(expected_counts)
  end

  def table_counts_from_api
    all_instance_hashes        = instances
    instance_hashes            = all_instance_hashes.select { |x| x["state"]["name"] != "terminated" }
    image_hashes               = private_images + shared_images + public_images
    # Only new refresh can collect a referenced images
    image_hashes += referenced_images(instance_hashes, image_hashes) if options.inventory_object_refresh
    instances_count            = instance_hashes.size
    images_count               = image_hashes.size
    instances_and_images_count = instances_count + images_count

    # Custom attributes of all Vms and Images
    custom_attributes_count    = (instance_hashes + image_hashes).map { |x| x["tags"] }.flatten.compact.size

    # Networks for all Vms is a list of private and public addresses of a 1 interface of a Vm
    networks_count             = instance_hashes.map { |x| [x['public_ip_address'], x['private_ip_address']] }.flatten.compact.size

    indexed_flavors        = Flavor.all.index_by(&:name)
    ephemeral_disk_count   = instance_hashes.map { |x| indexed_flavors[x["instance_type"]].ephemeral_disk_count }.sum
    attached_volumes_count = instance_hashes.map { |x| x["block_device_mappings"] }.flatten.size
    # Total number of disks is ephemeral_disk_count driven by a Flavor + number of attached volumes
    disks_count            = ephemeral_disk_count + attached_volumes_count

    security_group_hashes = security_groups.all
    security_groups_count = security_group_hashes.count

    # Total number of firewall rules is inferred from security groups
    firewall_rules_count  = security_group_hashes.map do |rule|
      (
      rule["ip_permissions"].map { |perm| perm["user_id_group_pairs"] } +
        rule["ip_permissions"].map { |perm| perm["ip_ranges"] } +
        rule["ip_permissions_egress"].map { |perm| perm["user_id_group_pairs"] } +
        rule["ip_permissions_egress"].map { |perm| perm["ip_ranges"] }
      )
    end.flatten.size

    # TODO(lsmola) we don't filter out terminated instances for getting network_ports from EC2 classic instances, fix it
    # then we can use instance_hashes instead of all_instance_hashes here.
    ec2_classic_instance_hashes = all_instance_hashes.select { |x| x['network_interfaces'].blank? }
    network_port_hashes         = network_ports.all
    # Network ports count consists of VPC ENIs + ec2_classic_instances
    network_ports_count         = network_port_hashes.size + ec2_classic_instance_hashes.size

    floating_ip_hashes = floating_ips.all
    # VPC floating IPs are taken from floating_ips and network_ports + EC2 classic floating ips are taken from
    # floating_ips and instances. FloatingIp model then holds all AWS public and Elastic IPs.
    floating_ips_refs  = Set.new
    floating_ips_refs.merge(
      network_port_hashes.map do |network_port|
        network_port["private_ip_addresses"].map do |private_address|
          private_address.fetch_path('association', 'allocation_id') ||
            private_address.fetch_path('association', 'public_ip')
        end
      end.flatten.compact
    )
    floating_ips_refs.merge(
      floating_ip_hashes.map { |floating_ip| floating_ip['allocation_id'] || floating_ip['public_ip'] }.compact
    )
    floating_ips_refs.merge(
      ec2_classic_instance_hashes.map { |instance| instance['public_ip_address'] }.compact
    )

    orchestration_stack_hashes = stacks.all
    orchestration_stacks_count = orchestration_stack_hashes.size

    orchestration_stack_parameters_count = orchestration_stack_hashes.map { |x| x["parameters"] }.flatten.compact.size
    orchestration_stack_outputs_count    = orchestration_stack_hashes.map { |x| x["outputs"] }.flatten.compact.size
    orchestration_stack_resources_count  = stacks_resources.size

    base_inventory_counts.merge(
      :auth_private_key              => key_pairs.size,
      :availability_zone             => availability_zones.size,
      :cloud_network                 => cloud_networks.size,
      :cloud_subnet                  => cloud_subnets.size,
      :cloud_volume                  => cloud_volumes.size,
      :cloud_volume_snapshot         => cloud_volume_snapshots.size,
      :custom_attribute              => custom_attributes_count,
      :disk                          => disks_count,
      :firewall_rule                 => firewall_rules_count,
      :flavor                        => 76,
      :floating_ip                   => floating_ips_refs.size,
      :hardware                      => instances_and_images_count,
      :miq_template                  => images_count,
      :network                       => networks_count,
      :network_port                  => network_ports_count,
      :orchestration_stack           => orchestration_stacks_count,
      :orchestration_stack_output    => orchestration_stack_outputs_count,
      :orchestration_stack_parameter => orchestration_stack_parameters_count,
      :orchestration_stack_resource  => orchestration_stack_resources_count,<commit_msg>Add LB's network ports and floating ips to counts spec<commit_after>module AwsRefresherSpecCounts
  extend ActiveSupport::Concern

  def expected_ext_management_systems_count
    ::Settings.prototype.amazon.s3 ? 4 : 3
  end

  def base_inventory_counts
    {
      :auth_private_key              => 0,
      :availability_zone             => 0,
      :cloud_network                 => 0,
      :cloud_subnet                  => 0,
      :cloud_volume                  => 0,
      :cloud_volume_backup           => 0,
      :cloud_volume_snapshot         => 0,
      :custom_attribute              => 0,
      :disk                          => 0,
      :ext_management_system         => expected_ext_management_systems_count,
      :firewall_rule                 => 0,
      :flavor                        => 0,
      :floating_ip                   => 0,
      :guest_device                  => 0,
      :hardware                      => 0,
      :miq_template                  => 0,
      :network                       => 0,
      :network_port                  => 0,
      :network_router                => 0,
      :operating_system              => 0,
      :orchestration_stack           => 0,
      :orchestration_stack_output    => 0,
      :orchestration_stack_parameter => 0,
      :orchestration_stack_resource  => 0,
      :orchestration_template        => 0,
      :security_group                => 0,
      :snapshot                      => 0,
      :system_service                => 0,
      :vm                            => 0,
      :vm_or_template                => 0
    }
  end

  def assert_counts(expected_table_counts)
    expected_counts = base_inventory_counts.merge(expected_table_counts)
    assert_table_counts(expected_counts)
    assert_ems(expected_counts)
  end

  def table_counts_from_api
    all_instance_hashes        = instances
    instance_hashes            = all_instance_hashes.select { |x| x["state"]["name"] != "terminated" }
    image_hashes               = private_images + shared_images + public_images
    # Only new refresh can collect a referenced images
    image_hashes += referenced_images(instance_hashes, image_hashes) if options.inventory_object_refresh
    instances_count            = instance_hashes.size
    images_count               = image_hashes.size
    instances_and_images_count = instances_count + images_count

    # Custom attributes of all Vms and Images
    custom_attributes_count    = (instance_hashes + image_hashes).map { |x| x["tags"] }.flatten.compact.size

    # Networks for all Vms is a list of private and public addresses of a 1 interface of a Vm
    networks_count             = instance_hashes.map { |x| [x['public_ip_address'], x['private_ip_address']] }.flatten.compact.size

    indexed_flavors        = Flavor.all.index_by(&:name)
    ephemeral_disk_count   = instance_hashes.map { |x| indexed_flavors[x["instance_type"]].ephemeral_disk_count }.sum
    attached_volumes_count = instance_hashes.map { |x| x["block_device_mappings"] }.flatten.size
    # Total number of disks is ephemeral_disk_count driven by a Flavor + number of attached volumes
    disks_count            = ephemeral_disk_count + attached_volumes_count

    security_group_hashes = security_groups.all
    security_groups_count = security_group_hashes.count

    load_balancers_size = load_balancers.size

    # Total number of firewall rules is inferred from security groups
    firewall_rules_count  = security_group_hashes.map do |rule|
      (
      rule["ip_permissions"].map { |perm| perm["user_id_group_pairs"] } +
        rule["ip_permissions"].map { |perm| perm["ip_ranges"] } +
        rule["ip_permissions_egress"].map { |perm| perm["user_id_group_pairs"] } +
        rule["ip_permissions_egress"].map { |perm| perm["ip_ranges"] }
      )
    end.flatten.size

    # TODO(lsmola) we don't filter out terminated instances for getting network_ports from EC2 classic instances, fix it
    # then we can use instance_hashes instead of all_instance_hashes here.
    ec2_classic_instance_hashes = all_instance_hashes.select { |x| x['network_interfaces'].blank? }
    network_port_hashes         = network_ports.all
    # Network ports count consists of VPC ENIs + ec2_classic_instances
    network_ports_count         = network_port_hashes.size + ec2_classic_instance_hashes.size + load_balancers_size

    floating_ip_hashes = floating_ips.all
    # VPC floating IPs are taken from floating_ips and network_ports + EC2 classic floating ips are taken from
    # floating_ips and instances. FloatingIp model then holds all AWS public and Elastic IPs.
    floating_ips_refs  = Set.new
    floating_ips_refs.merge(
      network_port_hashes.map do |network_port|
        network_port["private_ip_addresses"].map do |private_address|
          private_address.fetch_path('association', 'allocation_id') ||
            private_address.fetch_path('association', 'public_ip')
        end
      end.flatten.compact
    )
    floating_ips_refs.merge(
      floating_ip_hashes.map { |floating_ip| floating_ip['allocation_id'] || floating_ip['public_ip'] }.compact
    )
    floating_ips_refs.merge(
      ec2_classic_instance_hashes.map { |instance| instance['public_ip_address'] }.compact
    )

    floating_ips_count = floating_ips_refs.size + load_balancers_size

    orchestration_stack_hashes = stacks.all
    orchestration_stacks_count = orchestration_stack_hashes.size

    orchestration_stack_parameters_count = orchestration_stack_hashes.map { |x| x["parameters"] }.flatten.compact.size
    orchestration_stack_outputs_count    = orchestration_stack_hashes.map { |x| x["outputs"] }.flatten.compact.size
    orchestration_stack_resources_count  = stacks_resources.size

    base_inventory_counts.merge(
      :auth_private_key              => key_pairs.size,
      :availability_zone             => availability_zones.size,
      :cloud_network                 => cloud_networks.size,
      :cloud_subnet                  => cloud_subnets.size,
      :cloud_volume                  => cloud_volumes.size,
      :cloud_volume_snapshot         => cloud_volume_snapshots.size,
      :custom_attribute              => custom_attributes_count,
      :disk                          => disks_count,
      :firewall_rule                 => firewall_rules_count,
      :flavor                        => 76,
      :floating_ip                   => floating_ips_count,
      :hardware                      => instances_and_images_count,
      :miq_template                  => images_count,
      :network                       => networks_count,
      :network_port                  => network_ports_count,
      :orchestration_stack           => orchestration_stacks_count,
      :orchestration_stack_output    => orchestration_stack_outputs_count,
      :orchestration_stack_parameter => orchestration_stack_parameters_count,
      :orchestration_stack_resource  => orchestration_stack_resources_count,
{'fraction_non_alphanumeric': 0.07896845632694689, 'fraction_numerical': 0.005973628615138049, 'mean_word_length': 2.385450061652281, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 101, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4154072', 'n_tokens_mistral': 4144, 'n_tokens_neox': 3965, 'n_words': 744}