<commit_before>---
version: '2.0'
name: tripleo.derive_params_formulas.v1
description: TripleO Workflows to derive deployment parameters from the introspected data

workflows:


  dpdk_derive_params:
    description: >
      Workflow to derive parameters for DPDK service.
    input:
      - plan
      - role_name
      - heat_resource_tree
      - hw_data # introspection data
      - user_inputs
      - derived_parameters: {}

    output:
      derived_parameters: <% $.derived_parameters.mergeWith($.get('dpdk_parameters', {})) %>

    tags:
      - tripleo-common-managed

    tasks:
      get_network_config:
        action: tripleo.parameters.get_network_config
        input:
          container: <% $.plan %>
          role_name: <% $.role_name %>
        publish:
          network_configs: <% task().result.get('network_config', []) %>
        on-success: get_dpdk_nics_numa_info
        on-error: set_status_failed_get_network_config

      get_dpdk_nics_numa_info:
        action: tripleo.derive_params.get_dpdk_nics_numa_info
        input:
          network_configs: <% $.network_configs %>
          inspect_data: <% $.hw_data %>
        publish:
          dpdk_nics_numa_info: <% task().result %>
        on-success:
          # TODO: Need to remove condtions here
          # adding condition and throw error in action for empty check
          - get_dpdk_nics_numa_nodes: <% $.dpdk_nics_numa_info %>
          - set_status_failed_get_dpdk_nics_numa_info: <% not $.dpdk_nics_numa_info %>
        on-error: set_status_failed_on_error_get_dpdk_nics_numa_info

      get_dpdk_nics_numa_nodes:
        publish:
          dpdk_nics_numa_nodes: <% $.dpdk_nics_numa_info.groupBy($.numa_node).select($[0]).orderBy($) %>
        on-success:
          - get_numa_nodes: <% $.dpdk_nics_numa_nodes %>
          - set_status_failed_get_dpdk_nics_numa_nodes: <% not $.dpdk_nics_numa_nodes %>

      get_numa_nodes:
        publish:
          numa_nodes: <% $.hw_data.numa_topology.ram.select($.numa_node).orderBy($) %>
        on-success:
          - get_num_phy_cores_per_numa_for_pmd: <% $.numa_nodes %>
          - set_status_failed_get_numa_nodes: <% not $.numa_nodes %>

      get_num_phy_cores_per_numa_for_pmd:
        publish:
          num_phy_cores_per_numa_node_for_pmd: <% $.user_inputs.get('num_phy_cores_per_numa_node_for_pmd', 0) %>
        on-success:
          - get_num_cores_per_numa_nodes: <% isInteger($.num_phy_cores_per_numa_node_for_pmd) and $.num_phy_cores_per_numa_node_for_pmd > 0 %>
          - set_status_failed_get_num_phy_cores_per_numa_for_pmd_invalid: <% not isInteger($.num_phy_cores_per_numa_node_for_pmd) %>
          - set_status_failed_get_num_phy_cores_per_numa_for_pmd_not_provided: <% $.num_phy_cores_per_numa_node_for_pmd = 0 %>

      # For NUMA node with DPDK nic, number of cores should be used from user input
      # For NUMA node without DPDK nic, number of cores should be 1
      get_num_cores_per_numa_nodes:
        publish:
          num_cores_per_numa_nodes: <% let(dpdk_nics_nodes => $.dpdk_nics_numa_nodes, cores => $.num_phy_cores_per_numa_node_for_pmd) -> $.numa_nodes.select(switch($ in $dpdk_nics_nodes => $cores, not $ in $dpdk_nics_nodes => 1)) %>
        on-success: get_pmd_cpus

      get_pmd_cpus:
        action: tripleo.derive_params.get_dpdk_core_list
        input:
          inspect_data: <% $.hw_data %>
          numa_nodes_cores_count: <% $.num_cores_per_numa_nodes %>
        publish:
          pmd_cpus: <% task().result %>
        on-success:
          - get_pmd_cpus_range_list: <% $.pmd_cpus %>
          - set_status_failed_get_pmd_cpus: <% not $.pmd_cpus %>
        on-error: set_status_failed_on_error_get_pmd_cpus

      get_pmd_cpus_range_list:
        action: tripleo.derive_params.convert_number_to_range_list
        input:
          num_list: <% $.pmd_cpus %>
        publish:
          pmd_cpus: <% task().result %>
        on-success: get_host_cpus
        on-error: set_status_failed_get_pmd_cpus_range_list

      get_host_cpus:
        workflow: tripleo.derive_params_formulas.v1.get_host_cpus
        input:
          role_name: <% $.role_name %>
          hw_data: <% $.hw_data %>
        publish:
          host_cpus: <% task().result.get('host_cpus', '') %>
        on-success: get_sock_mem
        on-error: set_status_failed_get_host_cpus

      get_sock_mem:
        action: tripleo.derive_params.get_dpdk_socket_memory
        input:
          dpdk_nics_numa_info: <% $.dpdk_nics_numa_info %>
          numa_nodes: <% $.numa_nodes %>
          overhead: <% $.user_inputs.get('overhead', 800) %>
          packet_size_in_buffer: <% 4096*64 %>
        publish:
          sock_mem: <% task().result %>
        on-success:
          - get_neutron_bridge_mappings: <% $.sock_mem %>
          - set_status_failed_get_sock_mem: <% not $.sock_mem %>
        on-error: set_status_failed_on_error_get_sock_mem

      get_neutron_bridge_mappings:
        publish:
          neutron_bridge_mappings: <% $.heat_resource_tree.parameters.get('NeutronBridgeMappings', {}).get('default', '') %>
        on-success:
          - get_phy_nw_bridge_mappings: <% $.neutron_bridge_mappings %>
          - get_neutron_network_type: <% not $.neutron_bridge_mappings %>

      # Gets the physical network and ovs bridge mappings
      get_phy_nw_bridge_mappings:
        publish:
          phy_nw_bridge_mappings: <% $.neutron_bridge_mappings.split(',').select(let(mapping => $.split(':')) -> dict($mapping[0] => $mapping[1])).sum() %>
        on-success: get_bridge_numa_nodes_mappings

      # Gets the ovs bridge and NUMA nodes mappings
      get_bridge_numa_nodes_mappings:
        publish:
          bridge_numa_nodes_mappings: <% $.dpdk_nics_numa_info.groupBy($.bridge_name).select(dict($[0]=>$[1].select($.numa_node).distinct())).sum() %>
        on-success: get_phy_nw_numa_nodes_mappings

      # Gets the physical network and NUMA nodes mappings
      get_phy_nw_numa_nodes_mappings:
        publish:
          phy_nw_numa_nodes_mappings: <% let(br_numa_mappings => $.bridge_numa_nodes_mappings.items()) -> $.phy_nw_bridge_mappings.items().select(dict($[0]=>$br_numa_mappings.select($[1]))).sum() %>
        on-success: get_neutron_network_type

      get_neutron_network_type:
        publish:
          neutron_network_type: <% $.heat_resource_tree.parameters.get('NeutronNetworkType', {}).get('default', '') %>
        on-success:
          - get_tunnel_numa_nodes_mappings: <% 'vxlan' in $.neutron_network_type %>
          - get_dpdk_parameters: <% not 'vxlan' in $.neutron_network_type %>

      # Gets the list of NUMA nodes associated to all tunneled networks
      # OVS-DPDK on VxLAN tunnel requires Tenant Network IP to be applied on the OVS User Bridge itself.
      # With this assumption, if the IP is set on the OVS User Bridge, then OVS-DPDK is used for VxLAN tunnels also.
      # Here dpdk_nics_numa_info will have the OVS User Bridges with DPDK ports only.
      get_tunnel_numa_nodes_mappings:
        publish:
          tunnel_numa_nodes_mappings: <% $.dpdk_nics_numa_info.where($.addresses.any($.ip_netmask)).select($.numa_node).distinct() %>
        on-success: get_dpdk_parameters

      get_dpdk_parameters:
        publish:
          dpdk_parameters: <% dict(concat($.role_name, 'Parameters') => dict('OvsPmdCoreList' => $.get('pmd_cpus', ''), 'OvsDpdkCoreList' => $.get('host_cpus', ''), 'OvsDpdkSocketMemory' => $.get('sock_mem', ''))) %>
        on-success:
          - add_phy_nw_numa_nodes_mappings: <% $.get('phy_nw_numa_nodes_mappings', {}) %>
          - add_tunnel_numa_nodes_mappings: <% $.get('tunnel_numa_nodes_mappings', []) %>

      add_phy_nw_numa_nodes_mappings:<commit_msg>NeutronPhysnetNUMANodesMapping parameter issue<commit_after>---
version: '2.0'
name: tripleo.derive_params_formulas.v1
description: TripleO Workflows to derive deployment parameters from the introspected data

workflows:


  dpdk_derive_params:
    description: >
      Workflow to derive parameters for DPDK service.
    input:
      - plan
      - role_name
      - heat_resource_tree
      - hw_data # introspection data
      - user_inputs
      - derived_parameters: {}

    output:
      derived_parameters: <% $.derived_parameters.mergeWith($.get('dpdk_parameters', {})) %>

    tags:
      - tripleo-common-managed

    tasks:
      get_network_config:
        action: tripleo.parameters.get_network_config
        input:
          container: <% $.plan %>
          role_name: <% $.role_name %>
        publish:
          network_configs: <% task().result.get('network_config', []) %>
        on-success: get_dpdk_nics_numa_info
        on-error: set_status_failed_get_network_config

      get_dpdk_nics_numa_info:
        action: tripleo.derive_params.get_dpdk_nics_numa_info
        input:
          network_configs: <% $.network_configs %>
          inspect_data: <% $.hw_data %>
        publish:
          dpdk_nics_numa_info: <% task().result %>
        on-success:
          # TODO: Need to remove condtions here
          # adding condition and throw error in action for empty check
          - get_dpdk_nics_numa_nodes: <% $.dpdk_nics_numa_info %>
          - set_status_failed_get_dpdk_nics_numa_info: <% not $.dpdk_nics_numa_info %>
        on-error: set_status_failed_on_error_get_dpdk_nics_numa_info

      get_dpdk_nics_numa_nodes:
        publish:
          dpdk_nics_numa_nodes: <% $.dpdk_nics_numa_info.groupBy($.numa_node).select($[0]).orderBy($) %>
        on-success:
          - get_numa_nodes: <% $.dpdk_nics_numa_nodes %>
          - set_status_failed_get_dpdk_nics_numa_nodes: <% not $.dpdk_nics_numa_nodes %>

      get_numa_nodes:
        publish:
          numa_nodes: <% $.hw_data.numa_topology.ram.select($.numa_node).orderBy($) %>
        on-success:
          - get_num_phy_cores_per_numa_for_pmd: <% $.numa_nodes %>
          - set_status_failed_get_numa_nodes: <% not $.numa_nodes %>

      get_num_phy_cores_per_numa_for_pmd:
        publish:
          num_phy_cores_per_numa_node_for_pmd: <% $.user_inputs.get('num_phy_cores_per_numa_node_for_pmd', 0) %>
        on-success:
          - get_num_cores_per_numa_nodes: <% isInteger($.num_phy_cores_per_numa_node_for_pmd) and $.num_phy_cores_per_numa_node_for_pmd > 0 %>
          - set_status_failed_get_num_phy_cores_per_numa_for_pmd_invalid: <% not isInteger($.num_phy_cores_per_numa_node_for_pmd) %>
          - set_status_failed_get_num_phy_cores_per_numa_for_pmd_not_provided: <% $.num_phy_cores_per_numa_node_for_pmd = 0 %>

      # For NUMA node with DPDK nic, number of cores should be used from user input
      # For NUMA node without DPDK nic, number of cores should be 1
      get_num_cores_per_numa_nodes:
        publish:
          num_cores_per_numa_nodes: <% let(dpdk_nics_nodes => $.dpdk_nics_numa_nodes, cores => $.num_phy_cores_per_numa_node_for_pmd) -> $.numa_nodes.select(switch($ in $dpdk_nics_nodes => $cores, not $ in $dpdk_nics_nodes => 1)) %>
        on-success: get_pmd_cpus

      get_pmd_cpus:
        action: tripleo.derive_params.get_dpdk_core_list
        input:
          inspect_data: <% $.hw_data %>
          numa_nodes_cores_count: <% $.num_cores_per_numa_nodes %>
        publish:
          pmd_cpus: <% task().result %>
        on-success:
          - get_pmd_cpus_range_list: <% $.pmd_cpus %>
          - set_status_failed_get_pmd_cpus: <% not $.pmd_cpus %>
        on-error: set_status_failed_on_error_get_pmd_cpus

      get_pmd_cpus_range_list:
        action: tripleo.derive_params.convert_number_to_range_list
        input:
          num_list: <% $.pmd_cpus %>
        publish:
          pmd_cpus: <% task().result %>
        on-success: get_host_cpus
        on-error: set_status_failed_get_pmd_cpus_range_list

      get_host_cpus:
        workflow: tripleo.derive_params_formulas.v1.get_host_cpus
        input:
          role_name: <% $.role_name %>
          hw_data: <% $.hw_data %>
        publish:
          host_cpus: <% task().result.get('host_cpus', '') %>
        on-success: get_sock_mem
        on-error: set_status_failed_get_host_cpus

      get_sock_mem:
        action: tripleo.derive_params.get_dpdk_socket_memory
        input:
          dpdk_nics_numa_info: <% $.dpdk_nics_numa_info %>
          numa_nodes: <% $.numa_nodes %>
          overhead: <% $.user_inputs.get('overhead', 800) %>
          packet_size_in_buffer: <% 4096*64 %>
        publish:
          sock_mem: <% task().result %>
        on-success:
          - get_neutron_bridge_mappings: <% $.sock_mem %>
          - set_status_failed_get_sock_mem: <% not $.sock_mem %>
        on-error: set_status_failed_on_error_get_sock_mem

      get_neutron_bridge_mappings:
        publish:
          neutron_bridge_mappings: <% $.heat_resource_tree.parameters.get('NeutronBridgeMappings', {}).get('default', '') %>
        on-success:
          - get_phy_nw_bridge_mappings: <% $.neutron_bridge_mappings %>
          - get_neutron_network_type: <% not $.neutron_bridge_mappings %>

      # Gets the physical network and ovs bridge mappings
      get_phy_nw_bridge_mappings:
        publish:
          phy_nw_bridge_mappings: <% $.neutron_bridge_mappings.split(',').select(let(mapping => $.split(':')) -> dict($mapping[0] => $mapping[1])).sum() %>
        on-success: get_bridge_numa_nodes_mappings

      # Gets the ovs bridge and NUMA nodes mappings
      get_bridge_numa_nodes_mappings:
        publish:
          bridge_numa_nodes_mappings: <% $.dpdk_nics_numa_info.groupBy($.bridge_name).select(dict($[0]=>$[1].select($.numa_node).distinct())).sum() %>
        on-success: get_phy_nw_numa_nodes_mappings

      # Gets the physical network and NUMA nodes mappings
      get_phy_nw_numa_nodes_mappings:
        publish:
          phy_nw_numa_nodes_mappings: <% let(nw_bridge_mappings => $.phy_nw_bridge_mappings) -> $.bridge_numa_nodes_mappings.items().select(let(br => $[0], nodes => $[1]) -> $nw_bridge_mappings.items().where($[1]=$br).select(dict($[0] => $nodes))).sum() %>
        on-success: get_neutron_network_type

      get_neutron_network_type:
        publish:
          neutron_network_type: <% $.heat_resource_tree.parameters.get('NeutronNetworkType', {}).get('default', '') %>
        on-success:
          - get_tunnel_numa_nodes_mappings: <% 'vxlan' in $.neutron_network_type %>
          - get_dpdk_parameters: <% not 'vxlan' in $.neutron_network_type %>

      # Gets the list of NUMA nodes associated to all tunneled networks
      # OVS-DPDK on VxLAN tunnel requires Tenant Network IP to be applied on the OVS User Bridge itself.
      # With this assumption, if the IP is set on the OVS User Bridge, then OVS-DPDK is used for VxLAN tunnels also.
      # Here dpdk_nics_numa_info will have the OVS User Bridges with DPDK ports only.
      get_tunnel_numa_nodes_mappings:
        publish:
          tunnel_numa_nodes_mappings: <% $.dpdk_nics_numa_info.where($.addresses.any($.ip_netmask)).select($.numa_node).distinct() %>
        on-success: get_dpdk_parameters

      get_dpdk_parameters:
        publish:
          dpdk_parameters: <% dict(concat($.role_name, 'Parameters') => dict('OvsPmdCoreList' => $.get('pmd_cpus', ''), 'OvsDpdkCoreList' => $.get('host_cpus', ''), 'OvsDpdkSocketMemory' => $.get('sock_mem', ''))) %>
        on-success:
          - add_phy_nw_numa_nodes_mappings: <% $.get('phy_nw_numa_nodes_mappings', {}) %>
          - add_tunnel_numa_nodes_mappings: <% $.get('tunnel_numa_nodes_mappings', []) %>

      add_phy_nw_numa_nodes_mappings:
{'fraction_non_alphanumeric': 0.11215736859077717, 'fraction_numerical': 0.0033537568526281845, 'mean_word_length': 2.9963917525773196, 'pattern_counts': {'":': 0, '<': 103, '<?xml version=': 0, '>': 142, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1442264', 'n_tokens_mistral': 5700, 'n_tokens_neox': 5456, 'n_words': 806}