<commit_before>//*****************************************************************************
// Copyright 2017-2018 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//*****************************************************************************

/// In-place-concat optimization makes the argument nodes of a concatenation node use the concatenation node's memory buffer
/// for their outputs. As a result, we eliminate memory copies from the memory buffers of the argument nodes to
/// that of the concatenation node. When there is a chain of in place concatenation nodes, we propagate the
/// memory buffer starting from the last concatenation node. Not all concatenation nodes can be optimized. This pass
/// marks all the nodes that can be optimized.
///
/// Example1:
/// parameter1 parameter2        parameter3 parameter4        parameter5 parameter6
///    \          /                 \          /                 \          /
///         add1                        add2                         add3
///           \                           |                            /
///                                    concat
///
/// Before optimization: the result of add1 is stored to the memory buffer assigned to add1, same for add2 and add3;
///                      then those results are copied to the memory buffer assigned to concat.
/// After optimization: the result of add1 is stored to the memory buffer assigned to concat, same for add2 and add3.
///                     there is no need to copy those results.
///
///
/// Example2:
/// parameter1 parameter2      parameter3 parameter4
///    \          /               \          /
///        add1                       add2
///          \                         /
///                     concat1                     parameter5
///                      |     \                        /
///                      |                 add3
///                       \                 /
///                               concat
///
/// After optimization: the result of add1 is stored to the memory buffer assigned to concat, same for add2 and add3.

#include <cassert>

#include "ngraph/runtime/cpu/pass/cpu_memory_optimization.hpp"

#include "ngraph/descriptor/output.hpp"
#include "ngraph/graph_util.hpp"
#include "ngraph/op/concat.hpp"
#include "ngraph/op/constant.hpp"
#include "ngraph/runtime/cpu/cpu_op_annotations.hpp"
#include "ngraph/runtime/cpu/mkldnn_utils.hpp"

using namespace ngraph;

bool runtime::cpu::pass::CPUMemoryOptimization::run_on_function(std::shared_ptr<Function> function)
{
    for (auto n : function->get_ordered_ops())
    {
        if (auto concat = std::dynamic_pointer_cast<op::Concat>(n))
        {
            auto shape = concat->get_input_shape(0);
            auto axis = concat->get_concatenation_axis();
            auto product = 1;
            for (int i = 0; i < axis; i++)
            {
                product *= shape[i];
            }
            if (product != 1)
            {
                NGRAPH_DEBUG << "cpu_memory_optimization: The product of Concat's shape "
                                "before concat axis is not 1, no in place concat";
                continue;
            }

            bool in_place_concat = true;
            AxisVector axis_list;
            for (auto i = 0; i < shape.size(); i++)
            {
                axis_list.push_back(i);
            }
            auto index = 0;
            for (descriptor::Input& input : concat->get_inputs())
            {
                // no tensors with zero-sized dimensions after zero_dim_tensor_elimination
                assert(shape_size(input.get_shape()) != 0);

                // check if input layout is padded
                auto input_md = mkldnn_utils::get_input_mkldnn_md(n.get(), index);
                index++;
                if (mkldnn_utils::is_mkldnn_padded_layout(input_md, axis_list))
                {
                    NGRAPH_DEBUG
                        << "cpu_memory_optimization: padded input layout, no in place concat";
                    in_place_concat = false;
                    break;
                }

                const auto& output = input.get_output();
                auto arg = output.get_node();
                if (std::dynamic_pointer_cast<op::Constant>(arg) ||
                    std::dynamic_pointer_cast<op::Parameter>(arg))
                {
                    NGRAPH_DEBUG << "cpu_memory_optimization: " << arg->get_name()
                                 << ": constant or parameter, no in place concat";
                    in_place_concat = false;
                    break;
                }

                if (arg->get_output_size() != 1)
                {
                    NGRAPH_DEBUG << "cpu_memory_optimization: " << arg->get_name()
                                 << ": multiple outputs, no in place concat";
                    in_place_concat = false;
                    break;
                }

                if (!std::dynamic_pointer_cast<op::Concat>(arg))
                {
                    if (auto op = std::dynamic_pointer_cast<op::Op>(arg))
                    {
                        auto annotation = op->get_op_annotations();
                        if (annotation && annotation->get_in_place_oi_pairs().size() > 0)

                        {
                            NGRAPH_DEBUG << "cpu_memory_optimization: " << arg->get_name()
                                         << ": in place non-concat op, no in place concat";
                            in_place_concat = false;
                            break;
                        }
                    }
                }
<commit_msg>Addressed PR feedback.<commit_after>//*****************************************************************************
// Copyright 2017-2018 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//*****************************************************************************

/// In-place-concat optimization makes the argument nodes of a concatenation node use the concatenation node's memory buffer
/// for their outputs. As a result, we eliminate memory copies from the memory buffers of the argument nodes to
/// that of the concatenation node. When there is a chain of in place concatenation nodes, we propagate the
/// memory buffer starting from the last concatenation node. Not all concatenation nodes can be optimized. This pass
/// marks all the nodes that can be optimized.
///
/// Example1:
/// parameter1 parameter2        parameter3 parameter4        parameter5 parameter6
///    \          /                 \          /                 \          /
///         add1                        add2                         add3
///           \                           |                            /
///                                    concat
///
/// Before optimization: the result of add1 is stored to the memory buffer assigned to add1, same for add2 and add3;
///                      then those results are copied to the memory buffer assigned to concat.
/// After optimization: the result of add1 is stored to the memory buffer assigned to concat, same for add2 and add3.
///                     there is no need to copy those results.
///
///
/// Example2:
/// parameter1 parameter2      parameter3 parameter4
///    \          /               \          /
///        add1                       add2
///          \                         /
///                     concat1                     parameter5
///                      |     \                        /
///                      |                 add3
///                       \                 /
///                               concat
///
/// After optimization: the result of add1 is stored to the memory buffer assigned to concat, same for add2 and add3.

#include "ngraph/runtime/cpu/pass/cpu_memory_optimization.hpp"

#include "ngraph/descriptor/output.hpp"
#include "ngraph/graph_util.hpp"
#include "ngraph/op/concat.hpp"
#include "ngraph/op/constant.hpp"
#include "ngraph/runtime/cpu/cpu_op_annotations.hpp"
#include "ngraph/runtime/cpu/mkldnn_utils.hpp"

using namespace ngraph;

bool runtime::cpu::pass::CPUMemoryOptimization::run_on_function(std::shared_ptr<Function> function)
{
    for (auto n : function->get_ordered_ops())
    {
        if (auto concat = std::dynamic_pointer_cast<op::Concat>(n))
        {
            auto shape = concat->get_input_shape(0);
            auto axis = concat->get_concatenation_axis();
            auto product = 1;
            for (int i = 0; i < axis; i++)
            {
                product *= shape[i];
            }
            if (product != 1)
            {
                NGRAPH_DEBUG << "cpu_memory_optimization: The product of Concat's shape "
                                "before concat axis is not 1, no in place concat";
                continue;
            }

            bool in_place_concat = true;
            AxisVector axis_list = ngraph::get_default_order(shape);

            auto index = 0;
            for (descriptor::Input& input : concat->get_inputs())
            {
                // no tensors with zero-sized dimensions after zero_dim_tensor_elimination
                NGRAPH_ASSERT(shape_size(input.get_shape()) != 0);

                // check if input layout is padded
                auto input_md = mkldnn_utils::get_input_mkldnn_md(n.get(), index);
                index++;
                if (mkldnn_utils::is_mkldnn_padded_layout(input_md, axis_list))
                {
                    NGRAPH_DEBUG
                        << "cpu_memory_optimization: padded input layout, no in place concat";
                    in_place_concat = false;
                    break;
                }

                const auto& output = input.get_output();
                auto arg = output.get_node();
                if (std::dynamic_pointer_cast<op::Constant>(arg) ||
                    std::dynamic_pointer_cast<op::Parameter>(arg))
                {
                    NGRAPH_DEBUG << "cpu_memory_optimization: " << arg->get_name()
                                 << ": constant or parameter, no in place concat";
                    in_place_concat = false;
                    break;
                }

                NGRAPH_ASSERT(arg->get_output_size() == 1);

                if (!std::dynamic_pointer_cast<op::Concat>(arg))
                {
                    if (auto op = std::dynamic_pointer_cast<op::Op>(arg))
                    {
                        auto annotation = op->get_op_annotations();
                        if (annotation && annotation->get_in_place_oi_pairs().size() > 0)

                        {
                            NGRAPH_DEBUG << "cpu_memory_optimization: " << arg->get_name()
                                         << ": in place non-concat op, no in place concat";
                            in_place_concat = false;
                            break;
                        }
                    }
                }

{'fraction_non_alphanumeric': 0.11503397977788828, 'fraction_numerical': 0.008536383225592574, 'mean_word_length': 1.4372853968895172, 'pattern_counts': {'":': 5, '<': 57, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 31, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '252702', 'n_tokens_mistral': 3017, 'n_tokens_neox': 2801, 'n_words': 884}