<commit_before>/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "paddle/fluid/framework/paddle2cinn/build_cinn_pass.h"

#include <algorithm>
#include <iterator>
#include <memory>
#include <regex>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>

#include "cinn/frontend/op_mapper_registry.h"
#include "cinn/frontend/op_mappers/use_op_mappers.h"
#include "gflags/gflags.h"
#include "glog/logging.h"
#include "paddle/fluid/framework/ir/graph.h"
#include "paddle/fluid/framework/ir/graph_pattern_detector.h"
#include "paddle/fluid/framework/ir/node.h"
#include "paddle/fluid/framework/ir/subgraph_detector.h"
#include "paddle/fluid/framework/op_info.h"
#include "paddle/fluid/framework/op_proto_maker.h"
#include "paddle/fluid/framework/paddle2cinn/cinn_compiler.h"
#include "paddle/fluid/operators/cinn/cinn_launch_op.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/fluid/platform/errors.h"

DECLARE_string(allow_cinn_ops);
DECLARE_string(deny_cinn_ops);

namespace paddle {
namespace framework {
namespace paddle2cinn {

using framework::ir::Graph;
using framework::ir::Node;

using GraphNodeVec = std::vector<Node*>;
using GraphNodeMap = std::unordered_map<Node*, Node*>;

OpTransInfo::OpTransInfo() {
  // judgment condition for the dynamic slice
  dynamic_op_cond_.emplace("slice", [](const ir::Node& node) -> bool {
    if (!node.IsOp()) {
      return false;
    }
    auto* op_desc = node.Op();
    auto infer_flags =
        op_desc->GetAttrIfExists<std::vector<int>>("infer_flags");
    return std::find_if(infer_flags.begin(), infer_flags.end(), [](int v) {
             return v < 0;
           }) != infer_flags.end();
  });

  // judgment condition for the dynamic reshape
  dynamic_op_cond_.emplace("reshape", [](const ir::Node& node) -> bool {
    if (!node.IsOp()) {
      return false;
    }
    auto* op_desc = node.Op();
    bool has_shape_tensor = op_desc->Inputs().count("ShapeTensor") &&
                            op_desc->Inputs().at("ShapeTensor").size();
    bool has_shape = op_desc->Inputs().count("Shape") &&
                     op_desc->Inputs().at("Shape").size();
    return has_shape_tensor || has_shape;
  });

  // judgment condition for the dynamic reshape2
  dynamic_op_cond_.emplace("reshape2", dynamic_op_cond_.at("reshape"));
}

std::unordered_set<std::string> OpTransInfo::GetDenyVarNames(
    const GraphNodeSet& cluster) const {
  std::unordered_set<std::string> deny_var_set;

  auto get_debug_info = [](const std::unordered_set<std::string>& var_names) {
    std::string debug_info = "[";
    for (auto& var : var_names) {
      debug_info.append(var);
      debug_info.append(", ");
    }
    debug_info.append("]");<commit_msg>[CINN] support expand with static expand_times (#46776)<commit_after>/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "paddle/fluid/framework/paddle2cinn/build_cinn_pass.h"

#include <algorithm>
#include <iterator>
#include <memory>
#include <regex>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>

#include "cinn/frontend/op_mapper_registry.h"
#include "cinn/frontend/op_mappers/use_op_mappers.h"
#include "gflags/gflags.h"
#include "glog/logging.h"
#include "paddle/fluid/framework/ir/graph.h"
#include "paddle/fluid/framework/ir/graph_pattern_detector.h"
#include "paddle/fluid/framework/ir/node.h"
#include "paddle/fluid/framework/ir/subgraph_detector.h"
#include "paddle/fluid/framework/op_info.h"
#include "paddle/fluid/framework/op_proto_maker.h"
#include "paddle/fluid/framework/paddle2cinn/cinn_compiler.h"
#include "paddle/fluid/operators/cinn/cinn_launch_op.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/fluid/platform/errors.h"

DECLARE_string(allow_cinn_ops);
DECLARE_string(deny_cinn_ops);

namespace paddle {
namespace framework {
namespace paddle2cinn {

using framework::ir::Graph;
using framework::ir::Node;

using GraphNodeVec = std::vector<Node*>;
using GraphNodeMap = std::unordered_map<Node*, Node*>;

OpTransInfo::OpTransInfo() {
  // judgment condition for the dynamic slice
  dynamic_op_cond_.emplace("slice", [](const ir::Node& node) -> bool {
    if (!node.IsOp()) {
      return false;
    }
    auto* op_desc = node.Op();
    auto infer_flags =
        op_desc->GetAttrIfExists<std::vector<int>>("infer_flags");
    return std::find_if(infer_flags.begin(), infer_flags.end(), [](int v) {
             return v < 0;
           }) != infer_flags.end();
  });

  // judgment condition for the dynamic reshape
  dynamic_op_cond_.emplace("reshape", [](const ir::Node& node) -> bool {
    if (!node.IsOp()) {
      return false;
    }
    auto* op_desc = node.Op();
    bool has_shape_tensor = op_desc->Inputs().count("ShapeTensor") &&
                            op_desc->Inputs().at("ShapeTensor").size();
    bool has_shape = op_desc->Inputs().count("Shape") &&
                     op_desc->Inputs().at("Shape").size();
    return has_shape_tensor || has_shape;
  });

  // judgment condition for the dynamic reshape2
  dynamic_op_cond_.emplace("reshape2", dynamic_op_cond_.at("reshape"));

  // judgment condition for the dynamic expand
  dynamic_op_cond_.emplace("expand", [](const ir::Node& node) -> bool {
    if (!node.IsOp()) {
      return false;
    }
    auto* op_desc = node.Op();
    bool has_expand_times_tensor =
        op_desc->Inputs().count("expand_times_tensor") &&
        op_desc->Inputs().at("expand_times_tensor").size();
    bool has_expand_times = op_desc->Inputs().count("ExpandTimes") &&
                            op_desc->Inputs().at("ExpandTimes").size();
    return has_expand_times_tensor || has_expand_times;
  });

  // judgment condition for the dynamic expand_v2
  dynamic_op_cond_.emplace("expand_v2", [](const ir::Node& node) -> bool {
    if (!node.IsOp()) {
      return false;
    }
    auto* op_desc = node.Op();
    bool has_expand_shapes_tensor =
        op_desc->Inputs().count("expand_shapes_tensor") &&
        op_desc->Inputs().at("expand_shapes_tensor").size();
    bool has_shape = op_desc->Inputs().count("Shape") &&
                     op_desc->Inputs().at("Shape").size();
    return has_expand_shapes_tensor || has_shape;
  });
}

std::unordered_set<std::string> OpTransInfo::GetDenyVarNames(
    const GraphNodeSet& cluster) const {
  std::unordered_set<std::string> deny_var_set;

  auto get_debug_info = [](const std::unordered_set<std::string>& var_names) {
    std::string debug_info = "[";
    for (auto& var : var_names) {
      debug_info.append(var);
      debug_info.append(", ");
    }
    debug_info.append("]");
{'fraction_non_alphanumeric': 0.13979050821156083, 'fraction_numerical': 0.004526057157636105, 'mean_word_length': 4.211590296495957, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3950801', 'n_tokens_mistral': 2558, 'n_tokens_neox': 2494, 'n_words': 551}