<commit_before>// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

#include "core/session/onnxruntime_c_api.h"
#include "core/session/onnxruntime_cxx_api.h"
#include "core/session/inference_session.h"
#include "core/session/ort_env.h"
#include "asserts.h"
#include <iterator>
#include "gtest/gtest.h"
#include <core/platform/path_lib.h>
#include "test/onnx/TestCase.h"
#include "test/onnx/runner.h"
#include "test/compare_ortvalue.h"
#include "default_providers.h"
#include "test/onnx/onnx_model_info.h"

extern std::unique_ptr<Ort::Env> ort_env;

namespace onnxruntime {
namespace test {
// parameter is provider_name + "_" + model_path
class ModelTest : public testing::TestWithParam<std::basic_string<ORTCHAR_T>> {};
namespace {
struct BrokenTest {
  std::string test_name_;
  std::string reason_;
  std::set<std::string> broken_versions_ = {};  // apply to all versions if empty
  BrokenTest(std::string name, std::string reason) : test_name_(std::move(name)), reason_(std::move(reason)) {
  }

  BrokenTest(std::string name, std::string reason, const std::initializer_list<std::string>& versions)
      : test_name_(std::move(name)), reason_(std::move(reason)), broken_versions_(versions) {
  }

  bool operator<(const struct BrokenTest& test) const {
    return strcmp(test_name_.c_str(), test.test_name_.c_str()) < 0;
  }
};
}  // namespace
TEST_P(ModelTest, Run) {
  std::basic_string<ORTCHAR_T> param = GetParam();
  size_t pos = param.find(ORT_TSTR("_"));
  ASSERT_NE(pos, std::string::npos);
  std::string provider_name = ToMBString(param.substr(0, pos));
  std::basic_string<ORTCHAR_T> model_path = param.substr(pos + 1);
  double per_sample_tolerance = 1e-3;
  // when cuda is enabled, set it to a larger value for resolving random MNIST test failure
  // when openvino is enabled, set it to a larger value for resolving MNIST accuracy mismatch
  double relative_per_sample_tolerance = 1e-3;
  if (provider_name == "openvino") {
    relative_per_sample_tolerance = 0.009;
  }

  std::unique_ptr<OnnxModelInfo> model_info = onnxruntime::make_unique<OnnxModelInfo>(model_path.c_str());
  if (model_info->GetONNXOpSetVersion() != 8 && provider_name == "tensorrt") {
    // TensorRT can run most of the model tests, but only part of
    // them is enabled here to save CI build time.
    return;
  }
  if (model_info->GetONNXOpSetVersion() != 12 && provider_name == "dnnl") {
    // DNNL can run most of the model tests, but only part of
    // them is enabled here to save CI build time.
    return;
  }
#ifndef ENABLE_TRAINING
  if (model_info->HasDomain(ONNX_NAMESPACE::AI_ONNX_TRAINING_DOMAIN) ||
      model_info->HasDomain(ONNX_NAMESPACE::AI_ONNX_PREVIEW_TRAINING_DOMAIN)) {
    return;
  }
#endif
  // TODO: filter model based on opset
  std::set<BrokenTest> broken_tests = {
      {"mnist", "Input data isn't in valid range"},
      {"BERT_Squad", "test data bug"},
      {"constantofshape_float_ones", "test data bug", {"onnx141", "onnx150"}},
      {"constantofshape_int_zeros", "test data bug", {"onnx141", "onnx150"}},
      {"cast_STRING_to_FLOAT", "Linux CI has old ONNX python package with bad test data", {"onnx141"}},
      // Numpy float to string has unexpected rounding for some results given numpy default precision is meant to be 8.
      // "e.g. 0.296140194 -> '0.2961402' not '0.29614019'. ORT produces the latter with precision set to 8,
      // which doesn't match the expected output that was generated with numpy.
      {"cast_FLOAT_to_STRING", "Numpy float to string has unexpected rounding for some results."},
      {"tf_nasnet_large", "disable temporarily"},
      {"tf_nasnet_mobile", "disable temporarily"},<commit_msg>dnnl only run opset 10 model tests to reduce footprint/runtime (#4663)<commit_after>// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

#include "core/session/onnxruntime_c_api.h"
#include "core/session/onnxruntime_cxx_api.h"
#include "core/session/inference_session.h"
#include "core/session/ort_env.h"
#include "asserts.h"
#include <iterator>
#include "gtest/gtest.h"
#include <core/platform/path_lib.h>
#include "test/onnx/TestCase.h"
#include "test/onnx/runner.h"
#include "test/compare_ortvalue.h"
#include "default_providers.h"
#include "test/onnx/onnx_model_info.h"

extern std::unique_ptr<Ort::Env> ort_env;

namespace onnxruntime {
namespace test {
// parameter is provider_name + "_" + model_path
class ModelTest : public testing::TestWithParam<std::basic_string<ORTCHAR_T>> {};
namespace {
struct BrokenTest {
  std::string test_name_;
  std::string reason_;
  std::set<std::string> broken_versions_ = {};  // apply to all versions if empty
  BrokenTest(std::string name, std::string reason) : test_name_(std::move(name)), reason_(std::move(reason)) {
  }

  BrokenTest(std::string name, std::string reason, const std::initializer_list<std::string>& versions)
      : test_name_(std::move(name)), reason_(std::move(reason)), broken_versions_(versions) {
  }

  bool operator<(const struct BrokenTest& test) const {
    return strcmp(test_name_.c_str(), test.test_name_.c_str()) < 0;
  }
};
}  // namespace
TEST_P(ModelTest, Run) {
  std::basic_string<ORTCHAR_T> param = GetParam();
  size_t pos = param.find(ORT_TSTR("_"));
  ASSERT_NE(pos, std::string::npos);
  std::string provider_name = ToMBString(param.substr(0, pos));
  std::basic_string<ORTCHAR_T> model_path = param.substr(pos + 1);
  double per_sample_tolerance = 1e-3;
  // when cuda is enabled, set it to a larger value for resolving random MNIST test failure
  // when openvino is enabled, set it to a larger value for resolving MNIST accuracy mismatch
  double relative_per_sample_tolerance = 1e-3;
  if (provider_name == "openvino") {
    relative_per_sample_tolerance = 0.009;
  }

  std::unique_ptr<OnnxModelInfo> model_info = onnxruntime::make_unique<OnnxModelInfo>(model_path.c_str());
  if (model_info->GetONNXOpSetVersion() != 8 && provider_name == "tensorrt") {
    // TensorRT can run most of the model tests, but only part of
    // them is enabled here to save CI build time.
    return;
  }
  if (model_info->GetONNXOpSetVersion() == 10 && provider_name == "dnnl") {
    // DNNL can run most of the model tests, but only part of
    // them is enabled here to save CI build time.
    return;
  }
#ifndef ENABLE_TRAINING
  if (model_info->HasDomain(ONNX_NAMESPACE::AI_ONNX_TRAINING_DOMAIN) ||
      model_info->HasDomain(ONNX_NAMESPACE::AI_ONNX_PREVIEW_TRAINING_DOMAIN)) {
    return;
  }
#endif
  // TODO: filter model based on opset
  std::set<BrokenTest> broken_tests = {
      {"mnist", "Input data isn't in valid range"},
      {"BERT_Squad", "test data bug"},
      {"constantofshape_float_ones", "test data bug", {"onnx141", "onnx150"}},
      {"constantofshape_int_zeros", "test data bug", {"onnx141", "onnx150"}},
      {"cast_STRING_to_FLOAT", "Linux CI has old ONNX python package with bad test data", {"onnx141"}},
      // Numpy float to string has unexpected rounding for some results given numpy default precision is meant to be 8.
      // "e.g. 0.296140194 -> '0.2961402' not '0.29614019'. ORT produces the latter with precision set to 8,
      // which doesn't match the expected output that was generated with numpy.
      {"cast_FLOAT_to_STRING", "Numpy float to string has unexpected rounding for some results."},
      {"tf_nasnet_large", "disable temporarily"},
      {"tf_nasnet_mobile", "disable temporarily"},
{'fraction_non_alphanumeric': 0.12923076923076923, 'fraction_numerical': 0.01632107023411371, 'mean_word_length': 4.629518072289157, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7549095', 'n_tokens_mistral': 2579, 'n_tokens_neox': 2463, 'n_words': 659}