<commit_before>// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

#include "pch.h"
#include "ImageFeatureValue.h"
#include "LearningModelBinding.h"
#include "LearningModelDevice.h"
#include "LearningModelSession.h"
#include <windows.media.h>
#include <wrl\wrappers\corewrappers.h>
#include "LearningModelBinding.h"
#include "LearningModelSession.h"
#include "LearningModelDevice.h"
#include "ImageConversionTypes.h"
#include "ConverterResourceStore.h"
#include "ImageFeatureDescriptor.h"

#include "core/session/onnxruntime_c_api.h"

#include "D3DDeviceCache.h"
#include "TensorFeatureDescriptor.h"

using namespace WinML;
using namespace winrt::Windows::Graphics::Imaging;
using namespace winrt::Windows::Graphics::DirectX::Direct3D11;
using namespace winrt::Windows::Graphics::DirectX;
using namespace Windows::AI::MachineLearning::Internal;
using namespace winrt::Windows::Foundation::Collections;

namespace winrt::Windows::AI::MachineLearning::implementation {

struct ImageFeatureValue::ImageResourceMetadata {
  std::vector<Windows::Graphics::Imaging::BitmapBounds> Bounds;
  ::Windows::AI::MachineLearning::Internal::ImageTensorDescription TensorDescriptor;
};

Windows::AI::MachineLearning::ImageFeatureValue ImageFeatureValue::Create(
    uint32_t batchSize,
    BitmapPixelFormat format,
    uint32_t width,
    uint32_t height) {
  std::vector<Windows::Media::VideoFrame> videoFrames = {};
  for (uint32_t i = 0; i < batchSize; ++i) {
    SoftwareBitmap bitmap(format, width, height);
    Windows::Media::VideoFrame frame = Windows::Media::VideoFrame::CreateWithSoftwareBitmap(bitmap);
    videoFrames.emplace_back(frame);
  }
  return make<ImageFeatureValue>(winrt::single_threaded_vector(std::move(videoFrames)));
}

Windows::AI::MachineLearning::ImageFeatureValue ImageFeatureValue::CreateFromVideoFrame(Windows::Media::VideoFrame const& image) try {
  return make<ImageFeatureValue>(image);
}
WINML_CATCH_ALL

void ImageFeatureValue::Initialize() {
  m_batchSize = m_videoFrames.Size();
  for (auto videoFrame : m_videoFrames) {
    // TODO: Check all videoFrames come from either CPU or GPU.
    if (auto surface = videoFrame.Direct3DSurface()) {
      Direct3DSurfaceDescription description = surface.Description();
      m_widths.emplace_back(description.Width);
      m_heights.emplace_back(description.Height);
    } else {
      ISoftwareBitmap softwarebitmap(videoFrame.SoftwareBitmap());
      m_widths.emplace_back(softwarebitmap.PixelWidth());
      m_heights.emplace_back(softwarebitmap.PixelHeight());
    }
  }
}

ImageFeatureValue::ImageFeatureValue(Windows::Media::VideoFrame const& image) {
  std::vector<Windows::Media::VideoFrame> frame = {image};
  m_videoFrames = winrt::single_threaded_vector(std::move(frame));
  Initialize();
}

ImageFeatureValue::ImageFeatureValue(IVector<Windows::Media::VideoFrame> const& images) : m_videoFrames(images) {
  Initialize();
}

ImageFeatureValue::ImageFeatureValue(IVectorView<Windows::Media::VideoFrame> const& images) {
  std::vector<Windows::Media::VideoFrame> videoFrames = {};
  for (uint32_t i = 0; i < images.Size(); ++i) {
    videoFrames.emplace_back(images.GetAt(i));
  }
  m_videoFrames = winrt::single_threaded_vector(std::move(videoFrames));
  Initialize();
}

static std::optional<BitmapPixelFormat> GetBitmapPixelFormatFromMetadata(const IPropertySet& properties) {
  if (properties != nullptr && properties.HasKey(L"BitmapPixelFormat")) {
    if (auto pixelFormatInspectable = properties.Lookup(L"BitmapPixelFormat")) {
      auto pixelFormatValue = pixelFormatInspectable.as<Windows::Foundation::IPropertyValue>();
      auto pixelFormat = static_cast<BitmapPixelFormat>(pixelFormatValue.GetInt32());
      WINML_THROW_HR_IF_FALSE_MSG(
          WINML_ERR_INVALID_BINDING,
          pixelFormat == BitmapPixelFormat::Rgba8 ||
              pixelFormat == BitmapPixelFormat::Bgra8 ||
              pixelFormat == BitmapPixelFormat::Gray8,
          "BitmapPixelFormat must be either Rgba8, Bgra8, or Gray8");

      return pixelFormat;
    }
  }

  return {};
}

static std::optional<BitmapBounds> GetBoundsFromMetadata(const IPropertySet& properties) {
  if (properties != nullptr && properties.HasKey(L"BitmapBounds")) {
    if (auto boundsInspectable = properties.Lookup(L"BitmapBounds")) {
      auto boundsPropertyValue = boundsInspectable.as<Windows::Foundation::IPropertyValue>();
      WINML_THROW_HR_IF_FALSE_MSG(
          WINML_ERR_INVALID_BINDING,
          boundsPropertyValue.Type() == Windows::Foundation::PropertyType::UInt32Array,
          "BitmapBounds must reference a property value with type UInt32Array with 4 elements.");

      com_array<uint32_t> bounds;
      boundsPropertyValue.GetUInt32Array(bounds);
      WINML_THROW_HR_IF_FALSE_MSG(
          WINML_ERR_INVALID_BINDING,
          bounds.size() == 4,
          "BitmapBounds must reference a property value with type UInt32Array with 4 elements.");

      return Windows::Graphics::Imaging::BitmapBounds{bounds[0], bounds[1], bounds[2], bounds[3]};
    }
  }

  return {};
}

BitmapBounds ImageFeatureValue::CenterAndCropBounds(
    uint32_t idx,
    uint32_t desiredWidth,
    uint32_t desiredHeight) {
  BitmapBounds bounds = {};
  float RequiredAspectRatio = static_cast<float>(desiredWidth) / static_cast<float>(desiredHeight);

  // crop to center while maintaining size
  if (RequiredAspectRatio * m_heights[idx] < m_widths[idx]) {
    // actual width is too wide. Cut off left and right of image
    bounds.Width = std::min((UINT)(RequiredAspectRatio * m_heights[idx] + 0.5f), m_widths[idx]);
    bounds.Height = m_heights[idx];
    bounds.X = (m_widths[idx] - bounds.Width) / 2;
    bounds.Y = 0;
  } else {
    // actual height is too long. Cut off top and bottom
    bounds.Width = m_widths[idx];
    bounds.Height = std::min((UINT)(m_widths[idx] / RequiredAspectRatio + 0.5f), m_heights[idx]);
    bounds.X = 0;
    bounds.Y = (m_heights[idx] - bounds.Height) / 2;
  }

  // TODO: Do we allow smaller images?
  WINML_THROW_HR_IF_FALSE_MSG(
      WINML_ERR_INVALID_BINDING,
      (bounds.X >= 0 && bounds.X <= m_widths[idx]) &&
          (bounds.Y >= 0 && bounds.Y <= m_heights[idx]),
      "Failed to center crop the provided input image. The calculated bounds exceed the dimensions of the image, or do not match the model inputs dimensions.");

  return bounds;
}

static ImageTensorDataType GetTensorDataTypeFromTensorKind(TensorKind kind) {
  switch (kind) {
    case TensorKind::Float:
      return kImageTensorDataTypeFloat32;
    case TensorKind::Float16:
      return kImageTensorDataTypeFloat16;
    default:
      WINML_THROW_HR_IF_FALSE_MSG(WINML_ERR_INVALID_BINDING, false, "Model image inputs must have tensor type of Float or Float16.");
  }

  FAIL_FAST_HR(E_INVALIDARG);
}

static unsigned GetSizeFromTensorDataType(ImageTensorDataType type) {
  switch (type) {
    case kImageTensorDataTypeFloat32:
      return sizeof(float);
    case kImageTensorDataTypeFloat16:
      return sizeof(uint16_t);
    default:
      WINML_THROW_HR_IF_FALSE_MSG(WINML_ERR_INVALID_BINDING, false, "Model image inputs must have tensor type of Float or Float16.");
  }

  FAIL_FAST_HR(E_INVALIDARG);
}<commit_msg>checking condition is wrong (#3025)<commit_after>// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

#include "pch.h"
#include "ImageFeatureValue.h"
#include "LearningModelBinding.h"
#include "LearningModelDevice.h"
#include "LearningModelSession.h"
#include <windows.media.h>
#include <wrl\wrappers\corewrappers.h>
#include "LearningModelBinding.h"
#include "LearningModelSession.h"
#include "LearningModelDevice.h"
#include "ImageConversionTypes.h"
#include "ConverterResourceStore.h"
#include "ImageFeatureDescriptor.h"

#include "core/session/onnxruntime_c_api.h"

#include "D3DDeviceCache.h"
#include "TensorFeatureDescriptor.h"

using namespace WinML;
using namespace winrt::Windows::Graphics::Imaging;
using namespace winrt::Windows::Graphics::DirectX::Direct3D11;
using namespace winrt::Windows::Graphics::DirectX;
using namespace Windows::AI::MachineLearning::Internal;
using namespace winrt::Windows::Foundation::Collections;

namespace winrt::Windows::AI::MachineLearning::implementation {

struct ImageFeatureValue::ImageResourceMetadata {
  std::vector<Windows::Graphics::Imaging::BitmapBounds> Bounds;
  ::Windows::AI::MachineLearning::Internal::ImageTensorDescription TensorDescriptor;
};

Windows::AI::MachineLearning::ImageFeatureValue ImageFeatureValue::Create(
    uint32_t batchSize,
    BitmapPixelFormat format,
    uint32_t width,
    uint32_t height) {
  std::vector<Windows::Media::VideoFrame> videoFrames = {};
  for (uint32_t i = 0; i < batchSize; ++i) {
    SoftwareBitmap bitmap(format, width, height);
    Windows::Media::VideoFrame frame = Windows::Media::VideoFrame::CreateWithSoftwareBitmap(bitmap);
    videoFrames.emplace_back(frame);
  }
  return make<ImageFeatureValue>(winrt::single_threaded_vector(std::move(videoFrames)));
}

Windows::AI::MachineLearning::ImageFeatureValue ImageFeatureValue::CreateFromVideoFrame(Windows::Media::VideoFrame const& image) try {
  return make<ImageFeatureValue>(image);
}
WINML_CATCH_ALL

void ImageFeatureValue::Initialize() {
  m_batchSize = m_videoFrames.Size();
  for (auto videoFrame : m_videoFrames) {
    // TODO: Check all videoFrames come from either CPU or GPU.
    if (auto surface = videoFrame.Direct3DSurface()) {
      Direct3DSurfaceDescription description = surface.Description();
      m_widths.emplace_back(description.Width);
      m_heights.emplace_back(description.Height);
    } else {
      ISoftwareBitmap softwarebitmap(videoFrame.SoftwareBitmap());
      m_widths.emplace_back(softwarebitmap.PixelWidth());
      m_heights.emplace_back(softwarebitmap.PixelHeight());
    }
  }
}

ImageFeatureValue::ImageFeatureValue(Windows::Media::VideoFrame const& image) {
  std::vector<Windows::Media::VideoFrame> frame = {image};
  m_videoFrames = winrt::single_threaded_vector(std::move(frame));
  Initialize();
}

ImageFeatureValue::ImageFeatureValue(IVector<Windows::Media::VideoFrame> const& images) : m_videoFrames(images) {
  Initialize();
}

ImageFeatureValue::ImageFeatureValue(IVectorView<Windows::Media::VideoFrame> const& images) {
  std::vector<Windows::Media::VideoFrame> videoFrames = {};
  for (uint32_t i = 0; i < images.Size(); ++i) {
    videoFrames.emplace_back(images.GetAt(i));
  }
  m_videoFrames = winrt::single_threaded_vector(std::move(videoFrames));
  Initialize();
}

static std::optional<BitmapPixelFormat> GetBitmapPixelFormatFromMetadata(const IPropertySet& properties) {
  if (properties != nullptr && properties.HasKey(L"BitmapPixelFormat")) {
    if (auto pixelFormatInspectable = properties.Lookup(L"BitmapPixelFormat")) {
      auto pixelFormatValue = pixelFormatInspectable.as<Windows::Foundation::IPropertyValue>();
      auto pixelFormat = static_cast<BitmapPixelFormat>(pixelFormatValue.GetInt32());
      WINML_THROW_HR_IF_FALSE_MSG(
          WINML_ERR_INVALID_BINDING,
          pixelFormat == BitmapPixelFormat::Rgba8 ||
              pixelFormat == BitmapPixelFormat::Bgra8 ||
              pixelFormat == BitmapPixelFormat::Gray8,
          "BitmapPixelFormat must be either Rgba8, Bgra8, or Gray8");

      return pixelFormat;
    }
  }

  return {};
}

static std::optional<BitmapBounds> GetBoundsFromMetadata(const IPropertySet& properties) {
  if (properties != nullptr && properties.HasKey(L"BitmapBounds")) {
    if (auto boundsInspectable = properties.Lookup(L"BitmapBounds")) {
      auto boundsPropertyValue = boundsInspectable.as<Windows::Foundation::IPropertyValue>();
      WINML_THROW_HR_IF_FALSE_MSG(
          WINML_ERR_INVALID_BINDING,
          boundsPropertyValue.Type() == Windows::Foundation::PropertyType::UInt32Array,
          "BitmapBounds must reference a property value with type UInt32Array with 4 elements.");

      com_array<uint32_t> bounds;
      boundsPropertyValue.GetUInt32Array(bounds);
      WINML_THROW_HR_IF_FALSE_MSG(
          WINML_ERR_INVALID_BINDING,
          bounds.size() == 4,
          "BitmapBounds must reference a property value with type UInt32Array with 4 elements.");

      return Windows::Graphics::Imaging::BitmapBounds{bounds[0], bounds[1], bounds[2], bounds[3]};
    }
  }

  return {};
}

BitmapBounds ImageFeatureValue::CenterAndCropBounds(
    uint32_t idx,
    uint32_t desiredWidth,
    uint32_t desiredHeight) {
  BitmapBounds bounds = {};
  float RequiredAspectRatio = static_cast<float>(desiredWidth) / static_cast<float>(desiredHeight);

  // crop to center while maintaining size
  if (RequiredAspectRatio * m_heights[idx] < m_widths[idx]) {
    // actual width is too wide. Cut off left and right of image
    bounds.Width = std::min((UINT)(RequiredAspectRatio * m_heights[idx] + 0.5f), m_widths[idx]);
    bounds.Height = m_heights[idx];
    bounds.X = (m_widths[idx] - bounds.Width) / 2;
    bounds.Y = 0;
  } else {
    // actual height is too long. Cut off top and bottom
    bounds.Width = m_widths[idx];
    bounds.Height = std::min((UINT)(m_widths[idx] / RequiredAspectRatio + 0.5f), m_heights[idx]);
    bounds.X = 0;
    bounds.Y = (m_heights[idx] - bounds.Height) / 2;
  }

  return bounds;
}

static ImageTensorDataType GetTensorDataTypeFromTensorKind(TensorKind kind) {
  switch (kind) {
    case TensorKind::Float:
      return kImageTensorDataTypeFloat32;
    case TensorKind::Float16:
      return kImageTensorDataTypeFloat16;
    default:
      WINML_THROW_HR_IF_FALSE_MSG(WINML_ERR_INVALID_BINDING, false, "Model image inputs must have tensor type of Float or Float16.");
  }

  FAIL_FAST_HR(E_INVALIDARG);
}

static unsigned GetSizeFromTensorDataType(ImageTensorDataType type) {
  switch (type) {
    case kImageTensorDataTypeFloat32:
      return sizeof(float);
    case kImageTensorDataTypeFloat16:
      return sizeof(uint16_t);
    default:
      WINML_THROW_HR_IF_FALSE_MSG(WINML_ERR_INVALID_BINDING, false, "Model image inputs must have tensor type of Float or Float16.");
  }

  FAIL_FAST_HR(E_INVALIDARG);
}
{'fraction_non_alphanumeric': 0.11469254746387078, 'fraction_numerical': 0.010767922924341174, 'mean_word_length': 5.21073471183458, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6797618', 'n_tokens_mistral': 4391, 'n_tokens_neox': 4233, 'n_words': 841}