<commit_before>#include "chainerx/python/array.h"

#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <memory>
#include <string>
#include <tuple>
#include <utility>
#include <vector>

#include <absl/types/optional.h>
#include <pybind11/operators.h>

#include "chainerx/array.h"
#include "chainerx/array_body.h"
#include "chainerx/array_index.h"
#include "chainerx/axes.h"
#include "chainerx/backend_util.h"
#include "chainerx/backward.h"
#include "chainerx/constant.h"
#include "chainerx/context.h"
#include "chainerx/device.h"
#include "chainerx/dtype.h"
#include "chainerx/error.h"
#include "chainerx/graph.h"
#include "chainerx/indexable_array.h"
#include "chainerx/indexer.h"
#include "chainerx/native/data_type.h"
#include "chainerx/native/native_backend.h"
#include "chainerx/routines/arithmetic.h"
#include "chainerx/routines/creation.h"
#include "chainerx/routines/indexing.h"
#include "chainerx/routines/manipulation.h"
#include "chainerx/routines/misc.h"
#include "chainerx/routines/sorting.h"
#include "chainerx/shape.h"
#include "chainerx/slice.h"
#include "chainerx/strides.h"

#include "chainerx/python/array_index.h"
#include "chainerx/python/axes.h"
#include "chainerx/python/common.h"
#include "chainerx/python/device.h"
#include "chainerx/python/dtype.h"
#include "chainerx/python/py_cached_objects.h"
#include "chainerx/python/shape.h"
#include "chainerx/python/strides.h"

namespace chainerx {
namespace python {
namespace python_internal {
namespace {

using internal::GetArrayBody;
using internal::MoveArrayBody;

}  // namespace

namespace py = pybind11;
using py::literals::operator""_a;

py::tuple ToTuple(const std::vector<Array>& ary) {
    py::tuple ret{ary.size()};
    for (size_t i = 0; i < ary.size(); i++) {
        ret[i] = GetArrayBody(ary[i]);
    }
    return ret;
}

std::vector<ArrayBodyPtr> ToArrayBodyPtr(const std::vector<Array>& ary) {
    std::vector<ArrayBodyPtr> ret{ary.size()};
    for (size_t i = 0; i < ary.size(); i++) {
        ArrayBodyPtr array_body = GetArrayBody(ary[i]);
        ret[i] = std::move(array_body);
    }
    return ret;
}

ArrayBodyPtr MakeArrayFromNumpyArray(py::array array, Device& device) {
    Shape shape{array.shape(), array.shape() + array.ndim()};
    Dtype dtype = GetDtypeFromNumpyDtype(array.dtype());
    Strides strides{array.strides(), array.strides() + array.ndim()};

    int64_t first{};
    int64_t last{};
    std::tie(first, last) = GetDataRange(shape, strides, array.itemsize());
    py::buffer_info info = array.request();

    // Some backends may perform zero copy, so increment refcount of numpy ndarray not to be released in user codes.
    // Note that inc_ref() / dec_ref() is performed by the lambda capture.
    std::shared_ptr<void> data{static_cast<char*>(info.ptr) + first, [array](void*) {}};

    return MoveArrayBody(internal::FromHostData(shape, dtype, data, strides, -first, device));
}

namespace {

py::array MakeNumpyArrayFromArray(const py::module& m, const ArrayBodyPtr& self, bool copy) {
    Array array = Array{self}.ToNative();

    py::object dtype = GetNumpyDtypeFromModule(m, array.dtype());
    const Shape& shape = array.shape();
    const Strides& strides = array.strides();
    const void* ptr = internal::GetRawOffsetData(array);

    if (copy) {
        return py::array{dtype, shape, strides, ptr};
    }
    return py::array{dtype, shape, strides, ptr, py::cast(internal::MoveArrayBody(std::move(array)))};
}

// TODO(okapies): this is a workaround for improving performance
py::object MakeCupyArrayFromArray(const py::module& m, py::handle self) {
    Array array{py::cast<ArrayBodyPtr>(self)};
    Device& device = array.device();
    // TODO(okapies): rejects if array's device is not compatible with cupy

    py::object dtype = GetNumpyDtypeFromModule(m, array.dtype());
    const Shape& shape = array.shape();
    const Strides& strides = array.strides();

    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
    const intptr_t ptr = reinterpret_cast<intptr_t>(array.raw_data());
    const auto range = GetDataRange(shape, strides, array.GetItemSize());
    const auto data_size = std::get<1>(range) - std::get<0>(range);
    const auto device_index = device.index();

    // Convert object to CuPy array using cupy.ndarray()
    auto memory_pointer = GetCachedCupyMemoryPointer();
    auto unowned_memory = GetCachedCupyUnownedMemory();
    py::object memptr = memory_pointer(unowned_memory(ptr, data_size, self, device_index), array.offset());

    auto ndarray = GetCachedCupyNdarray();
    return ndarray(ToTuple(shape), dtype, memptr, ToTuple(strides));
}

}  // namespace

ArrayBodyPtr MakeArray(py::handle object, py::handle dtype, bool copy, py::handle device) {
    absl::optional<Dtype> dtype_ = dtype.is_none() ? absl::nullopt : absl::optional<Dtype>(GetDtype(dtype));
    Device& dev = GetDevice(device);

    return MakeArray(object, dtype_, copy, dev);<commit_msg>Forward chainerx::MakeArray in some case<commit_after>#include "chainerx/python/array.h"

#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <memory>
#include <string>
#include <tuple>
#include <utility>
#include <vector>

#include <absl/types/optional.h>
#include <pybind11/operators.h>

#include "chainerx/array.h"
#include "chainerx/array_body.h"
#include "chainerx/array_index.h"
#include "chainerx/axes.h"
#include "chainerx/backend_util.h"
#include "chainerx/backward.h"
#include "chainerx/constant.h"
#include "chainerx/context.h"
#include "chainerx/device.h"
#include "chainerx/dtype.h"
#include "chainerx/error.h"
#include "chainerx/graph.h"
#include "chainerx/indexable_array.h"
#include "chainerx/indexer.h"
#include "chainerx/native/data_type.h"
#include "chainerx/native/native_backend.h"
#include "chainerx/routines/arithmetic.h"
#include "chainerx/routines/creation.h"
#include "chainerx/routines/indexing.h"
#include "chainerx/routines/manipulation.h"
#include "chainerx/routines/misc.h"
#include "chainerx/routines/sorting.h"
#include "chainerx/shape.h"
#include "chainerx/slice.h"
#include "chainerx/strides.h"

#include "chainerx/python/array_index.h"
#include "chainerx/python/axes.h"
#include "chainerx/python/common.h"
#include "chainerx/python/device.h"
#include "chainerx/python/dtype.h"
#include "chainerx/python/py_cached_objects.h"
#include "chainerx/python/shape.h"
#include "chainerx/python/strides.h"

namespace chainerx {
namespace python {
namespace python_internal {
namespace {

using internal::GetArrayBody;
using internal::MoveArrayBody;

}  // namespace

namespace py = pybind11;
using py::literals::operator""_a;

py::tuple ToTuple(const std::vector<Array>& ary) {
    py::tuple ret{ary.size()};
    for (size_t i = 0; i < ary.size(); i++) {
        ret[i] = GetArrayBody(ary[i]);
    }
    return ret;
}

std::vector<ArrayBodyPtr> ToArrayBodyPtr(const std::vector<Array>& ary) {
    std::vector<ArrayBodyPtr> ret{ary.size()};
    for (size_t i = 0; i < ary.size(); i++) {
        ArrayBodyPtr array_body = GetArrayBody(ary[i]);
        ret[i] = std::move(array_body);
    }
    return ret;
}

ArrayBodyPtr MakeArrayFromNumpyArray(py::array array, Device& device) {
    Shape shape{array.shape(), array.shape() + array.ndim()};
    Dtype dtype = GetDtypeFromNumpyDtype(array.dtype());
    Strides strides{array.strides(), array.strides() + array.ndim()};

    int64_t first{};
    int64_t last{};
    std::tie(first, last) = GetDataRange(shape, strides, array.itemsize());
    py::buffer_info info = array.request();

    // Some backends may perform zero copy, so increment refcount of numpy ndarray not to be released in user codes.
    // Note that inc_ref() / dec_ref() is performed by the lambda capture.
    std::shared_ptr<void> data{static_cast<char*>(info.ptr) + first, [array](void*) {}};

    return MoveArrayBody(internal::FromHostData(shape, dtype, data, strides, -first, device));
}

namespace {

py::array MakeNumpyArrayFromArray(const py::module& m, const ArrayBodyPtr& self, bool copy) {
    Array array = Array{self}.ToNative();

    py::object dtype = GetNumpyDtypeFromModule(m, array.dtype());
    const Shape& shape = array.shape();
    const Strides& strides = array.strides();
    const void* ptr = internal::GetRawOffsetData(array);

    if (copy) {
        return py::array{dtype, shape, strides, ptr};
    }
    return py::array{dtype, shape, strides, ptr, py::cast(internal::MoveArrayBody(std::move(array)))};
}

// TODO(okapies): this is a workaround for improving performance
py::object MakeCupyArrayFromArray(const py::module& m, py::handle self) {
    Array array{py::cast<ArrayBodyPtr>(self)};
    Device& device = array.device();
    // TODO(okapies): rejects if array's device is not compatible with cupy

    py::object dtype = GetNumpyDtypeFromModule(m, array.dtype());
    const Shape& shape = array.shape();
    const Strides& strides = array.strides();

    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
    const intptr_t ptr = reinterpret_cast<intptr_t>(array.raw_data());
    const auto range = GetDataRange(shape, strides, array.GetItemSize());
    const auto data_size = std::get<1>(range) - std::get<0>(range);
    const auto device_index = device.index();

    // Convert object to CuPy array using cupy.ndarray()
    auto memory_pointer = GetCachedCupyMemoryPointer();
    auto unowned_memory = GetCachedCupyUnownedMemory();
    py::object memptr = memory_pointer(unowned_memory(ptr, data_size, self, device_index), array.offset());

    auto ndarray = GetCachedCupyNdarray();
    return ndarray(ToTuple(shape), dtype, memptr, ToTuple(strides));
}

}  // namespace

ArrayBodyPtr MakeArray(py::handle object, py::handle dtype, bool copy, py::handle device) {
    if (!copy && py::isinstance<ArrayBody>(object) && device.is_none() && dtype.is_none()) {
        return py::cast<ArrayBodyPtr>(object);
    }

    absl::optional<Dtype> dtype_ = dtype.is_none() ? absl::nullopt : absl::optional<Dtype>(GetDtype(dtype));
    Device& dev = GetDevice(device);

    return MakeArray(object, dtype_, copy, dev);
{'fraction_non_alphanumeric': 0.14884737678855325, 'fraction_numerical': 0.0023847376788553257, 'mean_word_length': 4.976840855106889, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5508397', 'n_tokens_mistral': 3508, 'n_tokens_neox': 3438, 'n_words': 738}