<commit_before>#include "xchainer/array_repr.h"

#include <gtest/gtest.h>

#include <algorithm>
#include <cmath>
#include <limits>
#include <sstream>
#include <string>
#include <vector>

#include "xchainer/device.h"
#include "xchainer/native_backend.h"

namespace xchainer {
namespace {

template <typename T>
void CheckArrayRepr(const std::string& expected, const std::vector<T>& data_vec, Shape shape, const Device& device,
                    const std::vector<GraphId> graph_ids = {}) {
    // Copy to a contiguous memory block because std::vector<bool> is not packed as a sequence of bool's.
    std::shared_ptr<T> data_ptr = std::make_unique<T[]>(data_vec.size());
    std::copy(data_vec.begin(), data_vec.end(), data_ptr.get());
    Array array = Array::FromBuffer(shape, TypeToDtype<T>, static_cast<std::shared_ptr<void>>(data_ptr), device);
    for (const GraphId& graph_id : graph_ids) {
        array.RequireGrad(graph_id);
    }

    // std::string version
    EXPECT_EQ(expected, ArrayRepr(array));

    // std::ostream version
    std::ostringstream os;
    os << array;
    EXPECT_EQ(expected, os.str());
}

TEST(ArrayReprTest, AllDtypesOnNativeBackend) {
    NativeBackend backend;

    // bool
    CheckArrayRepr<bool>("array([False], dtype=bool, device='cpu')", {false}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<bool>("array([ True], dtype=bool, device='cpu')", {true}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<bool>(
        "array([[False,  True,  True],\n"
        "       [ True, False,  True]], dtype=bool, device='cpu')",
        {false, true, true, true, false, true}, Shape({2, 3}), Device{"cpu", &backend});
    CheckArrayRepr<bool>("array([[[[ True]]]], dtype=bool, device='cpu')", {true}, Shape({1, 1, 1, 1}), Device{"cpu", &backend});

    // int8
    CheckArrayRepr<int8_t>("array([0], dtype=int8, device='cpu')", {0}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<int8_t>("array([-2], dtype=int8, device='cpu')", {-2}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<int8_t>(
        "array([[0, 1, 2],\n"
        "       [3, 4, 5]], dtype=int8, device='cpu')",
        {0, 1, 2, 3, 4, 5}, Shape({2, 3}), Device{"cpu", &backend});
    CheckArrayRepr<int8_t>(
        "array([[ 0,  1,  2],\n"
        "       [-3,  4,  5]], dtype=int8, device='cpu')",
        {0, 1, 2, -3, 4, 5}, Shape({2, 3}), Device{"cpu", &backend});
    CheckArrayRepr<int8_t>("array([[[[3]]]], dtype=int8, device='cpu')", {3}, Shape({1, 1, 1, 1}), Device{"cpu", &backend});

    // int16
    CheckArrayRepr<int16_t>("array([0], dtype=int16, device='cpu')", {0}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<int16_t>("array([-2], dtype=int16, device='cpu')", {-2}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<int16_t>(
        "array([[0, 1, 2],\n"
        "       [3, 4, 5]], dtype=int16, device='cpu')",
        {0, 1, 2, 3, 4, 5}, Shape({2, 3}), Device{"cpu", &backend});
    CheckArrayRepr<int16_t>(
        "array([[ 0,  1,  2],\n"
        "       [-3,  4,  5]], dtype=int16, device='cpu')",
        {0, 1, 2, -3, 4, 5}, Shape({2, 3}), Device{"cpu", &backend});
    CheckArrayRepr<int16_t>("array([[[[3]]]], dtype=int16, device='cpu')", {3}, Shape({1, 1, 1, 1}), Device{"cpu", &backend});

    // int32
    CheckArrayRepr<int32_t>("array([0], dtype=int32, device='cpu')", {0}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<int32_t>("array([-2], dtype=int32, device='cpu')", {-2}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<int32_t>(
        "array([[0, 1, 2],\n"
        "       [3, 4, 5]], dtype=int32, device='cpu')",
        {0, 1, 2, 3, 4, 5}, Shape({2, 3}), Device{"cpu", &backend});
    CheckArrayRepr<int32_t>(
        "array([[ 0,  1,  2],\n"
        "       [-3,  4,  5]], dtype=int32, device='cpu')",
        {0, 1, 2, -3, 4, 5}, Shape({2, 3}), Device{"cpu", &backend});
    CheckArrayRepr<int32_t>("array([[[[3]]]], dtype=int32, device='cpu')", {3}, Shape({1, 1, 1, 1}), Device{"cpu", &backend});

    // int64
    CheckArrayRepr<int64_t>("array([0], dtype=int64, device='cpu')", {0}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<int64_t>("array([-2], dtype=int64, device='cpu')", {-2}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<int64_t>(
        "array([[0, 1, 2],\n"
        "       [3, 4, 5]], dtype=int64, device='cpu')",
        {0, 1, 2, 3, 4, 5}, Shape({2, 3}), Device{"cpu", &backend});
    CheckArrayRepr<int64_t>(
        "array([[ 0,  1,  2],\n"
        "       [-3,  4,  5]], dtype=int64, device='cpu')",
        {0, 1, 2, -3, 4, 5}, Shape({2, 3}), Device{"cpu", &backend});
    CheckArrayRepr<int64_t>("array([[[[3]]]], dtype=int64, device='cpu')", {3}, Shape({1, 1, 1, 1}), Device{"cpu", &backend});

    // uint8
    CheckArrayRepr<uint8_t>("array([0], dtype=uint8, device='cpu')", {0}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<uint8_t>("array([2], dtype=uint8, device='cpu')", {2}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<uint8_t>(
        "array([[0, 1, 2],\n"
        "       [3, 4, 5]], dtype=uint8, device='cpu')",
        {0, 1, 2, 3, 4, 5}, Shape({2, 3}), Device{"cpu", &backend});
    CheckArrayRepr<uint8_t>("array([[[[3]]]], dtype=uint8, device='cpu')", {3}, Shape({1, 1, 1, 1}), Device{"cpu", &backend});

    // float32
    CheckArrayRepr<float>("array([0.], dtype=float32, device='cpu')", {0}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<float>("array([3.25], dtype=float32, device='cpu')", {3.25}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<float>("array([-3.25], dtype=float32, device='cpu')", {-3.25}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<float>("array([ inf], dtype=float32, device='cpu')", {std::numeric_limits<float>::infinity()}, Shape({1}),
                          Device{"cpu", &backend});
    CheckArrayRepr<float>("array([ -inf], dtype=float32, device='cpu')", {-std::numeric_limits<float>::infinity()}, Shape({1}),
                          Device{"cpu", &backend});
    CheckArrayRepr<float>("array([ nan], dtype=float32, device='cpu')", {std::nanf("")}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<float>(
        "array([[0., 1., 2.],\n"
        "       [3., 4., 5.]], dtype=float32, device='cpu')",
        {0, 1, 2, 3, 4, 5}, Shape({2, 3}), Device{"cpu", &backend});
    CheckArrayRepr<float>(
        "array([[0.  , 1.  , 2.  ],\n"
        "       [3.25, 4.  , 5.  ]], dtype=float32, device='cpu')",
        {0, 1, 2, 3.25, 4, 5}, Shape({2, 3}), Device{"cpu", &backend});
    CheckArrayRepr<float>(
        "array([[ 0.  ,  1.  ,  2.  ],\n"
        "       [-3.25,  4.  ,  5.  ]], dtype=float32, device='cpu')",
        {0, 1, 2, -3.25, 4, 5}, Shape({2, 3}), Device{"cpu", &backend});
    CheckArrayRepr<float>("array([[[[3.25]]]], dtype=float32, device='cpu')", {3.25}, Shape({1, 1, 1, 1}), Device{"cpu", &backend});

    // float64
    CheckArrayRepr<double>("array([0.], dtype=float64, device='cpu')", {0}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<double>("array([3.25], dtype=float64, device='cpu')", {3.25}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<double>("array([-3.25], dtype=float64, device='cpu')", {-3.25}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<double>("array([ inf], dtype=float64, device='cpu')", {std::numeric_limits<double>::infinity()}, Shape({1}),
                           Device{"cpu", &backend});
    CheckArrayRepr<double>("array([ -inf], dtype=float64, device='cpu')", {-std::numeric_limits<double>::infinity()}, Shape({1}),
                           Device{"cpu", &backend});
    CheckArrayRepr<double>("array([ nan], dtype=float64, device='cpu')", {std::nan("")}, Shape({1}), Device{"cpu", &backend});
    CheckArrayRepr<double>(
        "array([[0., 1., 2.],\n"
        "       [3., 4., 5.]], dtype=float64, device='cpu')",
        {0, 1, 2, 3, 4, 5}, Shape({2, 3}), Device{"cpu", &backend});
    CheckArrayRepr<double>(
        "array([[0.  , 1.  , 2.  ],\n"
        "       [3.25, 4.  , 5.  ]], dtype=float64, device='cpu')",
        {0, 1, 2, 3.25, 4, 5}, Shape({2, 3}), Device{"cpu", &backend});
    CheckArrayRepr<double>(
        "array([[ 0.  ,  1.  ,  2.  ],\n"
        "       [-3.25,  4.  ,  5.  ]], dtype=float64, device='cpu')",
        {0, 1, 2, -3.25, 4, 5}, Shape({2, 3}), Device{"cpu", &backend});
    CheckArrayRepr<double>("array([[[[3.25]]]], dtype=float64, device='cpu')", {3.25}, Shape({1, 1, 1, 1}), Device{"cpu", &backend});

    // Single graph
    CheckArrayRepr<int32_t>("array([-2], dtype=int32, device='cpu', graph_ids=['graph_1'])", {-2}, Shape({1}), Device{"cpu", &backend},
                            {"graph_1"});

    // Two graphs
    CheckArrayRepr<int32_t>("array([1], dtype=int32, device='cpu', graph_ids=['graph_1', 'graph_2'])", {1}, Shape({1}),
                            Device{"cpu", &backend}, {"graph_1", "graph_2"});

    // Multiple graphs
    CheckArrayRepr<int32_t>("array([-9], dtype=int32, device='cpu', graph_ids=['graph_1', 'graph_2', 'graph_3', 'graph_4', 'graph_5'])",
                            {-9}, Shape({1}), Device{"cpu", &backend}, {"graph_1", "graph_2", "graph_3", "graph_4", "graph_5"});
}

}  // namespace
}  // namespace xchainer<commit_msg>Fix array_repr_test to use new DeviceId<commit_after>#include "xchainer/array_repr.h"

#include <gtest/gtest.h>

#include <algorithm>
#include <cmath>
#include <limits>
#include <sstream>
#include <string>
#include <vector>

#include "xchainer/device_id.h"
#include "xchainer/native_backend.h"

namespace xchainer {
namespace {

template <typename T>
void CheckArrayRepr(const std::string& expected, const std::vector<T>& data_vec, Shape shape, const DeviceId& device_id,
                    const std::vector<GraphId> graph_ids = {}) {
    // Copy to a contiguous memory block because std::vector<bool> is not packed as a sequence of bool's.
    std::shared_ptr<T> data_ptr = std::make_unique<T[]>(data_vec.size());
    std::copy(data_vec.begin(), data_vec.end(), data_ptr.get());
    Array array = Array::FromBuffer(shape, TypeToDtype<T>, static_cast<std::shared_ptr<void>>(data_ptr), device_id);
    for (const GraphId& graph_id : graph_ids) {
        array.RequireGrad(graph_id);
    }

    // std::string version
    EXPECT_EQ(expected, ArrayRepr(array));

    // std::ostream version
    std::ostringstream os;
    os << array;
    EXPECT_EQ(expected, os.str());
}

TEST(ArrayReprTest, AllDtypesOnNativeBackend) {
    NativeBackend backend;

    // bool
    CheckArrayRepr<bool>("array([False], dtype=bool, device_id=('cpu', 0))", {false}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<bool>("array([ True], dtype=bool, device_id=('cpu', 0))", {true}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<bool>(
        "array([[False,  True,  True],\n"
        "       [ True, False,  True]], dtype=bool, device_id=('cpu', 0))",
        {false, true, true, true, false, true}, Shape({2, 3}), DeviceId{&backend});
    CheckArrayRepr<bool>("array([[[[ True]]]], dtype=bool, device_id=('cpu', 0))", {true}, Shape({1, 1, 1, 1}), DeviceId{&backend});

    // int8
    CheckArrayRepr<int8_t>("array([0], dtype=int8, device_id=('cpu', 0))", {0}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<int8_t>("array([-2], dtype=int8, device_id=('cpu', 0))", {-2}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<int8_t>(
        "array([[0, 1, 2],\n"
        "       [3, 4, 5]], dtype=int8, device_id=('cpu', 0))",
        {0, 1, 2, 3, 4, 5}, Shape({2, 3}), DeviceId{&backend});
    CheckArrayRepr<int8_t>(
        "array([[ 0,  1,  2],\n"
        "       [-3,  4,  5]], dtype=int8, device_id=('cpu', 0))",
        {0, 1, 2, -3, 4, 5}, Shape({2, 3}), DeviceId{&backend});
    CheckArrayRepr<int8_t>("array([[[[3]]]], dtype=int8, device_id=('cpu', 0))", {3}, Shape({1, 1, 1, 1}), DeviceId{&backend});

    // int16
    CheckArrayRepr<int16_t>("array([0], dtype=int16, device_id=('cpu', 0))", {0}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<int16_t>("array([-2], dtype=int16, device_id=('cpu', 0))", {-2}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<int16_t>(
        "array([[0, 1, 2],\n"
        "       [3, 4, 5]], dtype=int16, device_id=('cpu', 0))",
        {0, 1, 2, 3, 4, 5}, Shape({2, 3}), DeviceId{&backend});
    CheckArrayRepr<int16_t>(
        "array([[ 0,  1,  2],\n"
        "       [-3,  4,  5]], dtype=int16, device_id=('cpu', 0))",
        {0, 1, 2, -3, 4, 5}, Shape({2, 3}), DeviceId{&backend});
    CheckArrayRepr<int16_t>("array([[[[3]]]], dtype=int16, device_id=('cpu', 0))", {3}, Shape({1, 1, 1, 1}), DeviceId{&backend});

    // int32
    CheckArrayRepr<int32_t>("array([0], dtype=int32, device_id=('cpu', 0))", {0}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<int32_t>("array([-2], dtype=int32, device_id=('cpu', 0))", {-2}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<int32_t>(
        "array([[0, 1, 2],\n"
        "       [3, 4, 5]], dtype=int32, device_id=('cpu', 0))",
        {0, 1, 2, 3, 4, 5}, Shape({2, 3}), DeviceId{&backend});
    CheckArrayRepr<int32_t>(
        "array([[ 0,  1,  2],\n"
        "       [-3,  4,  5]], dtype=int32, device_id=('cpu', 0))",
        {0, 1, 2, -3, 4, 5}, Shape({2, 3}), DeviceId{&backend});
    CheckArrayRepr<int32_t>("array([[[[3]]]], dtype=int32, device_id=('cpu', 0))", {3}, Shape({1, 1, 1, 1}), DeviceId{&backend});

    // int64
    CheckArrayRepr<int64_t>("array([0], dtype=int64, device_id=('cpu', 0))", {0}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<int64_t>("array([-2], dtype=int64, device_id=('cpu', 0))", {-2}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<int64_t>(
        "array([[0, 1, 2],\n"
        "       [3, 4, 5]], dtype=int64, device_id=('cpu', 0))",
        {0, 1, 2, 3, 4, 5}, Shape({2, 3}), DeviceId{&backend});
    CheckArrayRepr<int64_t>(
        "array([[ 0,  1,  2],\n"
        "       [-3,  4,  5]], dtype=int64, device_id=('cpu', 0))",
        {0, 1, 2, -3, 4, 5}, Shape({2, 3}), DeviceId{&backend});
    CheckArrayRepr<int64_t>("array([[[[3]]]], dtype=int64, device_id=('cpu', 0))", {3}, Shape({1, 1, 1, 1}), DeviceId{&backend});

    // uint8
    CheckArrayRepr<uint8_t>("array([0], dtype=uint8, device_id=('cpu', 0))", {0}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<uint8_t>("array([2], dtype=uint8, device_id=('cpu', 0))", {2}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<uint8_t>(
        "array([[0, 1, 2],\n"
        "       [3, 4, 5]], dtype=uint8, device_id=('cpu', 0))",
        {0, 1, 2, 3, 4, 5}, Shape({2, 3}), DeviceId{&backend});
    CheckArrayRepr<uint8_t>("array([[[[3]]]], dtype=uint8, device_id=('cpu', 0))", {3}, Shape({1, 1, 1, 1}), DeviceId{&backend});

    // float32
    CheckArrayRepr<float>("array([0.], dtype=float32, device_id=('cpu', 0))", {0}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<float>("array([3.25], dtype=float32, device_id=('cpu', 0))", {3.25}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<float>("array([-3.25], dtype=float32, device_id=('cpu', 0))", {-3.25}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<float>("array([ inf], dtype=float32, device_id=('cpu', 0))", {std::numeric_limits<float>::infinity()}, Shape({1}),
                          DeviceId{&backend});
    CheckArrayRepr<float>("array([ -inf], dtype=float32, device_id=('cpu', 0))", {-std::numeric_limits<float>::infinity()}, Shape({1}),
                          DeviceId{&backend});
    CheckArrayRepr<float>("array([ nan], dtype=float32, device_id=('cpu', 0))", {std::nanf("")}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<float>(
        "array([[0., 1., 2.],\n"
        "       [3., 4., 5.]], dtype=float32, device_id=('cpu', 0))",
        {0, 1, 2, 3, 4, 5}, Shape({2, 3}), DeviceId{&backend});
    CheckArrayRepr<float>(
        "array([[0.  , 1.  , 2.  ],\n"
        "       [3.25, 4.  , 5.  ]], dtype=float32, device_id=('cpu', 0))",
        {0, 1, 2, 3.25, 4, 5}, Shape({2, 3}), DeviceId{&backend});
    CheckArrayRepr<float>(
        "array([[ 0.  ,  1.  ,  2.  ],\n"
        "       [-3.25,  4.  ,  5.  ]], dtype=float32, device_id=('cpu', 0))",
        {0, 1, 2, -3.25, 4, 5}, Shape({2, 3}), DeviceId{&backend});
    CheckArrayRepr<float>("array([[[[3.25]]]], dtype=float32, device_id=('cpu', 0))", {3.25}, Shape({1, 1, 1, 1}), DeviceId{&backend});

    // float64
    CheckArrayRepr<double>("array([0.], dtype=float64, device_id=('cpu', 0))", {0}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<double>("array([3.25], dtype=float64, device_id=('cpu', 0))", {3.25}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<double>("array([-3.25], dtype=float64, device_id=('cpu', 0))", {-3.25}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<double>("array([ inf], dtype=float64, device_id=('cpu', 0))", {std::numeric_limits<double>::infinity()}, Shape({1}),
                           DeviceId{&backend});
    CheckArrayRepr<double>("array([ -inf], dtype=float64, device_id=('cpu', 0))", {-std::numeric_limits<double>::infinity()}, Shape({1}),
                           DeviceId{&backend});
    CheckArrayRepr<double>("array([ nan], dtype=float64, device_id=('cpu', 0))", {std::nan("")}, Shape({1}), DeviceId{&backend});
    CheckArrayRepr<double>(
        "array([[0., 1., 2.],\n"
        "       [3., 4., 5.]], dtype=float64, device_id=('cpu', 0))",
        {0, 1, 2, 3, 4, 5}, Shape({2, 3}), DeviceId{&backend});
    CheckArrayRepr<double>(
        "array([[0.  , 1.  , 2.  ],\n"
        "       [3.25, 4.  , 5.  ]], dtype=float64, device_id=('cpu', 0))",
        {0, 1, 2, 3.25, 4, 5}, Shape({2, 3}), DeviceId{&backend});
    CheckArrayRepr<double>(
        "array([[ 0.  ,  1.  ,  2.  ],\n"
        "       [-3.25,  4.  ,  5.  ]], dtype=float64, device_id=('cpu', 0))",
        {0, 1, 2, -3.25, 4, 5}, Shape({2, 3}), DeviceId{&backend});
    CheckArrayRepr<double>("array([[[[3.25]]]], dtype=float64, device_id=('cpu', 0))", {3.25}, Shape({1, 1, 1, 1}), DeviceId{&backend});

    // Single graph
    CheckArrayRepr<int32_t>("array([-2], dtype=int32, device_id=('cpu', 0), graph_ids=['graph_1'])", {-2}, Shape({1}), DeviceId{&backend},
                            {"graph_1"});

    // Two graphs
    CheckArrayRepr<int32_t>("array([1], dtype=int32, device_id=('cpu', 0), graph_ids=['graph_1', 'graph_2'])", {1}, Shape({1}),
                            DeviceId{&backend}, {"graph_1", "graph_2"});

    // Multiple graphs
    CheckArrayRepr<int32_t>(
        "array([-9], dtype=int32, device_id=('cpu', 0), graph_ids=['graph_1', 'graph_2', 'graph_3', 'graph_4', 'graph_5'])", {-9},
        Shape({1}), DeviceId{&backend}, {"graph_1", "graph_2", "graph_3", "graph_4", "graph_5"});
}

}  // namespace
}  // namespace xchainer
{'fraction_non_alphanumeric': 0.25465805349420745, 'fraction_numerical': 0.05835246383001441, 'mean_word_length': 3.7006273525721456, 'pattern_counts': {'":': 0, '<': 149, '<?xml version=': 0, '>': 145, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '490321', 'n_tokens_mistral': 7650, 'n_tokens_neox': 6999, 'n_words': 1435}