<commit_before>#pragma once

#include <cmath>
#include <cstdint>
#include <memory>

#include <gtest/gtest.h>

#include "xchainer/array.h"
#include "xchainer/backend.h"
#include "xchainer/context.h"
#include "xchainer/device.h"
#include "xchainer/indexable_array.h"
#include "xchainer/indexer.h"
#include "xchainer/native/native_backend.h"

// TODO(hvy): Make it independent from gtest.
namespace xchainer {
namespace testing {

namespace {

Array ToNativeDevice(const Array& array) {
    array.device().Synchronize();
    Context& context = array.device().backend().context();
    Backend& native_backend = context.GetBackend(native::NativeBackend::kDefaultName);
    Device& native_device = native_backend.GetDevice(0);
    Array native_array = array.ToDevice(native_device);

    native_device.Synchronize();
    return native_array;
}

}  // namespace

template <typename T>
void ExpectDataEqual(const Array& expected, const Array& actual) {
    actual.device().Synchronize();
    Array native_expected = ToNativeDevice(expected);
    Array native_actual = ToNativeDevice(actual);
    IndexableArray<const T> expected_iarray{native_expected};
    IndexableArray<const T> actual_iarray{native_actual};
    Indexer<> indexer{actual.shape()};
    for (int64_t i = 0; i < indexer.total_size(); ++i) {
        indexer.Set(i);
        T expected_value = expected_iarray[indexer];
        T actual_value = actual_iarray[indexer];
        if (std::isnan(expected_value)) {
            EXPECT_TRUE(std::isnan(actual_value)) << "where i is " << i;
        } else {
            EXPECT_EQ(expected_value, actual_value) << "where i is " << i;
        }<commit_msg>No need of synchronizing before ToDevice<commit_after>#pragma once

#include <cmath>
#include <cstdint>
#include <memory>

#include <gtest/gtest.h>

#include "xchainer/array.h"
#include "xchainer/backend.h"
#include "xchainer/context.h"
#include "xchainer/device.h"
#include "xchainer/indexable_array.h"
#include "xchainer/indexer.h"
#include "xchainer/native/native_backend.h"

// TODO(hvy): Make it independent from gtest.
namespace xchainer {
namespace testing {

namespace {

Array ToNativeDevice(const Array& array) {
    Context& context = array.device().backend().context();
    Backend& native_backend = context.GetBackend(native::NativeBackend::kDefaultName);
    Device& native_device = native_backend.GetDevice(0);
    Array native_array = array.ToDevice(native_device);

    native_device.Synchronize();
    return native_array;
}

}  // namespace

template <typename T>
void ExpectDataEqual(const Array& expected, const Array& actual) {
    actual.device().Synchronize();
    Array native_expected = ToNativeDevice(expected);
    Array native_actual = ToNativeDevice(actual);
    IndexableArray<const T> expected_iarray{native_expected};
    IndexableArray<const T> actual_iarray{native_actual};
    Indexer<> indexer{actual.shape()};
    for (int64_t i = 0; i < indexer.total_size(); ++i) {
        indexer.Set(i);
        T expected_value = expected_iarray[indexer];
        T actual_value = actual_iarray[indexer];
        if (std::isnan(expected_value)) {
            EXPECT_TRUE(std::isnan(actual_value)) << "where i is " << i;
        } else {
            EXPECT_EQ(expected_value, actual_value) << "where i is " << i;
        }
{'fraction_non_alphanumeric': 0.12753710996667678, 'fraction_numerical': 0.0024235080278703423, 'mean_word_length': 4.30016051364366, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1531684', 'n_tokens_mistral': 1060, 'n_tokens_neox': 1060, 'n_words': 222}