<commit_before>#include "ntuple_test.hxx"

TEST(RNTuple, View)
{
   FileRaii fileGuard("test_ntuple_view.root");

   auto model = RNTupleModel::Create();
   auto fieldPt = model->MakeField<float>("pt", 42.0);
   auto fieldTag = model->MakeField<std::string>("tag", "xyz");
   auto fieldJets = model->MakeField<std::vector<std::int32_t>>("jets");
   fieldJets->push_back(1);
   fieldJets->push_back(2);
   fieldJets->push_back(3);

   {
      RNTupleWriter ntuple(std::move(model),
         std::make_unique<RPageSinkFile>("myNTuple", fileGuard.GetPath(), RNTupleWriteOptions()));
      ntuple.Fill();
      ntuple.CommitCluster();
      fieldJets->clear();
      ntuple.Fill();
   }

   RNTupleReader ntuple(std::make_unique<RPageSourceFile>("myNTuple", fileGuard.GetPath(), RNTupleReadOptions()));
   auto viewPt = ntuple.GetView<float>("pt");
   int n = 0;
   for (auto i : ntuple.GetEntryRange()) {
      EXPECT_EQ(42.0, viewPt(i));
      n++;
   }
   EXPECT_EQ(2, n);

   auto viewJets = ntuple.GetView<std::vector<std::int32_t>>("jets");
   n = 0;
   for (auto i : ntuple.GetEntryRange()) {
      if (i == 0) {
         EXPECT_EQ(3U, viewJets(i).size());
         EXPECT_EQ(1, viewJets(i)[0]);
         EXPECT_EQ(2, viewJets(i)[1]);
         EXPECT_EQ(3, viewJets(i)[2]);
      } else {
         EXPECT_EQ(0U, viewJets(i).size());
      }
      n++;
   }
   EXPECT_EQ(2, n);

   auto viewJetElements = ntuple.GetView<std::int32_t>("jets.std::int32_t");
   n = 0;
   for (auto i : viewJetElements.GetFieldRange()) {
      n++;
      EXPECT_EQ(n, viewJetElements(i));
   }
   EXPECT_EQ(3, n);
}

TEST(RNTuple, BulkView)
{
   FileRaii fileGuard("test_ntuple_bulk_view.root");

   auto model = RNTupleModel::Create();
   auto fieldPt = model->MakeField<float>("pt", 42.0);
   auto eltsPerPage = 10'000;
   {
      RNTupleWriteOptions opt;
      opt.SetNElementsPerPage(eltsPerPage);
      auto ntuple = RNTupleWriter::Recreate(std::move(model), "myNTuple",
         fileGuard.GetPath(), opt);
      for (int i = 0; i < 100'000; i++) {
         ntuple->Fill();
      }
   }
   auto ntuple = RNTupleReader::Open("myNTuple", fileGuard.GetPath());
   auto viewPt = ntuple->GetView<float>("pt");

   NTupleSize_t nPageItems = 0;
   const float *buf = viewPt.MapV(0, nPageItems);
   ASSERT_EQ(eltsPerPage, nPageItems);
   auto dest = std::make_unique<float[]>(nPageItems);
   memcpy(dest.get(), buf, sizeof(float) * nPageItems);
   for (NTupleSize_t i = 0; i < nPageItems; i++) {
      ASSERT_EQ(42.0f, dest.get()[i]) << i;
   }
   // second last element
   buf = viewPt.MapV(eltsPerPage - 2, nPageItems);
   ASSERT_EQ(2, nPageItems);
   for (NTupleSize_t i = 0; i < nPageItems; i++) {
      ASSERT_EQ(42.0f, buf[i]) << i;
   }
   // last element
   buf = viewPt.MapV(eltsPerPage - 1, nPageItems);
   ASSERT_EQ(1, nPageItems);
   for (NTupleSize_t i = 0; i < nPageItems; i++) {
      ASSERT_EQ(42.0f, buf[i]) << i;
   }
}

TEST(RNTuple, BulkViewCollection)
{
   FileRaii fileGuard("test_ntuple_bulk_view_collection.root");

   auto model = RNTupleModel::Create();
   auto fieldVec = model->MakeField<std::vector<double>>({"vec", "some data"});
   auto eltsPerPage = 10'000;
   {
      RNTupleWriteOptions opt;
      opt.SetNElementsPerPage(eltsPerPage);
      auto ntuple = RNTupleWriter::Recreate(std::move(model), "myNTuple",
         fileGuard.GetPath(), opt);
      for (int i = 0; i < 100'000; i++) {
         *fieldVec = std::vector<double>(i % 5, 100);
         ntuple->Fill();
      }
   }
   auto ntuple = RNTupleReader::Open("myNTuple", fileGuard.GetPath());
   auto viewVecOffsets = ntuple->GetView<ClusterSize_t>("vec");
   auto viewVecData = ntuple->GetView<double>("vec.double");

   NTupleSize_t nPageItems = 0;
   const ClusterSize_t *offsets_buf = viewVecOffsets.MapV(0, nPageItems);
   ASSERT_EQ(eltsPerPage, nPageItems);
   std::unique_ptr<ClusterSize_t[]> offsets = std::make_unique<ClusterSize_t[]>(nPageItems + 1);
   auto raw_offsets = offsets.get();
   raw_offsets[0] = 0;
   memcpy(raw_offsets + 1, offsets_buf, sizeof(ClusterSize_t) * nPageItems);
   for (NTupleSize_t i = 1; i < nPageItems + 1; i++) {
      ASSERT_EQ((i - 1) % 5, raw_offsets[i] - raw_offsets[i-1]) << i;
   }
   const double *buf = viewVecData.MapV(0, nPageItems);
   ASSERT_EQ(eltsPerPage, nPageItems);
   for (NTupleSize_t i = 0; i < nPageItems; i++) {
      ASSERT_EQ(100.0f, buf[i]) << i;
   }<commit_msg>[ntuple] Remove unnecessary memcpy from test<commit_after>#include "ntuple_test.hxx"

TEST(RNTuple, View)
{
   FileRaii fileGuard("test_ntuple_view.root");

   auto model = RNTupleModel::Create();
   auto fieldPt = model->MakeField<float>("pt", 42.0);
   auto fieldTag = model->MakeField<std::string>("tag", "xyz");
   auto fieldJets = model->MakeField<std::vector<std::int32_t>>("jets");
   fieldJets->push_back(1);
   fieldJets->push_back(2);
   fieldJets->push_back(3);

   {
      RNTupleWriter ntuple(std::move(model),
         std::make_unique<RPageSinkFile>("myNTuple", fileGuard.GetPath(), RNTupleWriteOptions()));
      ntuple.Fill();
      ntuple.CommitCluster();
      fieldJets->clear();
      ntuple.Fill();
   }

   RNTupleReader ntuple(std::make_unique<RPageSourceFile>("myNTuple", fileGuard.GetPath(), RNTupleReadOptions()));
   auto viewPt = ntuple.GetView<float>("pt");
   int n = 0;
   for (auto i : ntuple.GetEntryRange()) {
      EXPECT_EQ(42.0, viewPt(i));
      n++;
   }
   EXPECT_EQ(2, n);

   auto viewJets = ntuple.GetView<std::vector<std::int32_t>>("jets");
   n = 0;
   for (auto i : ntuple.GetEntryRange()) {
      if (i == 0) {
         EXPECT_EQ(3U, viewJets(i).size());
         EXPECT_EQ(1, viewJets(i)[0]);
         EXPECT_EQ(2, viewJets(i)[1]);
         EXPECT_EQ(3, viewJets(i)[2]);
      } else {
         EXPECT_EQ(0U, viewJets(i).size());
      }
      n++;
   }
   EXPECT_EQ(2, n);

   auto viewJetElements = ntuple.GetView<std::int32_t>("jets.std::int32_t");
   n = 0;
   for (auto i : viewJetElements.GetFieldRange()) {
      n++;
      EXPECT_EQ(n, viewJetElements(i));
   }
   EXPECT_EQ(3, n);
}

TEST(RNTuple, BulkView)
{
   FileRaii fileGuard("test_ntuple_bulk_view.root");

   auto model = RNTupleModel::Create();
   auto fieldPt = model->MakeField<float>("pt", 42.0);
   auto eltsPerPage = 10'000;
   {
      RNTupleWriteOptions opt;
      opt.SetNElementsPerPage(eltsPerPage);
      auto ntuple = RNTupleWriter::Recreate(std::move(model), "myNTuple",
         fileGuard.GetPath(), opt);
      for (int i = 0; i < 100'000; i++) {
         ntuple->Fill();
      }
   }
   auto ntuple = RNTupleReader::Open("myNTuple", fileGuard.GetPath());
   auto viewPt = ntuple->GetView<float>("pt");

   NTupleSize_t nPageItems = 0;
   const float *buf = viewPt.MapV(0, nPageItems);
   ASSERT_EQ(eltsPerPage, nPageItems);
   for (NTupleSize_t i = 0; i < nPageItems; i++) {
      ASSERT_EQ(42.0f, buf[i]) << i;
   }
   // second last element
   buf = viewPt.MapV(eltsPerPage - 2, nPageItems);
   ASSERT_EQ(2, nPageItems);
   for (NTupleSize_t i = 0; i < nPageItems; i++) {
      ASSERT_EQ(42.0f, buf[i]) << i;
   }
   // last element
   buf = viewPt.MapV(eltsPerPage - 1, nPageItems);
   ASSERT_EQ(1, nPageItems);
   for (NTupleSize_t i = 0; i < nPageItems; i++) {
      ASSERT_EQ(42.0f, buf[i]) << i;
   }
}

TEST(RNTuple, BulkViewCollection)
{
   FileRaii fileGuard("test_ntuple_bulk_view_collection.root");

   auto model = RNTupleModel::Create();
   auto fieldVec = model->MakeField<std::vector<double>>({"vec", "some data"});
   auto eltsPerPage = 10'000;
   {
      RNTupleWriteOptions opt;
      opt.SetNElementsPerPage(eltsPerPage);
      auto ntuple = RNTupleWriter::Recreate(std::move(model), "myNTuple",
         fileGuard.GetPath(), opt);
      for (int i = 0; i < 100'000; i++) {
         *fieldVec = std::vector<double>(i % 5, 100);
         ntuple->Fill();
      }
   }
   auto ntuple = RNTupleReader::Open("myNTuple", fileGuard.GetPath());
   auto viewVecOffsets = ntuple->GetView<ClusterSize_t>("vec");
   auto viewVecData = ntuple->GetView<double>("vec.double");

   NTupleSize_t nPageItems = 0;
   const ClusterSize_t *offsets_buf = viewVecOffsets.MapV(0, nPageItems);
   ASSERT_EQ(eltsPerPage, nPageItems);
   std::unique_ptr<ClusterSize_t[]> offsets = std::make_unique<ClusterSize_t[]>(nPageItems + 1);
   auto raw_offsets = offsets.get();
   // offsets implicitly start at zero, RNTuple does not store that information
   raw_offsets[0] = 0;
   memcpy(raw_offsets + 1, offsets_buf, sizeof(ClusterSize_t) * nPageItems);
   for (NTupleSize_t i = 1; i < nPageItems + 1; i++) {
      ASSERT_EQ((i - 1) % 5, raw_offsets[i] - raw_offsets[i-1]) << i;
   }
   const double *buf = viewVecData.MapV(0, nPageItems);
   ASSERT_EQ(eltsPerPage, nPageItems);
   for (NTupleSize_t i = 0; i < nPageItems; i++) {
      ASSERT_EQ(100.0f, buf[i]) << i;
   }
{'fraction_non_alphanumeric': 0.16549135577798, 'fraction_numerical': 0.022292993630573247, 'mean_word_length': 3.5654205607476634, 'pattern_counts': {'":': 0, '<': 76, '<?xml version=': 0, '>': 70, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5611350', 'n_tokens_mistral': 3423, 'n_tokens_neox': 3307, 'n_words': 597}