<commit_before>#include "../testing.t.hpp"

#include <iterator>

#include "util/bytefile.hpp"
#include "util/algorithm.hpp"

namespace top1 {

  using Chunk = ByteFile::Chunk;
  using Path = ByteFile::Path;
  using Error = ByteFile::Error;

  TEST_CASE("Bytefile", "[util]") {

    Path somePath = test::dir / "test1.bytes";

    ByteFile f;

    SECTION("Open / close") {

      REQUIRE(!f.is_open());
      REQUIRE_NOTHROW(f.open(somePath));

      REQUIRE_NOTHROW(f.close());

      REQUIRE(!f.is_open());
      // REQUIRE(filesystem::exists(somePath));
      // REQUIRE(filesystem::is_regular_file(somePath));
    }

    SECTION("seek") {

      ByteFile::Position somePos = 10;

      REQUIRE(!f.is_open());
      REQUIRE_THROWS_AS(f.position(), ByteFile::Error);

      REQUIRE_NOTHROW(f.open(somePath));
      REQUIRE_NOTHROW(f.position() == 0);
      REQUIRE(f.size() == 0);

      SECTION ("Seeking past the end") {
        REQUIRE_NOTHROW(f.seek(somePos) == somePos);
        REQUIRE(f.position() == somePos);
      }
    }

    SECTION("Write / Read bytes") {

      constexpr std::size_t someSize = 2048;

      REQUIRE_NOTHROW(f.open(somePath));

      std::array<std::byte, someSize> bytes;

      REQUIRE_NOTHROW(f.write_bytes(bytes.begin(), bytes.end()));
      REQUIRE_NOTHROW(f.position() == someSize);
      REQUIRE_NOTHROW(f.size() == someSize);
      REQUIRE_NOTHROW(f.close());

      std::array<std::byte, someSize> readBytes;
      REQUIRE_NOTHROW(f.open(somePath));
      REQUIRE(f.read_bytes(readBytes.begin(), readBytes.end()).is_ok());

      REQUIRE_NOTHROW(f.position() == someSize);
      REQUIRE_NOTHROW(f.size() == someSize);

      // Compare arrays
      REQUIRE(std::equal(readBytes.begin(), readBytes.end(), bytes.begin()));

      REQUIRE(f.read_bytes(readBytes.begin(), 10).is_err());
    }

    SECTION("Chunks") {

      Path somePath2 = "testdata/test2.bytes";
      int someLength = 4096;

      struct SomeChunk : Chunk {
        bytes<4> field1;
        bytes<12> field2;

        SomeChunk() : Chunk("Chu1") {}
        SomeChunk(Chunk& c) : Chunk(c) {}

        void write_fields(ByteFile& file) override {
          file.write_bytes(field1);
          file.write_bytes(field2);
        }

        void read_fields(ByteFile& file) override {
          REQUIRE(file.read_bytes(field1).is_ok());<commit_msg>Fix ByteFile tests<commit_after>#include "../testing.t.hpp"

#include <iterator>

#include "util/bytefile.hpp"
#include "util/algorithm.hpp"

namespace top1 {

  using Chunk = ByteFile::Chunk;
  using Path = ByteFile::Path;
  using Error = ByteFile::Error;

  TEST_CASE("Bytefile", "[util]") {

    Path somePath = test::dir / "test1.bytes";

    ByteFile f;

    SECTION("Open / close") {

      REQUIRE(!f.is_open());
      REQUIRE_NOTHROW(f.open(somePath));

      REQUIRE_NOTHROW(f.close());

      REQUIRE(!f.is_open());
      REQUIRE(fs::exists(somePath));
      REQUIRE(fs::is_regular_file(somePath));
    }

    SECTION("seek") {

      ByteFile::Position somePos = 10;

      REQUIRE(!f.is_open());
      REQUIRE_THROWS_AS(f.position(), ByteFile::Error);

      REQUIRE_NOTHROW(f.open(somePath));
      REQUIRE_NOTHROW(f.position() == 0);
      REQUIRE(f.size() == 0);

      SECTION ("Seeking past the end") {
        REQUIRE_NOTHROW(f.seek(somePos) == somePos);
        REQUIRE(f.position() == somePos);
      }
    }

    SECTION("Write / Read bytes") {

      constexpr std::size_t someSize = 2048;

      REQUIRE_NOTHROW(f.open(somePath));

      std::array<std::byte, someSize> bytes;

      REQUIRE_NOTHROW(f.write_bytes(bytes.begin(), bytes.end()));
      REQUIRE_NOTHROW(f.position() == someSize);
      REQUIRE_NOTHROW(f.size() == someSize);
      REQUIRE_NOTHROW(f.close());

      std::array<std::byte, someSize> readBytes;
      REQUIRE_NOTHROW(f.open(somePath));
      REQUIRE(f.read_bytes(readBytes.begin(), readBytes.end()).is_ok());

      REQUIRE_NOTHROW(f.position() == someSize);
      REQUIRE_NOTHROW(f.size() == someSize);

      // Compare arrays
      REQUIRE(std::equal(readBytes.begin(), readBytes.end(), bytes.begin()));

      REQUIRE(f.read_bytes(readBytes.begin(), 10).is_err());
    }

    SECTION("Chunks") {

      Path somePath2 = test::dir / "test2.bytes";
      int someLength = 4096;

      struct SomeChunk : Chunk {
        bytes<4> field1;
        bytes<12> field2;

        SomeChunk() : Chunk("Chu1") {}
        SomeChunk(Chunk& c) : Chunk(c) {}

        void write_fields(ByteFile& file) override {
          file.write_bytes(field1);
          file.write_bytes(field2);
        }

        void read_fields(ByteFile& file) override {
          REQUIRE(file.read_bytes(field1).is_ok());
{'fraction_non_alphanumeric': 0.16168941979522183, 'fraction_numerical': 0.01151877133105802, 'mean_word_length': 2.997442455242967, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1747224', 'n_tokens_mistral': 1771, 'n_tokens_neox': 1636, 'n_words': 244}