<commit_before>/*
    RawSpeed - RAW file decoder.

    Copyright (C) 2018 <NAME>

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

#include "common/Common.h" // for uchar8
#include "io/Buffer.h"     // for Buffer
#include "io/ByteStream.h" // for ByteStream
#include "io/Endianness.h" // for getHostEndianness, Endianness::big, Endia...
#include <array>           // for array
#include <gtest/gtest.h>   // for Message, AssertionResult, ASSERT_PRED_FOR...

using rawspeed::Buffer;
using rawspeed::ByteStream;
using rawspeed::DataBuffer;
using rawspeed::Endianness;

namespace rawspeed_test {

template <typename T> class BitPumpTest : public ::testing::Test {
public:
  using PumpT = typename T::PumpT;
  using PatternT = typename T::PatternT;

protected:
  static constexpr auto TestGetBits = [](PumpT* pump, auto gen) -> void {
    for (int len = 1; len <= 7; len++)
      ASSERT_EQ(pump->getBits(len), gen(len)) << "     Where len: " << len;
  };

  static constexpr auto TestGetBitsNoFill = [](PumpT* pump, auto gen) -> void {
    pump->fill(32); // Actually fills 32 bits
    for (int len = 1; len <= 7; len++)
      ASSERT_EQ(pump->getBitsNoFill(len), gen(len))
          << "     Where len: " << len;
  };

  static constexpr auto TestPeekBits = [](PumpT* pump, auto gen) -> void {
    for (int len = 1; len <= 7; len++) {
      ASSERT_EQ(pump->peekBits(len), gen(len)) << "     Where len: " << len;
      pump->skipBits(len);
    }
  };

  static constexpr auto TestPeekBitsNoFill = [](PumpT* pump, auto gen) -> void {
    pump->fill(32); // Actually fills 32 bits
    for (int len = 1; len <= 7; len++) {
      ASSERT_EQ(pump->peekBitsNoFill(len), gen(len))
          << "     Where len: " << len;
      pump->skipBitsNoFill(len);
    }
  };

  static constexpr auto TestIncreasingPeekLength = [](PumpT* pump,
                                                      auto data) -> void {
    static constexpr auto MaxLen = 28;
    for (int len = 1; len <= MaxLen; len++)
      ASSERT_EQ(pump->peekBits(len), data(len)) << "     Where len: " << len;
  };

  static constexpr auto TestIncreasingPeekLengthNoFill = [](PumpT* pump,
                                                            auto data) -> void {
    static constexpr auto MaxLen = 28;
    pump->fill(MaxLen); // Actually fills 32 bits
    for (int len = 1; len <= MaxLen; len++)
      ASSERT_EQ(pump->peekBitsNoFill(len), data(len))
          << "     Where len: " << len;
  };

  template <typename TestDataType, typename Test, typename L>
  void runTest(const TestDataType& data, Test test, L gen) {
    const Buffer b(data.data(), data.size());

    for (auto e : {Endianness::little, Endianness::big}) {
      const DataBuffer db(b, e);
      const ByteStream bs(db);

      PumpT pump(bs);
      test(&pump, gen);
    }
  }
};

TYPED_TEST_CASE_P(BitPumpTest);

TYPED_TEST_P(BitPumpTest, GetTest) {
  this->runTest(TypeParam::PatternT::Data, this->TestGetBits,
                TypeParam::PatternT::element);
}
TYPED_TEST_P(BitPumpTest, GetNoFillTest) {
  this->runTest(TypeParam::PatternT::Data, this->TestGetBitsNoFill,
                TypeParam::PatternT::element);
}
TYPED_TEST_P(BitPumpTest, PeekTest) {
  this->runTest(TypeParam::PatternT::Data, this->TestPeekBits,
                TypeParam::PatternT::element);
}
TYPED_TEST_P(BitPumpTest, PeekNoFillTest) {
  this->runTest(TypeParam::PatternT::Data, this->TestPeekBitsNoFill,
                TypeParam::PatternT::element);
}
TYPED_TEST_P(BitPumpTest, IncreasingPeekLengthTest) {
  this->runTest(TypeParam::PatternT::Data, this->TestIncreasingPeekLength,
                TypeParam::PatternT::data);
}
TYPED_TEST_P(BitPumpTest, IncreasingPeekLengthNoFillTest) {
  this->runTest(TypeParam::PatternT::Data, this->TestIncreasingPeekLengthNoFill,
                TypeParam::PatternT::data);
}

REGISTER_TYPED_TEST_CASE_P(BitPumpTest, GetTest, GetNoFillTest, PeekTest,
                           PeekNoFillTest, IncreasingPeekLengthTest,
                           IncreasingPeekLengthNoFillTest);

template <typename Pump, typename PatternTag> struct Pattern {
  static const std::array<rawspeed::uchar8, 4> Data;
  static rawspeed::uint32 element(int index);
  static rawspeed::uint32 data(int len);
};

struct ZerosTag;
template <typename Pump> struct Pattern<Pump, ZerosTag> {
  static const std::array<rawspeed::uchar8, 4> Data;
  static rawspeed::uint32 element(int index) { return 0U; }
  static rawspeed::uint32 data(int len) { return 0U; }
};
template <typename Pump>
const std::array<rawspeed::uchar8, 4> Pattern<Pump, ZerosTag>::Data{
    {/* zero-init */}};

struct OnesTag;
template <typename Pump> struct Pattern<Pump, OnesTag> {
  static const std::array<rawspeed::uchar8, 4> Data;
  static rawspeed::uint32 element(int index) { return 1U; }
  static rawspeed::uint32 data(int len);
};
<commit_msg>This is why you don't run in front of the train :)<commit_after>/*
    RawSpeed - RAW file decoder.

    Copyright (C) 2018 <NAME>

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

#include "common/Common.h" // for uchar8
#include "io/Buffer.h"     // for Buffer
#include "io/ByteStream.h" // for ByteStream
#include "io/Endianness.h" // for getHostEndianness, Endianness::big, Endia...
#include <array>           // for array
#include <gtest/gtest.h>   // for Message, AssertionResult, ASSERT_PRED_FOR...

using rawspeed::Buffer;
using rawspeed::ByteStream;
using rawspeed::DataBuffer;
using rawspeed::Endianness;

namespace rawspeed_test {

template <typename T, typename Tag> struct BitPumpPatternTest {};

struct TestGetBitsTag;
template <typename T> struct BitPumpPatternTest<T, TestGetBitsTag> {
  using PumpT = typename T::PumpT;
  using PatternT = typename T::PatternT;

  template <typename L> static void Test(PumpT* pump, L gen) {
    for (int len = 1; len <= 7; len++)
      ASSERT_EQ(pump->getBits(len), gen(len)) << "     Where len: " << len;
  }
};

struct TestGetBitsNoFillTag;
template <typename T> struct BitPumpPatternTest<T, TestGetBitsNoFillTag> {
  using PumpT = typename T::PumpT;
  using PatternT = typename T::PatternT;

  template <typename L> static void Test(PumpT* pump, L gen) {
    pump->fill(32); // Actually fills 32 bits
    for (int len = 1; len <= 7; len++)
      ASSERT_EQ(pump->getBitsNoFill(len), gen(len))
          << "     Where len: " << len;
  }
};

struct TestPeekBitsTag;
template <typename T> struct BitPumpPatternTest<T, TestPeekBitsTag> {
  using PumpT = typename T::PumpT;
  using PatternT = typename T::PatternT;

  template <typename L> static void Test(PumpT* pump, L gen) {
    for (int len = 1; len <= 7; len++) {
      ASSERT_EQ(pump->peekBits(len), gen(len)) << "     Where len: " << len;
      pump->skipBits(len);
    }
  }
};

struct TestPeekBitsNoFillTag;
template <typename T> struct BitPumpPatternTest<T, TestPeekBitsNoFillTag> {
  using PumpT = typename T::PumpT;
  using PatternT = typename T::PatternT;

  template <typename L> static void Test(PumpT* pump, L gen) {
    pump->fill(32); // Actually fills 32 bits
    for (int len = 1; len <= 7; len++) {
      ASSERT_EQ(pump->peekBitsNoFill(len), gen(len))
          << "     Where len: " << len;
      pump->skipBitsNoFill(len);
    }
  }
};

struct TestIncreasingPeekLengthTag;
template <typename T>
struct BitPumpPatternTest<T, TestIncreasingPeekLengthTag> {
  using PumpT = typename T::PumpT;
  using PatternT = typename T::PatternT;

  template <typename L> static void Test(PumpT* pump, L data) {
    static const auto MaxLen = 28;
    for (int len = 1; len <= MaxLen; len++)
      ASSERT_EQ(pump->peekBits(len), data(len)) << "     Where len: " << len;
  }
};

struct TestIncreasingPeekLengthNoFillTag;
template <typename T>
struct BitPumpPatternTest<T, TestIncreasingPeekLengthNoFillTag> {
  using PumpT = typename T::PumpT;
  using PatternT = typename T::PatternT;

  template <typename L> static void Test(PumpT* pump, L data) {
    static const auto MaxLen = 28;
    pump->fill(MaxLen); // Actually fills 32 bits
    for (int len = 1; len <= MaxLen; len++)
      ASSERT_EQ(pump->peekBitsNoFill(len), data(len))
          << "     Where len: " << len;
  }
};

template <typename T> class BitPumpTest : public ::testing::Test {
public:
  using PumpT = typename T::PumpT;
  using PatternT = typename T::PatternT;

protected:
  template <typename Tag, typename TestDataType, typename L>
  void runTest(const TestDataType& data, L gen) {
    const Buffer b(data.data(), data.size());

    for (auto e : {Endianness::little, Endianness::big}) {
      const DataBuffer db(b, e);
      const ByteStream bs(db);

      PumpT pump(bs);
      BitPumpPatternTest<T, Tag>::Test(&pump, gen);
    }
  }
};

TYPED_TEST_CASE_P(BitPumpTest);

TYPED_TEST_P(BitPumpTest, GetTest) {
  this->template runTest<TestGetBitsTag>(TypeParam::PatternT::Data,
                                         TypeParam::PatternT::element);
}
TYPED_TEST_P(BitPumpTest, GetNoFillTest) {
  this->template runTest<TestGetBitsNoFillTag>(TypeParam::PatternT::Data,
                                               TypeParam::PatternT::element);
}
TYPED_TEST_P(BitPumpTest, PeekTest) {
  this->template runTest<TestPeekBitsTag>(TypeParam::PatternT::Data,
                                          TypeParam::PatternT::element);
}
TYPED_TEST_P(BitPumpTest, PeekNoFillTest) {
  this->template runTest<TestPeekBitsNoFillTag>(TypeParam::PatternT::Data,
                                                TypeParam::PatternT::element);
}
TYPED_TEST_P(BitPumpTest, IncreasingPeekLengthTest) {
  this->template runTest<TestIncreasingPeekLengthTag>(
      TypeParam::PatternT::Data, TypeParam::PatternT::data);
}
TYPED_TEST_P(BitPumpTest, IncreasingPeekLengthNoFillTest) {
  this->template runTest<TestIncreasingPeekLengthNoFillTag>(
      TypeParam::PatternT::Data, TypeParam::PatternT::data);
}

REGISTER_TYPED_TEST_CASE_P(BitPumpTest, GetTest, GetNoFillTest, PeekTest,
                           PeekNoFillTest, IncreasingPeekLengthTest,
                           IncreasingPeekLengthNoFillTest);

template <typename Pump, typename PatternTag> struct Pattern {};

struct ZerosTag;
template <typename Pump> struct Pattern<Pump, ZerosTag> {
  static const std::array<rawspeed::uchar8, 4> Data;
  static rawspeed::uint32 element(int index) { return 0U; }
  static rawspeed::uint32 data(int len) { return 0U; }
};
template <typename Pump>
const std::array<rawspeed::uchar8, 4> Pattern<Pump, ZerosTag>::Data{
    {/* zero-init */}};

struct OnesTag;
template <typename Pump> struct Pattern<Pump, OnesTag> {
  static const std::array<rawspeed::uchar8, 4> Data;
  static rawspeed::uint32 element(int index) { return 1U; }
  static rawspeed::uint32 data(int len);
};

{'fraction_non_alphanumeric': 0.11908510814618381, 'fraction_numerical': 0.010110217949780393, 'mean_word_length': 3.492926284437826, 'pattern_counts': {'":': 0, '<': 120, '<?xml version=': 0, '>': 106, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6324586', 'n_tokens_mistral': 3964, 'n_tokens_neox': 3816, 'n_words': 1068}