<commit_before>/*
    RawSpeed - RAW file decoder.

    Copyright (C) 2009-2014 <NAME>
    Copyright (C) 2014 <NAME>
    Copyright (C) 2017-2019 <NAME>

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

#include "decompressors/UncompressedDecompressor.h"
#include "common/Common.h"                // for uint32_t, uint8_t, uint16_t
#include "common/FloatingPoint.h"         // for fp16ToFloat
#include "common/Point.h"                 // for iPoint2D
#include "decoders/RawDecoderException.h" // for ThrowRDE
#include "io/BitPumpLSB.h"                // for BitPumpLSB
#include "io/BitPumpMSB.h"                // for BitPumpMSB
#include "io/BitPumpMSB16.h"              // for BitPumpMSB16
#include "io/BitPumpMSB32.h"              // for BitPumpMSB32
#include "io/ByteStream.h"                // for ByteStream
#include "io/Endianness.h"                // for getHostEndianness, Endiann...
#include "io/IOException.h"               // for ThrowIOE
#include <algorithm>                      // for min
#include <cassert>                        // for assert

using std::min;

namespace rawspeed {

void UncompressedDecompressor::sanityCheck(const uint32_t* h,
                                           int bytesPerLine) const {
  assert(h != nullptr);
  assert(*h > 0);
  assert(bytesPerLine > 0);
  assert(input.getSize() > 0);

  // How many multiples of bpl are there in the input buffer?
  // The remainder is ignored/discarded.
  const auto fullRows = input.getRemainSize() / bytesPerLine;

  // If more than the output height, we are all good.
  if (fullRows >= *h)
    return; // all good!

  if (fullRows == 0)
    ThrowIOE("Not enough data to decode a single line. Image file truncated.");

  ThrowIOE("Image truncated, only %u of %u lines found", fullRows, *h);

  // FIXME: need to come up with some common variable to allow proceeding here
  // *h = min_h;
}

void UncompressedDecompressor::sanityCheck(uint32_t w, const uint32_t* h,
                                           int bpp) const {
  assert(w > 0);
  assert(bpp > 0);

  // bytes per line
  const auto bpl = bpp * w;
  assert(bpl > 0);

  sanityCheck(h, bpl);
}

int UncompressedDecompressor::bytesPerLine(int w, bool skips) {
  assert(w > 0);

  if ((12 * w) % 8 != 0)
    ThrowIOE("Bad image width");

  // Calculate expected bytes per line.
  auto perline = (12 * w) / 8;

  if (!skips)
    return perline;

  // Add skips every 10 pixels
  perline += ((w + 2) / 10);

  return perline;
}

template <typename Pump, typename NarrowFpType>
void UncompressedDecompressor::decodePackedFP(const iPoint2D& size,
                                              const iPoint2D& offset,
                                              uint32_t skipBytes, int rows,
                                              int row) const {
  const Array2DRef<float> out(mRaw->getF32DataAsUncroppedArray2DRef());
  Pump bits(input);

  int cols = size.x * mRaw->getCpp();
  for (; row < rows; row++) {
    for (int col = 0; col < cols; col++) {
      uint32_t b = bits.getBits(NarrowFpType::StorageWidth);
      uint32_t f =
          extendBinaryFloatingPoint<NarrowFpType, ieee_754_2008::Binary32>(b);
      out(row, offset.x + col) = bit_cast<float>(f);
    }
    bits.skipBytes(skipBytes);
  }
}

template <typename Pump>
void UncompressedDecompressor::decodePackedInt(const iPoint2D& size,
                                               const iPoint2D& offset,
                                               uint32_t skipBytes, int rows,
                                               int row, int bitPerPixel) const {
  const Array2DRef<uint16_t> out(mRaw->getU16DataAsUncroppedArray2DRef());
  Pump bits(input);

  int cols = size.x * mRaw->getCpp();
  for (; row < rows; row++) {
    for (int x = 0; x < cols; x++) {
      out(row, x) = bits.getBits(bitPerPixel);
    }
    bits.skipBytes(skipBytes);
  }
}

void UncompressedDecompressor::readUncompressedRaw(const iPoint2D& size,
                                                   const iPoint2D& offset,
                                                   int inputPitchBytes,
                                                   int bitPerPixel,
                                                   BitOrder order) {
  assert(inputPitchBytes > 0);
  assert(bitPerPixel > 0);

  uint8_t* data = mRaw->getData();
  uint32_t outPitch = mRaw->pitch;
  uint32_t w = size.x;
  uint32_t h = size.y;
  uint32_t cpp = mRaw->getCpp();
  uint64_t ox = offset.x;
  uint64_t oy = offset.y;

  if (bitPerPixel > 16 && mRaw->getDataType() == RawImageType::UINT16)
    ThrowRDE("Unsupported bit depth");

  const int outPixelBits = w * cpp * bitPerPixel;
  assert(outPixelBits > 0);

  if (outPixelBits % 8 != 0) {
    ThrowRDE("Bad combination of cpp (%u), bps (%u) and width (%u), the "
             "pitch is %u bits, which is not a multiple of 8 (1 byte)",
             cpp, bitPerPixel, w, outPixelBits);
  }

  const int outPixelBytes = outPixelBits / 8;

  // The input pitch might be larger than needed (i.e. have some padding),
  // but it can *not* be smaller than needed.
  if (inputPitchBytes < outPixelBytes)
    ThrowRDE("Specified pitch is smaller than minimally-required pitch");

  // Check the specified pitch, not the minimally-required pitch.
  sanityCheck(&h, inputPitchBytes);

  assert(inputPitchBytes >= outPixelBytes);
  uint32_t skipBytes = inputPitchBytes - outPixelBytes; // Skip per line

  if (oy > static_cast<uint64_t>(mRaw->dim.y))
    ThrowRDE("Invalid y offset");
  if (ox + size.x > static_cast<uint64_t>(mRaw->dim.x))
    ThrowRDE("Invalid x offset");

  uint64_t y = oy;
  h = min(h + oy, static_cast<uint64_t>(mRaw->dim.y));

  if (mRaw->getDataType() == RawImageType::F32) {
    if (bitPerPixel == 32) {
      copyPixels(&data[offset.x * sizeof(float) * cpp + y * outPitch], outPitch,
                 input.getData(inputPitchBytes * (h - y)), inputPitchBytes,
                 w * mRaw->getBpp(), h - y);
      return;
    }
    if (BitOrder::MSB == order && bitPerPixel == 16) {
      decodePackedFP<BitPumpMSB, ieee_754_2008::Binary16>(size, offset,
                                                          skipBytes, h, y);
      return;
    }
    if (BitOrder::LSB == order && bitPerPixel == 16) {
      decodePackedFP<BitPumpLSB, ieee_754_2008::Binary16>(size, offset,
                                                          skipBytes, h, y);
      return;
    }
    if (BitOrder::MSB == order && bitPerPixel == 24) {
      decodePackedFP<BitPumpMSB, ieee_754_2008::Binary24>(size, offset,
                                                          skipBytes, h, y);
      return;
    }
    if (BitOrder::LSB == order && bitPerPixel == 24) {
      decodePackedFP<BitPumpLSB, ieee_754_2008::Binary24>(size, offset,
                                                          skipBytes, h, y);
      return;
    }
    ThrowRDE("Unsupported floating-point input bitwidth/bit packing: %u / %u",
             bitPerPixel, static_cast<unsigned>(order));
  }

  if (BitOrder::MSB == order) {
    decodePackedInt<BitPumpMSB>(size, offset, skipBytes, h, y, bitPerPixel);
  } else if (BitOrder::MSB16 == order) {<commit_msg>`UncompressedDecompressor::readUncompressedRaw()`: finish porting to `Array2DRef`<commit_after>/*
    RawSpeed - RAW file decoder.

    Copyright (C) 2009-2014 <NAME>
    Copyright (C) 2014 <NAME>
    Copyright (C) 2017-2019 <NAME>

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

#include "decompressors/UncompressedDecompressor.h"
#include "common/Common.h"                // for uint32_t, uint8_t, uint16_t
#include "common/FloatingPoint.h"         // for fp16ToFloat
#include "common/Point.h"                 // for iPoint2D
#include "decoders/RawDecoderException.h" // for ThrowRDE
#include "io/BitPumpLSB.h"                // for BitPumpLSB
#include "io/BitPumpMSB.h"                // for BitPumpMSB
#include "io/BitPumpMSB16.h"              // for BitPumpMSB16
#include "io/BitPumpMSB32.h"              // for BitPumpMSB32
#include "io/ByteStream.h"                // for ByteStream
#include "io/Endianness.h"                // for getHostEndianness, Endiann...
#include "io/IOException.h"               // for ThrowIOE
#include <algorithm>                      // for min
#include <cassert>                        // for assert

using std::min;

namespace rawspeed {

void UncompressedDecompressor::sanityCheck(const uint32_t* h,
                                           int bytesPerLine) const {
  assert(h != nullptr);
  assert(*h > 0);
  assert(bytesPerLine > 0);
  assert(input.getSize() > 0);

  // How many multiples of bpl are there in the input buffer?
  // The remainder is ignored/discarded.
  const auto fullRows = input.getRemainSize() / bytesPerLine;

  // If more than the output height, we are all good.
  if (fullRows >= *h)
    return; // all good!

  if (fullRows == 0)
    ThrowIOE("Not enough data to decode a single line. Image file truncated.");

  ThrowIOE("Image truncated, only %u of %u lines found", fullRows, *h);

  // FIXME: need to come up with some common variable to allow proceeding here
  // *h = min_h;
}

void UncompressedDecompressor::sanityCheck(uint32_t w, const uint32_t* h,
                                           int bpp) const {
  assert(w > 0);
  assert(bpp > 0);

  // bytes per line
  const auto bpl = bpp * w;
  assert(bpl > 0);

  sanityCheck(h, bpl);
}

int UncompressedDecompressor::bytesPerLine(int w, bool skips) {
  assert(w > 0);

  if ((12 * w) % 8 != 0)
    ThrowIOE("Bad image width");

  // Calculate expected bytes per line.
  auto perline = (12 * w) / 8;

  if (!skips)
    return perline;

  // Add skips every 10 pixels
  perline += ((w + 2) / 10);

  return perline;
}

template <typename Pump, typename NarrowFpType>
void UncompressedDecompressor::decodePackedFP(const iPoint2D& size,
                                              const iPoint2D& offset,
                                              uint32_t skipBytes, int rows,
                                              int row) const {
  const Array2DRef<float> out(mRaw->getF32DataAsUncroppedArray2DRef());
  Pump bits(input);

  int cols = size.x * mRaw->getCpp();
  for (; row < rows; row++) {
    for (int col = 0; col < cols; col++) {
      uint32_t b = bits.getBits(NarrowFpType::StorageWidth);
      uint32_t f =
          extendBinaryFloatingPoint<NarrowFpType, ieee_754_2008::Binary32>(b);
      out(row, offset.x + col) = bit_cast<float>(f);
    }
    bits.skipBytes(skipBytes);
  }
}

template <typename Pump>
void UncompressedDecompressor::decodePackedInt(const iPoint2D& size,
                                               const iPoint2D& offset,
                                               uint32_t skipBytes, int rows,
                                               int row, int bitPerPixel) const {
  const Array2DRef<uint16_t> out(mRaw->getU16DataAsUncroppedArray2DRef());
  Pump bits(input);

  int cols = size.x * mRaw->getCpp();
  for (; row < rows; row++) {
    for (int x = 0; x < cols; x++) {
      out(row, x) = bits.getBits(bitPerPixel);
    }
    bits.skipBytes(skipBytes);
  }
}

void UncompressedDecompressor::readUncompressedRaw(const iPoint2D& size,
                                                   const iPoint2D& offset,
                                                   int inputPitchBytes,
                                                   int bitPerPixel,
                                                   BitOrder order) {
  assert(inputPitchBytes > 0);
  assert(bitPerPixel > 0);

  uint32_t outPitch = mRaw->pitch;
  uint32_t w = size.x;
  uint32_t h = size.y;
  uint32_t cpp = mRaw->getCpp();
  uint64_t ox = offset.x;
  uint64_t oy = offset.y;

  if (bitPerPixel > 16 && mRaw->getDataType() == RawImageType::UINT16)
    ThrowRDE("Unsupported bit depth");

  const int outPixelBits = w * cpp * bitPerPixel;
  assert(outPixelBits > 0);

  if (outPixelBits % 8 != 0) {
    ThrowRDE("Bad combination of cpp (%u), bps (%u) and width (%u), the "
             "pitch is %u bits, which is not a multiple of 8 (1 byte)",
             cpp, bitPerPixel, w, outPixelBits);
  }

  const int outPixelBytes = outPixelBits / 8;

  // The input pitch might be larger than needed (i.e. have some padding),
  // but it can *not* be smaller than needed.
  if (inputPitchBytes < outPixelBytes)
    ThrowRDE("Specified pitch is smaller than minimally-required pitch");

  // Check the specified pitch, not the minimally-required pitch.
  sanityCheck(&h, inputPitchBytes);

  assert(inputPitchBytes >= outPixelBytes);
  uint32_t skipBytes = inputPitchBytes - outPixelBytes; // Skip per line

  if (oy > static_cast<uint64_t>(mRaw->dim.y))
    ThrowRDE("Invalid y offset");
  if (ox + size.x > static_cast<uint64_t>(mRaw->dim.x))
    ThrowRDE("Invalid x offset");

  uint64_t y = oy;
  h = min(h + oy, static_cast<uint64_t>(mRaw->dim.y));

  if (mRaw->getDataType() == RawImageType::F32) {
    if (bitPerPixel == 32) {
      const Array2DRef<float> out(mRaw->getF32DataAsUncroppedArray2DRef());
      copyPixels(reinterpret_cast<uint8_t*>(&out(y, offset.x * cpp)), outPitch,
                 input.getData(inputPitchBytes * (h - y)), inputPitchBytes,
                 w * mRaw->getBpp(), h - y);
      return;
    }
    if (BitOrder::MSB == order && bitPerPixel == 16) {
      decodePackedFP<BitPumpMSB, ieee_754_2008::Binary16>(size, offset,
                                                          skipBytes, h, y);
      return;
    }
    if (BitOrder::LSB == order && bitPerPixel == 16) {
      decodePackedFP<BitPumpLSB, ieee_754_2008::Binary16>(size, offset,
                                                          skipBytes, h, y);
      return;
    }
    if (BitOrder::MSB == order && bitPerPixel == 24) {
      decodePackedFP<BitPumpMSB, ieee_754_2008::Binary24>(size, offset,
                                                          skipBytes, h, y);
      return;
    }
    if (BitOrder::LSB == order && bitPerPixel == 24) {
      decodePackedFP<BitPumpLSB, ieee_754_2008::Binary24>(size, offset,
                                                          skipBytes, h, y);
      return;
    }
    ThrowRDE("Unsupported floating-point input bitwidth/bit packing: %u / %u",
             bitPerPixel, static_cast<unsigned>(order));
  }

  if (BitOrder::MSB == order) {
    decodePackedInt<BitPumpMSB>(size, offset, skipBytes, h, y, bitPerPixel);
  } else if (BitOrder::MSB16 == order) {
{'fraction_non_alphanumeric': 0.10828589500316256, 'fraction_numerical': 0.024857685009487665, 'mean_word_length': 2.1691721787933456, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 101, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 64, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5829164', 'n_tokens_mistral': 5113, 'n_tokens_neox': 4772, 'n_words': 1390}