<commit_before>#include "JpegLoader.hpp"
#include <array>
#include <memory>
#include "jpeglib.h"

auto JpegLoader::Load(BufferType fileData) -> RawImage<BufferType>
{
    // The data that will be returned
    BufferType rawData;

    // Create decompress struct
    jpeg_decompress_struct cinfo;
    jpeg_create_decompress(&cinfo);

    // Set memory buffer as source
    jpeg_mem_src(&cinfo, fileData.data(), static_cast<unsigned long>(fileData.size()));
    if (!jpeg_read_header(&cinfo, TRUE))
    {
        mLastError = "Incorrect jpeg header";
        return RawImage<>(std::move(rawData), {0,0,0});
    }

    // Populate the decompress structure
    jpeg_start_decompress(&cinfo);

    // Gather image info
    uint32_t width = cinfo.output_width;
    uint32_t height = cinfo.output_height;
    uint32_t pixelSz = cinfo.output_components;
    //uint32_t stride = width * pixelSz;

    // Resize the ouput buffer
    rawData.resize(width * height * pixelSz);

    // Ptr to the start of the output buffer
    uint8_t* bufP = rawData.data();

    /*
    // Read by lines
    int nSamples;
    while (cinfo.output_scanline < cinfo.output_height)
    {
        nSamples = jpeg_read_scanlines(&cinfo, (JSAMPARRAY) &bufP, 1);
        bufP += nSamples * cinfo.image_width * cinfo.num_components;
    }
     */

    // Read whole file at once
    jpeg_read_raw_data(&cinfo, (JSAMPIMAGE) &bufP, height);

    // Release all internal buffers used by jpeg_start_decompress
    jpeg_finish_decompress(&cinfo);

    // Destroy decompress struct
    jpeg_destroy_decompress(&cinfo);<commit_msg>Used line scan approach when loading Jpeg images<commit_after>#include "JpegLoader.hpp"
#include <array>
#include <memory>
#include "jpeglib.h"

auto JpegLoader::Load(BufferType fileData) -> RawImage<BufferType>
{
    // The data that will be returned
    BufferType rawData;

    // Create decompress struct
    jpeg_decompress_struct cinfo;

    // Set the error mechanism
    jpeg_error_mgr errMgr;
    cinfo.err = jpeg_std_error(&errMgr);

    // Initialize decompression object
    jpeg_create_decompress(&cinfo);

    // Set memory buffer as source
    jpeg_mem_src(&cinfo, fileData.data(), static_cast<unsigned long>(fileData.size()));
    if (!jpeg_read_header(&cinfo, TRUE))
    {
        mLastError = "Incorrect jpeg header";
        return RawImage<>(std::move(rawData), {0,0,0});
    }

    // Populate the decompress structure
    jpeg_start_decompress(&cinfo);

    // Gather image info
    uint32_t width = cinfo.output_width;
    uint32_t height = cinfo.output_height;
    uint32_t pixelSz = cinfo.output_components;
    //uint32_t stride = width * pixelSz;

    // Resize the ouput buffer
    rawData.resize(width * height * pixelSz);

    // Ptr to the start of the output buffer
    uint8_t* bufP = rawData.data();

    // Read by lines
    int nSamples;
    while (cinfo.output_scanline < cinfo.output_height)
    {
        nSamples = jpeg_read_scanlines(&cinfo, (JSAMPARRAY) &bufP, 1);
        bufP += nSamples * cinfo.image_width * cinfo.num_components;
    }

    // Release all internal buffers used by jpeg_start_decompress
    jpeg_finish_decompress(&cinfo);

    // Destroy decompress struct
    jpeg_destroy_decompress(&cinfo);
{'fraction_non_alphanumeric': 0.10549043800123381, 'fraction_numerical': 0.008019740900678593, 'mean_word_length': 3.272727272727273, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4213995', 'n_tokens_mistral': 1089, 'n_tokens_neox': 1038, 'n_words': 252}