<commit_before>#include "thumbnailjob.h"
#include <string>
#include <memory>
#include <algorithm>
#include <libexif/exif-loader.h>
#include <QImageReader>
#include <QDir>
#include "thumbnailer.h"

#include "core/legacy/fm-config.h"

namespace Fm {

QThreadPool* ThumbnailJob::threadPool_ = nullptr;

bool ThumbnailJob::localFilesOnly_ = true;
int ThumbnailJob::maxThumbnailFileSize_ = 0;

ThumbnailJob::ThumbnailJob(FileInfoList files, int size):
    files_{std::move(files)},
    size_{size},
    md5Calc_{g_checksum_new(G_CHECKSUM_MD5)} {
}

ThumbnailJob::~ThumbnailJob() {
    g_checksum_free(md5Calc_);
    // qDebug("delete  ThumbnailJob");
}

void ThumbnailJob::exec() {
    for(auto& file: files_) {
        if(isCancelled()) {
            break;
        }
        auto image = loadForFile(file);
        Q_EMIT thumbnailLoaded(file, size_, image);
        results_.emplace_back(std::move(image));
    }
}

QImage ThumbnailJob::readImageFromStream(GInputStream* stream, size_t len) {
    // FIXME: should we set a limit here? Otherwise if len is too large, we can run out of memory.
    std::unique_ptr<unsigned char[]> buffer{new unsigned char[len]}; // allocate enough buffer
    unsigned char* pbuffer = buffer.get();
    size_t totalReadSize = 0;
    while(!isCancelled() && totalReadSize < len) {
        size_t bytesToRead = totalReadSize + 4096 > len ? len - totalReadSize : 4096;
        gssize readSize = g_input_stream_read(stream, pbuffer, bytesToRead, cancellable_.get(), nullptr);
        if(readSize == 0) { // end of file
            break;
        }
        else if(readSize == -1) { // error
            return QImage();
        }
        totalReadSize += readSize;
        pbuffer += readSize;
    }
    QImage image;
    image.loadFromData(buffer.get(), totalReadSize);
    return image;
}

QImage ThumbnailJob::loadForFile(const std::shared_ptr<const FileInfo> &file) {
    if(!file->canThumbnail()) {
        return QImage();
    }

    // thumbnails are stored in $XDG_CACHE_HOME/thumbnails/large|normal|failed
    QString thumbnailDir{g_get_user_cache_dir()};
    thumbnailDir += "/thumbnails/";

    // don't make thumbnails for files inside the thumbnail directory
    if(FilePath::fromLocalPath(thumbnailDir.toLocal8Bit().constData()).isParentOf(file->dirPath())) {
        return QImage();
    }

    const char* subdir = size_ > 128 ? "large" : "normal";
    thumbnailDir += subdir;

    // generate base name of the thumbnail  => {md5 of uri}.png
    auto origPath = file->path();
    auto uri = origPath.uri();

    char thumbnailName[32 + 5];
    // calculate md5 hash for the uri of the original file
    g_checksum_update(md5Calc_, reinterpret_cast<const unsigned char*>(uri.get()), -1);
    memcpy(thumbnailName, g_checksum_get_string(md5Calc_), 32);
    mempcpy(thumbnailName + 32, ".png", 5);
    g_checksum_reset(md5Calc_); // reset the checksum calculator for next use

    QString thumbnailFilename = thumbnailDir;
    thumbnailFilename += '/';
    thumbnailFilename += thumbnailName;
    // qDebug() << "thumbnail:" << file->getName().c_str() << thumbnailFilename;

    // try to load the thumbnail file if it exists
    QImage thumbnail{thumbnailFilename};<commit_msg>Use target URI for symlink thumbnails<commit_after>#include "thumbnailjob.h"
#include <string>
#include <memory>
#include <algorithm>
#include <libexif/exif-loader.h>
#include <QImageReader>
#include <QDir>
#include "thumbnailer.h"

#include "core/legacy/fm-config.h"

namespace Fm {

QThreadPool* ThumbnailJob::threadPool_ = nullptr;

bool ThumbnailJob::localFilesOnly_ = true;
int ThumbnailJob::maxThumbnailFileSize_ = 0;

ThumbnailJob::ThumbnailJob(FileInfoList files, int size):
    files_{std::move(files)},
    size_{size},
    md5Calc_{g_checksum_new(G_CHECKSUM_MD5)} {
}

ThumbnailJob::~ThumbnailJob() {
    g_checksum_free(md5Calc_);
    // qDebug("delete  ThumbnailJob");
}

void ThumbnailJob::exec() {
    for(auto& file: files_) {
        if(isCancelled()) {
            break;
        }
        auto image = loadForFile(file);
        Q_EMIT thumbnailLoaded(file, size_, image);
        results_.emplace_back(std::move(image));
    }
}

QImage ThumbnailJob::readImageFromStream(GInputStream* stream, size_t len) {
    // FIXME: should we set a limit here? Otherwise if len is too large, we can run out of memory.
    std::unique_ptr<unsigned char[]> buffer{new unsigned char[len]}; // allocate enough buffer
    unsigned char* pbuffer = buffer.get();
    size_t totalReadSize = 0;
    while(!isCancelled() && totalReadSize < len) {
        size_t bytesToRead = totalReadSize + 4096 > len ? len - totalReadSize : 4096;
        gssize readSize = g_input_stream_read(stream, pbuffer, bytesToRead, cancellable_.get(), nullptr);
        if(readSize == 0) { // end of file
            break;
        }
        else if(readSize == -1) { // error
            return QImage();
        }
        totalReadSize += readSize;
        pbuffer += readSize;
    }
    QImage image;
    image.loadFromData(buffer.get(), totalReadSize);
    return image;
}

QImage ThumbnailJob::loadForFile(const std::shared_ptr<const FileInfo> &file) {
    if(!file->canThumbnail()) {
        return QImage();
    }

    // thumbnails are stored in $XDG_CACHE_HOME/thumbnails/large|normal|failed
    QString thumbnailDir{g_get_user_cache_dir()};
    thumbnailDir += "/thumbnails/";

    // don't make thumbnails for files inside the thumbnail directory
    if(FilePath::fromLocalPath(thumbnailDir.toLocal8Bit().constData()).isParentOf(file->dirPath())) {
        return QImage();
    }

    const char* subdir = size_ > 128 ? "large" : "normal";
    thumbnailDir += subdir;

    // generate base name of the thumbnail  => {md5 of uri}.png
    auto origPath = file->path();
    CStrPtr uri;
    if(file->isSymlink()) {
        // use the symlink target in the name to update the thumbnail
        // if the file is changed to a symlink with the same time stamp
        auto target = file->target();
        if(!target.empty()) {
            uri = FilePath::fromLocalPath(target.c_str()).uri();
        }
    }
    if(!uri) {
        uri = origPath.uri();
    }

    char thumbnailName[32 + 5];
    // calculate md5 hash for the uri of the original file
    g_checksum_update(md5Calc_, reinterpret_cast<const unsigned char*>(uri.get()), -1);
    memcpy(thumbnailName, g_checksum_get_string(md5Calc_), 32);
    mempcpy(thumbnailName + 32, ".png", 5);
    g_checksum_reset(md5Calc_); // reset the checksum calculator for next use

    QString thumbnailFilename = thumbnailDir;
    thumbnailFilename += '/';
    thumbnailFilename += thumbnailName;
    // qDebug() << "thumbnail:" << file->getName().c_str() << thumbnailFilename;

    // try to load the thumbnail file if it exists
    QImage thumbnail{thumbnailFilename};
{'fraction_non_alphanumeric': 0.12536699941280094, 'fraction_numerical': 0.009688784497944804, 'mean_word_length': 3.384169884169884, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1328584', 'n_tokens_mistral': 2234, 'n_tokens_neox': 2184, 'n_words': 519}