<commit_before>/* ex: set ts=4 et: */


#include "httpd.h"
#include "http_config.h"
#include "http_log.h"
#include "apr_lib.h"
#include "apr_strings.h"
#include "apr_general.h"
#include "util_filter.h"
#include "apr_buckets.h"
#include "http_request.h"
#define APR_WANT_STRFUNC
#include "apr_want.h"

#include "imgmin.h"

static const char imgminFilterName[] = "IMGMIN";
module AP_MODULE_DECLARE_DATA imgmin_module;

/* per-... ? */
typedef struct {
    struct imgmin_options opt;
    apr_size_t bufferSize;
} imgmin_filter_config;

/*
 * NOTE: we need the entire file in a contiguous buffer (can ImageMagick handle a stream?)
 * and as far as I know the contents may come in multiple buckets, so we hardcode an
 * upper limit, allocate a buffer and append into it until we see the EOS, then
 * process it and pass it on
 */
#define DEFAULT_BUFFERSIZE (4 * 1024 * 1024)

static void *create_imgmin_server_config(apr_pool_t *p, server_rec *s)
{
    imgmin_filter_config *c = apr_pcalloc(p, sizeof *c);

    (void) imgmin_options_init(&c->opt);
    c->bufferSize = DEFAULT_BUFFERSIZE;
    /* intialize ImageMagick */
    MagickWandGenesis();
    return c;
}

static const char *imgmin_set_error_threshold(cmd_parms *cmd,
                                              void *dummy,
                                              const char *arg)
{
    imgmin_filter_config *c = ap_get_module_config(
                                cmd->server->module_config,
                                &imgmin_module);
    imgmin_opt_set_error_threshold(&c->opt, arg);
    return NULL;
}

typedef struct imgmin_ctx_t
{
    apr_bucket_brigade *bb;
    unsigned char *buffer;
    apr_size_t buflen;
} imgmin_ctx;

static apr_status_t imgmin_ctx_cleanup(void *data)
{
    imgmin_ctx *ctx = data;
    return APR_SUCCESS;
}

static void magickfree(void *data)
{
    (void) MagickRelinquishMemory(data);
}

/*
 * image file blob is in ctx->buffer[0..ctx->buflen)
 * read it into imagemagick, apply imgmin to it and append the resulting blob
 * into ctx->bb brigade
 */
static void do_imgmin(ap_filter_t *f, imgmin_ctx *ctx, imgmin_filter_config *c)
{
    MagickWand *mw = NewMagickWand();
    MagickReadImageBlob(mw, ctx->buffer, ctx->buflen);
    /*
     * FIXME TODO: implement per-file caching via content checksumming
     */
    {
        MagickWand *tmp = search_quality(mw, "-", &c->opt);
        size_t newsize = ctx->buflen + 1;
        unsigned char *blob = MagickGetImageBlob(tmp, &newsize);
        if (newsize > ctx->buflen)
        {
            (void) MagickRelinquishMemory(blob);
            blob = MagickGetImageBlob(mw, &newsize);
        }
        {
            apr_bucket *b = apr_bucket_heap_create((char *)blob,
                                                   newsize, magickfree,
                                                   f->c->bucket_alloc);
            APR_BRIGADE_INSERT_TAIL(ctx->bb, b);
        }
        DestroyMagickWand(tmp);
    }
    DestroyMagickWand(mw);
}

static apr_status_t imgmin_out_filter(ap_filter_t *f,
                                      apr_bucket_brigade *bb)
{
    apr_bucket *e;
    request_rec *r = f->r;
    imgmin_ctx *ctx = f->ctx;
    imgmin_filter_config *c;

    /* Do nothing if asked to filter nothing. */
    if (APR_BRIGADE_EMPTY(bb)) {
        return ap_pass_brigade(f->next, bb);
    }

    c = ap_get_module_config(r->server->module_config, &imgmin_module);

    /* If we don't have a context, we need to ensure that it is okay to send
     * the imgmind content.  If we have a context, that means we've done
     * this before and we liked it.
     * This could be not so nice if we always fail.  But, if we succeed,
     * we're in better shape.
     */
    if (!ctx) {
        char *token;
        const char *encoding;

        /* only work on main request/no subrequests */
        if (r->main != NULL) {
            ap_remove_output_filter(f);
            return ap_pass_brigade(f->next, bb);<commit_msg>cache integrated, but getting SIGTERM<commit_after>/* ex: set ts=4 et: */


#include "httpd.h"
#include "http_config.h"
#include "http_log.h"
#include "apr_lib.h"
#include "apr_strings.h"
#include "apr_general.h"
#include "util_filter.h"
#include "apr_buckets.h"
#include "apr_md5.h"
#include "http_request.h"
#define APR_WANT_STRFUNC
#include "apr_want.h"

#include "imgmin.h"

static const char imgminFilterName[] = "IMGMIN";
module AP_MODULE_DECLARE_DATA imgmin_module;

/* per-... ? */
typedef struct {
    struct imgmin_options opt;
    apr_size_t bufferSize;
} imgmin_filter_config;

/* TODO: make this runtime configurable */
#define MOD_IMGMIN_CACHE_PREFIX "/var/imgmin-cache"

/*
 * NOTE: we need the entire file in a contiguous buffer (can ImageMagick handle a stream?)
 * and as far as I know the contents may come in multiple buckets, so we hardcode an
 * upper limit, allocate a buffer and append into it until we see the EOS, then
 * process it and pass it on
 */
#define DEFAULT_BUFFERSIZE (4 * 1024 * 1024)

static void *create_imgmin_server_config(apr_pool_t *p, server_rec *s)
{
    imgmin_filter_config *c = apr_pcalloc(p, sizeof *c);

    (void) imgmin_options_init(&c->opt);
    c->bufferSize = DEFAULT_BUFFERSIZE;
    /* intialize ImageMagick */
    MagickWandGenesis();
    return c;
}

static const char *imgmin_set_error_threshold(cmd_parms *cmd,
                                              void *dummy,
                                              const char *arg)
{
    imgmin_filter_config *c = ap_get_module_config(
                                cmd->server->module_config,
                                &imgmin_module);
    imgmin_opt_set_error_threshold(&c->opt, arg);
    return NULL;
}

typedef struct imgmin_ctx_t
{
    apr_bucket_brigade *bb;
    unsigned char *buffer;
    apr_size_t buflen;
} imgmin_ctx;

static apr_status_t imgmin_ctx_cleanup(void *data)
{
    imgmin_ctx *ctx = data;
    return APR_SUCCESS;
}

static void magickfree(void *data)
{
    (void) MagickRelinquishMemory(data);
}

static char * cache_path(unsigned char *blob, size_t len, char path[PATH_MAX])
{
    unsigned char digest[APR_MD5_DIGESTSIZE];
    char *result = NULL;

    if (apr_md5(digest, blob, len) == APR_SUCCESS)
    {
        int fmt;

        fmt = snprintf(path, sizeof path, "%s/%c%c/%c%c/%s",
            MOD_IMGMIN_CACHE_PREFIX, digest[0], digest[1],
            digest[2], digest[3], digest+4);
        if (fmt >= 0 && fmt < sizeof path)
        {
            result = path;
        }
    }
    return result;
}

static int mkdir_n(const char path[PATH_MAX], size_t len)
{
    char tmppath[PATH_MAX];

    strcpy(tmppath, path);
    tmppath[len] = '\0';
    fprintf(stderr, "mkdir(%s)\n", tmppath);
    if (mkdir(tmppath, 0644) != 0)
    {
        if (errno != EEXIST)
        {
            perror("mkdir");
        }
        return 0;
    }
    return 1;
}

/*
 * given a path of prefix/XX/YY/ZZZZZZZZZZZZZZZ
 *  ensure that prefix/XX/YY exists
 */
static void cache_path_ensure(const char path[PATH_MAX])
{
    size_t prelen = 1 + strcspn(path+1, "/");
    (void) mkdir_n(path, prelen+3);
    (void) mkdir_n(path, prelen+3+3);
}

static void cache_set(unsigned char *blob, size_t len)
{
    char path[PATH_MAX];
    MagickWand *cached = NULL;

    if (cache_path(blob, len, path))
    {
        FILE *fd;

        cache_path_ensure(path);
        /* FIXME: need to `mkdir -p` to ensure subdirs exist */
        fd = fopen(path, "wb");
        if (fd)
        {
            if (fwrite(blob, 1, len, fd) != len)
            {
                /* write to an error log...? */
                perror("fwrite");
            }
            fclose(fd);
        } else {
            perror("fopen");
        }
    }
}

/*
 * given an image in a buffer, attempt to locate and retrieve it in our cache.
 * upon error or the file not being found return NULL
 */
static MagickWand * cache_get(imgmin_ctx *ctx)
{
    char path[PATH_MAX];
    MagickWand *cached = NULL;

    if (cache_path(ctx->buffer, ctx->buflen, path))
    {
        FILE *fd;

        fd = fopen(path, "rb");
        if (fd)
        {
            cached = NewMagickWand();
            if (cached)
            {
                if (MagickTrue != MagickReadImageFile(cached, fd))
                {
                    cached = DestroyMagickWand(cached);
                }
            }
            fclose(fd);
        } else {
            perror("fopen");
        }
    }
    return cached;
}

/*
 * image file blob is in ctx->buffer[0..ctx->buflen)
 * read it into imagemagick, apply imgmin to it and append the resulting blob
 * into ctx->bb brigade
 */
static void do_imgmin(ap_filter_t *f, imgmin_ctx *ctx, imgmin_filter_config *c)
{
    MagickWand *mw;
    unsigned char *blob;
    size_t bloblen;

    mw = cache_get(ctx);
    blob = NULL;
    bloblen = 0;

    if (mw)
    {
        blob = MagickGetImageBlob(mw, &bloblen);
    } else {
        /*
         * not in cache. generate result and save to cache.
         */
        MagickWand *tmp;
        mw = NewMagickWand();
        MagickReadImageBlob(mw, ctx->buffer, ctx->buflen);
        tmp = search_quality(mw, "-", &c->opt);
        blob = MagickGetImageBlob(tmp, &bloblen);
        /* if result is larger than original fall back */
        if (bloblen > ctx->buflen)
        {
            (void) MagickRelinquishMemory(blob);
            blob = MagickGetImageBlob(mw, &bloblen);
        }
        /*
         * if the results aren't from the cache, write to the cache for later use
         */
        cache_set(blob, bloblen);
        tmp = DestroyMagickWand(tmp);
    }
    /*
     *  by this point we've got the contents of our image response in 'blob',
     *  whether it's a cached image, a new response or the original image.
     */
    {
        apr_bucket *b = apr_bucket_heap_create((char *)blob,
                                               bloblen, magickfree,
                                               f->c->bucket_alloc);
        APR_BRIGADE_INSERT_TAIL(ctx->bb, b);
    }
    mw = DestroyMagickWand(mw);
}

static apr_status_t imgmin_out_filter(ap_filter_t *f,
                                      apr_bucket_brigade *bb)
{
    apr_bucket *e;
    request_rec *r = f->r;
    imgmin_ctx *ctx = f->ctx;
    imgmin_filter_config *c;

    /* Do nothing if asked to filter nothing. */
    if (APR_BRIGADE_EMPTY(bb)) {
        return ap_pass_brigade(f->next, bb);
    }

    c = ap_get_module_config(r->server->module_config, &imgmin_module);

    /* If we don't have a context, we need to ensure that it is okay to send
     * the imgmind content.  If we have a context, that means we've done
     * this before and we liked it.
     * This could be not so nice if we always fail.  But, if we succeed,
     * we're in better shape.
     */
    if (!ctx) {
        char *token;
        const char *encoding;

        /* only work on main request/no subrequests */
        if (r->main != NULL) {
            ap_remove_output_filter(f);
            return ap_pass_brigade(f->next, bb);
{'fraction_non_alphanumeric': 0.10822825002267986, 'fraction_numerical': 0.004263812029393087, 'mean_word_length': 2.209315866084425, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5415558', 'n_tokens_mistral': 3721, 'n_tokens_neox': 3615, 'n_words': 977}