<commit_before>
/*
 * Author: <NAME>(<EMAIL>)
 * Licence:This module could be distributed under the same terms as Nginx itself.
 */


#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>
#include <nginx.h>
#include <ngx_http_chain_buffer.h>


#if (NGX_DEBUG)
#define SUBS_DEBUG 1
#else
#define SUBS_DEBUG 0
#endif


#ifndef NGX_HTTP_MAX_CAPTURES
#define NGX_HTTP_MAX_CAPTURES 9
#endif


typedef struct {
    ngx_flag_t     once;
    ngx_flag_t     regex;
    ngx_flag_t     insensitive;

    /* If it has captured variables? */
    ngx_flag_t     dup_capture;

    ngx_str_t      match;
#if (NGX_PCRE)
    ngx_regex_t   *match_regex;
    int           *captures;
    ngx_int_t      ncaptures;
#endif

    ngx_str_t      sub;
    ngx_array_t   *sub_lengths;
    ngx_array_t   *sub_values;

    unsigned       matched; 
} sub_pair_t;


typedef struct {
    ngx_uint_t     max_linesize;
    ngx_hash_t     types;
    ngx_array_t   *sub_pairs;   /* array of sub_pair_t     */
    ngx_array_t   *types_keys;  /* array of ngx_hash_key_t */
} ngx_http_subs_loc_conf_t;


typedef struct {
    ngx_array_t   *sub_pairs;  /* array of sub_pair_t */

    ngx_chain_t   *in;

    /* 
     *  I use queue to store output buffer chain. The reason is that the
     *  overhead of fetching tail chain is too high when there are 1000+ 
     *  substitutions in the chain buffers. 
     *  */
    ngx_queue_buf_t *out;

    ngx_chain_t   *busy;

    /* store the freed chain buffers. */
    ngx_chain_t   *free;

    /* store the freed queue buffers. */
    ngx_queue_buf_t *free_queue;

    /* alloced by ctx->tpool */
    ngx_chain_t   *line_in;

    /* the temporary buffer to store the line buffer before substitution */
    ngx_buf_t     *line_src;
    /* the temporary buffer to store the line buffer after substitution */
    ngx_buf_t     *line_dst;

    /* alloced by r->pool */
    ngx_chain_t   *line_out;

    /* save every chain which does not contain the linefeed */
    ngx_chain_t   *saved;

    /* 
     * last_pos is the last not matched postion, the chain will
     * not be splited until a successful matching. This will reduce 
     * the split frequency of the chain. 
     * */
    u_char        *last_pos;

    ngx_pool_t    *tpool;
} ngx_http_subs_ctx_t;


static ngx_int_t ngx_http_subs_header_filter(ngx_http_request_t *r);
static ngx_int_t ngx_http_subs_init_context(ngx_http_request_t *r);

static ngx_int_t ngx_http_subs_body_filter(ngx_http_request_t *r,
    ngx_chain_t *in);
static ngx_int_t ngx_http_subs_body_filter_init_context(ngx_http_request_t *r,
    ngx_chain_t *in);
static ngx_int_t ngx_http_subs_body_filter_process_chain(ngx_http_request_t *r,
    ngx_chain_t *cl);
static ngx_int_t  ngx_http_subs_match(ngx_http_request_t *r,
    ngx_http_subs_ctx_t *ctx);
static ngx_buf_t *ngx_http_subs_match_read_chain_to_buffer(ngx_chain_t *in,
    ngx_buf_t *b, ngx_pool_t *tpool, ngx_flag_t quick);
static ngx_int_t ngx_http_subs_match_regex_substituion(ngx_http_request_t *r,
    sub_pair_t *pair, ngx_buf_t *b, ngx_buf_t *dst, ngx_pool_t *tpool); 
static ngx_int_t ngx_http_subs_match_fix_substituion(sub_pair_t *pair, 
    ngx_buf_t *b, ngx_buf_t *dst, ngx_pool_t *tpool);
static void ngx_http_subs_body_filter_greedy_split_chain(ngx_http_request_t *r,
    ngx_http_subs_ctx_t *ctx, ngx_chain_t *cl);
static void ngx_http_subs_body_filter_clean_context(ngx_http_request_t *r,
    ngx_http_subs_ctx_t *ctx);
static ngx_int_t ngx_http_subs_output(ngx_http_request_t *r,
    ngx_http_subs_ctx_t *ctx, ngx_chain_t *in);
static void ngx_http_subs_output_free_chain(ngx_http_subs_ctx_t *ctx);


static char * ngx_http_subs_filter(ngx_conf_t *cf, ngx_command_t *cmd,
    void *conf);
static ngx_int_t ngx_http_subs_filter_regex_compile(sub_pair_t *pair, 
    ngx_http_script_compile_t *sc, ngx_conf_t *cf);


static void *ngx_http_subs_create_conf(ngx_conf_t *cf);
static char *ngx_http_subs_merge_conf(ngx_conf_t *cf, void *parent,
    void *child);

static ngx_int_t ngx_http_subs_filter_init(ngx_conf_t *cf);

static ngx_int_t ngx_http_subs_regex_capture_count(ngx_regex_t *re);


static ngx_command_t  ngx_http_subs_filter_commands[] = {

    { ngx_string("subs_filter"),
      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_2MORE,
      ngx_http_subs_filter,
      NGX_HTTP_LOC_CONF_OFFSET,
      0,
      NULL },

    { ngx_string("subs_filter_types"),
      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_1MORE,
      ngx_http_types_slot,
      NGX_HTTP_LOC_CONF_OFFSET,
      offsetof(ngx_http_subs_loc_conf_t, types_keys),
      &ngx_http_html_default_types[0] },

    { ngx_string("subs_filter_max_linesize"),
      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
      ngx_conf_set_num_slot,
      NGX_HTTP_LOC_CONF_OFFSET,
      offsetof(ngx_http_subs_loc_conf_t, max_linesize),
      NULL },

    ngx_null_command
};


static ngx_http_module_t  ngx_http_subs_filter_module_ctx = {
    NULL,                                  /* preconfiguration */
    ngx_http_subs_filter_init,             /* postconfiguration */

    NULL,                                  /* create main configuration */
    NULL,                                  /* init main configuration */

    NULL,                                  /* create server configuration */
    NULL,                                  /* merge server configuration */

    ngx_http_subs_create_conf,             /* create location configuration */
    ngx_http_subs_merge_conf               /* merge location configuration */
};


ngx_module_t  ngx_http_subs_filter_module = {
    NGX_MODULE_V1,
    &ngx_http_subs_filter_module_ctx,      /* module context */
    ngx_http_subs_filter_commands,         /* module directives */
    NGX_HTTP_MODULE,                       /* module type */
    NULL,                                  /* init master */
    NULL,                                  /* init module */
    NULL,                                  /* init process */
    NULL,                                  /* init thread */
    NULL,                                  /* exit thread */
    NULL,                                  /* exit process */
    NULL,                                  /* exit master */
    NGX_MODULE_V1_PADDING
};


static ngx_http_output_header_filter_pt  ngx_http_next_header_filter;
static ngx_http_output_body_filter_pt    ngx_http_next_body_filter;

extern volatile ngx_cycle_t  *ngx_cycle;


static ngx_int_t 
ngx_http_subs_header_filter(ngx_http_request_t *r)
{
    ngx_http_subs_loc_conf_t  *slcf;

    slcf = ngx_http_get_module_loc_conf(r, ngx_http_subs_filter_module);

    /*Don't substitute the compressed content*/
    if (slcf->sub_pairs->nelts == 0
        || r->header_only
        || r->headers_out.content_type.len == 0
        || r->headers_out.content_length_n == 0 
        || r->headers_out.status != NGX_HTTP_OK
        || (r->headers_out.content_encoding  
            && r->headers_out.content_encoding->value.len))
    {
        ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
                       "http subs filter header ignored, this may be a special "
                       "or compressed response");

        return ngx_http_next_header_filter(r);
    }

    /* TODO: use the test_types interface */
    if (ngx_http_test_content_type(r, &slcf->types) == NULL) {
        return ngx_http_next_header_filter(r);
    }

    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
                   "http subs filter header \"%V\"", &r->uri);

    if (ngx_http_subs_init_context(r) == NGX_ERROR) {
        return NGX_ERROR;
    }

    r->filter_need_in_memory = 1;

    if (r == r->main) {
        ngx_http_clear_content_length(r);
        ngx_http_clear_last_modified(r);
    }

    return ngx_http_next_header_filter(r);
}


static ngx_int_t
ngx_http_subs_init_context(ngx_http_request_t *r)
{
    ngx_uint_t                 i;
    sub_pair_t                *src_pair, *dst_pair;
    ngx_http_subs_ctx_t       *ctx;
    ngx_http_subs_loc_conf_t  *slcf;

    slcf = ngx_http_get_module_loc_conf(r, ngx_http_subs_filter_module);

    /* Everything in ctx is NULL or 0. */
    ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_subs_ctx_t));
    if (ctx == NULL) {
        return NGX_ERROR;
    }

    ngx_http_set_ctx(r, ctx, ngx_http_subs_filter_module);

    ctx->sub_pairs = ngx_array_create(r->pool, slcf->sub_pairs->nelts,
                                      sizeof(sub_pair_t));
    if (slcf->sub_pairs == NULL) {
        return NGX_ERROR;
    }

    /* Deep copy sub_pairs from slcf to ctx */
    src_pair = (sub_pair_t *) slcf->sub_pairs->elts;

    for (i = 0; i < slcf->sub_pairs->nelts; i++) {

        dst_pair = ngx_array_push(ctx->sub_pairs);
        if (dst_pair == NULL) {
            return NGX_ERROR;
        }

        ngx_memcpy(dst_pair, src_pair + i, sizeof(sub_pair_t));
    }

    ctx->out = ngx_palloc(r->pool, sizeof(ngx_queue_buf_t));
    ngx_queue_init(&ctx->out->queue);

    ctx->free_queue = ngx_palloc(r->pool, sizeof(ngx_queue_buf_t));
    ngx_queue_init(&ctx->free_queue->queue);

    return NGX_OK;
}


static ngx_int_t 
ngx_http_subs_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
{
    ngx_buf_t                 *b = NULL;
    ngx_int_t		           rc; 
    ngx_log_t                 *log;
    ngx_chain_t               *cl;
    ngx_queue_t               *q;
    ngx_queue_buf_t           *qb;
    ngx_http_subs_ctx_t       *ctx;
    ngx_http_subs_loc_conf_t  *slcf;

    log = r->connection->log;

    slcf = ngx_http_get_module_loc_conf(r, ngx_http_subs_filter_module);
    if (slcf == NULL) {
        return ngx_http_next_body_filter(r, in);
    }

    ctx = ngx_http_get_module_ctx(r, ngx_http_subs_filter_module);
    if (ctx == NULL) {
        return ngx_http_next_body_filter(r, in);
    }

    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, log, 0, 
                   "http subs filter \"%V\"", &r->uri);

    if (in == NULL && ctx->in == NULL && ctx->busy == NULL) {
        return ngx_http_next_body_filter(r, in);
    }

    if (ngx_http_subs_body_filter_init_context(r, in) != NGX_OK){
        goto failed;
    }

    for (cl = ctx->in; cl; cl = cl->next) {

        rc = ngx_http_subs_body_filter_process_chain(r, cl);

        if (rc == NGX_DECLINED) {
            continue;
        }
        else if (rc == NGX_ERROR) {
            goto failed;
        }

        /* NGX_OK */
        if (ngx_queue_empty(&ctx->out->queue)) {
            qb = ngx_alloc_queue_buf(r->pool, ctx->free_queue);
            b = create_buffer(NULL, 0, r->pool);
            
            if (qb == NULL || b == NULL) {
                goto failed;
            }

            qb->buf = b;
            
            ngx_queue_insert_tail(&ctx->out->queue, &qb->queue);     
        }

        /* Add the shadow buffer for freeing after output */
        if (ngx_buf_in_memory(cl->buf)) {

            q = ngx_queue_last(&ctx->out->queue);
            qb = ngx_queue_data(q, ngx_queue_buf_t, queue);
            b = qb->buf;

            ngx_log_debug3(NGX_LOG_DEBUG_HTTP, log, 0,
                    "[subs_filter] add the shadow buffer: b=%p, b->shadow=%p, "
                    "cl->buf->shadow=%p", b, cl->buf, cl->buf->shadow);

            if (b) {
                insert_shadow_tail(&b->shadow, cl->buf);
                cl->buf->last_shadow = 1;
            }
            else {
                b = ngx_pcalloc(r->pool, sizeof(ngx_buf_t));
                b->shadow = cl->buf;
                b->sync = 1;
                cl->buf->last_shadow = 1;

                qb->buf = b;
            }
        }

        /* copy line_in to saved. */
        if (ctx->line_in) {
            delete_and_free_chain(&ctx->saved, &ctx->free);
            ctx->saved = duplicate_chains(ctx->line_in, &ctx->free, r->pool);
            if (ctx->saved == NULL) {
                goto failed;
            }
        }

        if (cl->next != NULL) {
            continue;
        }

        /* Last chain */
        if (cl->buf->last_buf){

            if (ctx->saved) {

                ngx_queue_chain_add_copy(r->pool, &ctx->out->queue, 
                                         ctx->saved, ctx->free_queue);

                ngx_log_debug0(NGX_LOG_DEBUG_HTTP, log, 0,
                               "[subs_filter] Lost last linefeed, "
                               "but output anyway.");
            }

            q = ngx_queue_last(&ctx->out->queue);
            qb = ngx_queue_data(q, ngx_queue_buf_t, queue);
            b = qb->buf;

            if (b == NULL) {
                b = ngx_pcalloc(r->pool, sizeof(ngx_buf_t));
                b->sync = 1;
            }

            b->last_buf = 1;
        }
    }

    ngx_http_subs_body_filter_clean_context(r, ctx);

    if (ngx_queue_empty(&ctx->out->queue) && ctx->busy == NULL) {
        return NGX_OK;
    }

    return ngx_http_subs_output(r, ctx, in);

failed:

    ngx_http_subs_body_filter_clean_context(r, ctx);

    ngx_log_error(NGX_LOG_ERR, log, 0, 
                  "[subs_filter] ngx_http_subs_body_filter error.");

    return NGX_ERROR;
}


static ngx_int_t 
ngx_http_subs_body_filter_init_context(ngx_http_request_t *r, ngx_chain_t *in) 
{
    size_t                     pool_size;
    ngx_log_t                 *log;
    ngx_chain_t               *cl;
    ngx_http_subs_ctx_t       *ctx;

    log = r->connection->log;

    ctx = ngx_http_get_module_ctx(r, ngx_http_subs_filter_module);
    if (ctx == NULL) {
        return ngx_http_next_body_filter(r, in);
    }

    if (in) {
        /*TODO: Deepcopy*/
        if (ngx_chain_add_copy(r->pool, &ctx->in, in) == NGX_ERROR) {
            return NGX_ERROR;
        }

        pool_size = 0;
        for (cl = ctx->in; cl; cl = cl->next) {
            if (cl->buf) {
                pool_size += ngx_buf_size(cl->buf);
                ngx_log_debug4(NGX_LOG_DEBUG_HTTP, log, 0,
                               "subs in buffer: %p, size:%uz, "
                               "sync:%d, last_buf:%d", 
                               cl->buf, ngx_buf_size(cl->buf),
                               cl->buf->sync, cl->buf->last_buf);
            }
        }


        pool_size = ngx_align(pool_size, ngx_pagesize) + ngx_pagesize;

        /*
         * tpool only exists in body filter. It's used
         * for the chain of 'ctx->line_in's temporary memory
         * allocation
         * */
        ctx->tpool = ngx_create_pool(pool_size, log);
        if (ctx->tpool == NULL) {
            return NGX_ERROR;
        }
        
        ctx->line_src = ngx_create_temp_buf(ctx->tpool, 1024);
        if (ctx->line_src == NULL) {
            return NGX_ERROR;
        }

        ctx->line_dst = ngx_create_temp_buf(ctx->tpool, 1024);
        if (ctx->line_dst == NULL) {
            return NGX_ERROR;
        }

        if (ctx->saved) {
#if SUBS_DEBUG
            for (cl = ctx->saved; cl; cl = cl->next) {
                if (cl->buf) {
                    ngx_log_debug3(NGX_LOG_DEBUG_HTTP, log, 0,
                                   "subs in saved: %p, size:%uz, sync:%d",
                                   cl->buf, ngx_buf_size(cl->buf), cl->buf->sync);
                }
            }
#endif
            ctx->line_in = duplicate_chains(ctx->saved, NULL, ctx->tpool);
            if (ctx->line_in == NULL) {
                return NGX_ERROR;
            }
        }
    }

    return NGX_OK;
}


static ngx_int_t 
ngx_http_subs_body_filter_process_chain(ngx_http_request_t *r, ngx_chain_t *cl)
{
    size_t                     linesize;
    u_char                    *p, *linefeed;
    ngx_log_t                 *log;
    ngx_buf_t                 *b;
    ngx_int_t                  len, rc; 
    ngx_chain_t               *temp_cl;
    ngx_chain_t               *part_line_in_cl;
    ngx_http_subs_ctx_t       *ctx;<commit_msg>add the deep copy function<commit_after>
/*
 * Author: <NAME>(<EMAIL>)
 * Licence:This module could be distributed under the same terms as Nginx itself.
 */


#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>
#include <nginx.h>
#include <ngx_http_chain_buffer.h>


#if (NGX_DEBUG)
#define SUBS_DEBUG 1
#else
#define SUBS_DEBUG 0
#endif


#ifndef NGX_HTTP_MAX_CAPTURES
#define NGX_HTTP_MAX_CAPTURES 9
#endif


typedef struct {
    ngx_flag_t     once;
    ngx_flag_t     regex;
    ngx_flag_t     insensitive;

    /* If it has captured variables? */
    ngx_flag_t     dup_capture;

    ngx_str_t      match;
#if (NGX_PCRE)
    ngx_regex_t   *match_regex;
    int           *captures;
    ngx_int_t      ncaptures;
#endif

    ngx_str_t      sub;
    ngx_array_t   *sub_lengths;
    ngx_array_t   *sub_values;

    unsigned       matched; 
} sub_pair_t;


typedef struct {
    ngx_uint_t     max_linesize;
    ngx_hash_t     types;
    ngx_array_t   *sub_pairs;   /* array of sub_pair_t     */
    ngx_array_t   *types_keys;  /* array of ngx_hash_key_t */
} ngx_http_subs_loc_conf_t;


typedef struct {
    ngx_array_t   *sub_pairs;  /* array of sub_pair_t */

    ngx_chain_t   *in;

    /* 
     *  I use queue to store output buffer chain. The reason is that the
     *  overhead of fetching tail chain is too high when there are 1000+ 
     *  substitutions in the chain buffers. 
     *  */
    ngx_queue_buf_t *out;

    ngx_chain_t   *busy;

    /* store the freed chain buffers. */
    ngx_chain_t   *free;

    /* store the freed queue buffers. */
    ngx_queue_buf_t *free_queue;

    /* alloced by ctx->tpool */
    ngx_chain_t   *line_in;

    /* the temporary buffer to store the line buffer before substitution */
    ngx_buf_t     *line_src;
    /* the temporary buffer to store the line buffer after substitution */
    ngx_buf_t     *line_dst;

    /* alloced by r->pool */
    ngx_chain_t   *line_out;

    /* save every chain which does not contain the linefeed */
    ngx_chain_t   *saved;

    /* 
     * last_pos is the last not matched postion, the chain will
     * not be splited until a successful matching. This will reduce 
     * the split frequency of the chain. 
     * */
    u_char        *last_pos;

    ngx_pool_t    *tpool;
} ngx_http_subs_ctx_t;


static ngx_int_t ngx_http_subs_header_filter(ngx_http_request_t *r);
static ngx_int_t ngx_http_subs_init_context(ngx_http_request_t *r);

static ngx_int_t ngx_http_subs_body_filter(ngx_http_request_t *r,
    ngx_chain_t *in);
static ngx_int_t ngx_http_subs_body_filter_init_context(ngx_http_request_t *r,
    ngx_chain_t *in);
static ngx_int_t ngx_http_subs_deep_copy_chain(ngx_pool_t *pool,
    ngx_chain_t **chain, ngx_chain_t *in);
static ngx_int_t ngx_http_subs_body_filter_process_chain(ngx_http_request_t *r,
    ngx_chain_t *cl);
static ngx_int_t  ngx_http_subs_match(ngx_http_request_t *r,
    ngx_http_subs_ctx_t *ctx);
static ngx_buf_t *ngx_http_subs_match_read_chain_to_buffer(ngx_chain_t *in,
    ngx_buf_t *b, ngx_pool_t *tpool, ngx_flag_t quick);
static ngx_int_t ngx_http_subs_match_regex_substituion(ngx_http_request_t *r,
    sub_pair_t *pair, ngx_buf_t *b, ngx_buf_t *dst, ngx_pool_t *tpool); 
static ngx_int_t ngx_http_subs_match_fix_substituion(sub_pair_t *pair, 
    ngx_buf_t *b, ngx_buf_t *dst, ngx_pool_t *tpool);
static void ngx_http_subs_body_filter_greedy_split_chain(ngx_http_request_t *r,
    ngx_http_subs_ctx_t *ctx, ngx_chain_t *cl);
static void ngx_http_subs_body_filter_clean_context(ngx_http_request_t *r,
    ngx_http_subs_ctx_t *ctx);
static ngx_int_t ngx_http_subs_output(ngx_http_request_t *r,
    ngx_http_subs_ctx_t *ctx, ngx_chain_t *in);
static void ngx_http_subs_output_free_chain(ngx_http_subs_ctx_t *ctx);


static char * ngx_http_subs_filter(ngx_conf_t *cf, ngx_command_t *cmd,
    void *conf);
static ngx_int_t ngx_http_subs_filter_regex_compile(sub_pair_t *pair, 
    ngx_http_script_compile_t *sc, ngx_conf_t *cf);


static void *ngx_http_subs_create_conf(ngx_conf_t *cf);
static char *ngx_http_subs_merge_conf(ngx_conf_t *cf, void *parent,
    void *child);

static ngx_int_t ngx_http_subs_filter_init(ngx_conf_t *cf);

static ngx_int_t ngx_http_subs_regex_capture_count(ngx_regex_t *re);


static ngx_command_t  ngx_http_subs_filter_commands[] = {

    { ngx_string("subs_filter"),
      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_2MORE,
      ngx_http_subs_filter,
      NGX_HTTP_LOC_CONF_OFFSET,
      0,
      NULL },

    { ngx_string("subs_filter_types"),
      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_1MORE,
      ngx_http_types_slot,
      NGX_HTTP_LOC_CONF_OFFSET,
      offsetof(ngx_http_subs_loc_conf_t, types_keys),
      &ngx_http_html_default_types[0] },

    { ngx_string("subs_filter_max_linesize"),
      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
      ngx_conf_set_num_slot,
      NGX_HTTP_LOC_CONF_OFFSET,
      offsetof(ngx_http_subs_loc_conf_t, max_linesize),
      NULL },

    ngx_null_command
};


static ngx_http_module_t  ngx_http_subs_filter_module_ctx = {
    NULL,                                  /* preconfiguration */
    ngx_http_subs_filter_init,             /* postconfiguration */

    NULL,                                  /* create main configuration */
    NULL,                                  /* init main configuration */

    NULL,                                  /* create server configuration */
    NULL,                                  /* merge server configuration */

    ngx_http_subs_create_conf,             /* create location configuration */
    ngx_http_subs_merge_conf               /* merge location configuration */
};


ngx_module_t  ngx_http_subs_filter_module = {
    NGX_MODULE_V1,
    &ngx_http_subs_filter_module_ctx,      /* module context */
    ngx_http_subs_filter_commands,         /* module directives */
    NGX_HTTP_MODULE,                       /* module type */
    NULL,                                  /* init master */
    NULL,                                  /* init module */
    NULL,                                  /* init process */
    NULL,                                  /* init thread */
    NULL,                                  /* exit thread */
    NULL,                                  /* exit process */
    NULL,                                  /* exit master */
    NGX_MODULE_V1_PADDING
};


static ngx_http_output_header_filter_pt  ngx_http_next_header_filter;
static ngx_http_output_body_filter_pt    ngx_http_next_body_filter;

extern volatile ngx_cycle_t  *ngx_cycle;


static ngx_int_t 
ngx_http_subs_header_filter(ngx_http_request_t *r)
{
    ngx_http_subs_loc_conf_t  *slcf;

    slcf = ngx_http_get_module_loc_conf(r, ngx_http_subs_filter_module);

    /*Don't substitute the compressed content*/
    if (slcf->sub_pairs->nelts == 0
        || r->header_only
        || r->headers_out.content_type.len == 0
        || r->headers_out.content_length_n == 0 
        || r->headers_out.status != NGX_HTTP_OK
        || (r->headers_out.content_encoding  
            && r->headers_out.content_encoding->value.len))
    {
        ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
                       "http subs filter header ignored, this may be a special "
                       "or compressed response");

        return ngx_http_next_header_filter(r);
    }

    if (ngx_http_test_content_type(r, &slcf->types) == NULL) {
        return ngx_http_next_header_filter(r);
    }

    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
                   "http subs filter header \"%V\"", &r->uri);

    if (ngx_http_subs_init_context(r) == NGX_ERROR) {
        return NGX_ERROR;
    }

    r->filter_need_in_memory = 1;

    if (r == r->main) {
        ngx_http_clear_content_length(r);
        ngx_http_clear_last_modified(r);
    }

    return ngx_http_next_header_filter(r);
}


static ngx_int_t
ngx_http_subs_init_context(ngx_http_request_t *r)
{
    ngx_uint_t                 i;
    sub_pair_t                *src_pair, *dst_pair;
    ngx_http_subs_ctx_t       *ctx;
    ngx_http_subs_loc_conf_t  *slcf;

    slcf = ngx_http_get_module_loc_conf(r, ngx_http_subs_filter_module);

    /* Everything in ctx is NULL or 0. */
    ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_subs_ctx_t));
    if (ctx == NULL) {
        return NGX_ERROR;
    }

    ngx_http_set_ctx(r, ctx, ngx_http_subs_filter_module);

    ctx->sub_pairs = ngx_array_create(r->pool, slcf->sub_pairs->nelts,
                                      sizeof(sub_pair_t));
    if (slcf->sub_pairs == NULL) {
        return NGX_ERROR;
    }

    /* Deep copy sub_pairs from slcf to ctx */
    src_pair = (sub_pair_t *) slcf->sub_pairs->elts;

    for (i = 0; i < slcf->sub_pairs->nelts; i++) {

        dst_pair = ngx_array_push(ctx->sub_pairs);
        if (dst_pair == NULL) {
            return NGX_ERROR;
        }

        ngx_memcpy(dst_pair, src_pair + i, sizeof(sub_pair_t));
    }

    ctx->out = ngx_palloc(r->pool, sizeof(ngx_queue_buf_t));
    ngx_queue_init(&ctx->out->queue);

    ctx->free_queue = ngx_palloc(r->pool, sizeof(ngx_queue_buf_t));
    ngx_queue_init(&ctx->free_queue->queue);

    return NGX_OK;
}


static ngx_int_t 
ngx_http_subs_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
{
    ngx_buf_t                 *b = NULL;
    ngx_int_t		           rc; 
    ngx_log_t                 *log;
    ngx_chain_t               *cl;
    ngx_queue_t               *q;
    ngx_queue_buf_t           *qb;
    ngx_http_subs_ctx_t       *ctx;
    ngx_http_subs_loc_conf_t  *slcf;

    log = r->connection->log;

    slcf = ngx_http_get_module_loc_conf(r, ngx_http_subs_filter_module);
    if (slcf == NULL) {
        return ngx_http_next_body_filter(r, in);
    }

    ctx = ngx_http_get_module_ctx(r, ngx_http_subs_filter_module);
    if (ctx == NULL) {
        return ngx_http_next_body_filter(r, in);
    }

    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, log, 0, 
                   "http subs filter \"%V\"", &r->uri);

    if (in == NULL && ctx->in == NULL && ctx->busy == NULL) {
        return ngx_http_next_body_filter(r, in);
    }

    if (ngx_http_subs_body_filter_init_context(r, in) != NGX_OK){
        goto failed;
    }

    for (cl = ctx->in; cl; cl = cl->next) {

        /* TODO: check the flush tag */
        rc = ngx_http_subs_body_filter_process_chain(r, cl);

        if (rc == NGX_DECLINED) {
            continue;
        }
        else if (rc == NGX_ERROR) {
            goto failed;
        }

        /* NGX_OK */
        if (ngx_queue_empty(&ctx->out->queue)) {
            qb = ngx_alloc_queue_buf(r->pool, ctx->free_queue);
            b = create_buffer(NULL, 0, r->pool);
            
            if (qb == NULL || b == NULL) {
                goto failed;
            }

            qb->buf = b;
            
            ngx_queue_insert_tail(&ctx->out->queue, &qb->queue);     
        }

        /* Add the shadow buffer for freeing after output */
        if (ngx_buf_in_memory(cl->buf)) {

            q = ngx_queue_last(&ctx->out->queue);
            qb = ngx_queue_data(q, ngx_queue_buf_t, queue);
            b = qb->buf;

            ngx_log_debug3(NGX_LOG_DEBUG_HTTP, log, 0,
                    "[subs_filter] add the shadow buffer: b=%p, b->shadow=%p, "
                    "cl->buf->shadow=%p", b, cl->buf, cl->buf->shadow);

            if (b) {
                insert_shadow_tail(&b->shadow, cl->buf);
                cl->buf->last_shadow = 1;
            }
            else {
                b = ngx_pcalloc(r->pool, sizeof(ngx_buf_t));
                b->shadow = cl->buf;
                b->sync = 1;
                cl->buf->last_shadow = 1;

                qb->buf = b;
            }
        }

        /* copy line_in to saved. */
        if (ctx->line_in) {
            delete_and_free_chain(&ctx->saved, &ctx->free);
            ctx->saved = duplicate_chains(ctx->line_in, &ctx->free, r->pool);
            if (ctx->saved == NULL) {
                goto failed;
            }
        }

        if (cl->next != NULL) {
            continue;
        }

        /* Last chain */
        if (cl->buf->last_buf){

            if (ctx->saved) {

                ngx_queue_chain_add_copy(r->pool, &ctx->out->queue, 
                                         ctx->saved, ctx->free_queue);

                ngx_log_debug0(NGX_LOG_DEBUG_HTTP, log, 0,
                               "[subs_filter] Lost last linefeed, "
                               "but output anyway.");
            }

            q = ngx_queue_last(&ctx->out->queue);
            qb = ngx_queue_data(q, ngx_queue_buf_t, queue);
            b = qb->buf;

            if (b == NULL) {
                b = ngx_pcalloc(r->pool, sizeof(ngx_buf_t));
                b->sync = 1;
            }

            b->last_buf = 1;
        }
    }

    ngx_http_subs_body_filter_clean_context(r, ctx);

    if (ngx_queue_empty(&ctx->out->queue) && ctx->busy == NULL) {
        return NGX_OK;
    }

    return ngx_http_subs_output(r, ctx, in);

failed:

    ngx_http_subs_body_filter_clean_context(r, ctx);

    ngx_log_error(NGX_LOG_ERR, log, 0, 
                  "[subs_filter] ngx_http_subs_body_filter error.");

    return NGX_ERROR;
}


static ngx_int_t 
ngx_http_subs_body_filter_init_context(ngx_http_request_t *r, ngx_chain_t *in) 
{
    size_t                     pool_size;
    ngx_log_t                 *log;
    ngx_chain_t               *cl;
    ngx_http_subs_ctx_t       *ctx;

    log = r->connection->log;

    ctx = ngx_http_get_module_ctx(r, ngx_http_subs_filter_module);
    if (ctx == NULL) {
        return ngx_http_next_body_filter(r, in);
    }

    if (in) {
        if (ngx_http_subs_deep_copy_chain(r->pool, &ctx->in, in) == NGX_ERROR) {
            return NGX_ERROR;
        }
    }

    pool_size = 0;
    for (cl = ctx->in; cl; cl = cl->next) {
        if (cl->buf) {
            pool_size += ngx_buf_size(cl->buf);
            ngx_log_debug4(NGX_LOG_DEBUG_HTTP, log, 0,
                    "subs in buffer: %p, size:%uz, "
                    "sync:%d, last_buf:%d", 
                    cl->buf, ngx_buf_size(cl->buf),
                    cl->buf->sync, cl->buf->last_buf);
        }
    }

    pool_size = ngx_align(pool_size, ngx_pagesize) + ngx_pagesize;

    /*
     * tpool only exists in body filter. It's used
     * for the chain of 'ctx->line_in's temporary memory
     * allocation
     * */
    ctx->tpool = ngx_create_pool(pool_size, log);
    if (ctx->tpool == NULL) {
        return NGX_ERROR;
    }

    ctx->line_src = ngx_create_temp_buf(ctx->tpool, 1024);
    if (ctx->line_src == NULL) {
        return NGX_ERROR;
    }

    ctx->line_dst = ngx_create_temp_buf(ctx->tpool, 1024);
    if (ctx->line_dst == NULL) {
        return NGX_ERROR;
    }

    if (ctx->saved) {
#if SUBS_DEBUG
        for (cl = ctx->saved; cl; cl = cl->next) {
            if (cl->buf) {
                ngx_log_debug3(NGX_LOG_DEBUG_HTTP, log, 0,
                        "subs in saved: %p, size:%uz, sync:%d",
                        cl->buf, ngx_buf_size(cl->buf), cl->buf->sync);
            }
        }
#endif
        ctx->line_in = duplicate_chains(ctx->saved, NULL, ctx->tpool);
        if (ctx->line_in == NULL) {
            return NGX_ERROR;
        }
    }

    return NGX_OK;
}


static ngx_int_t
ngx_http_subs_deep_copy_chain(ngx_pool_t *pool, ngx_chain_t **chain,
    ngx_chain_t *in)
{
    size_t           len;
    ngx_buf_t       *src;
    ngx_chain_t     *cl, **ll;

    ll = chain;

    for (cl = *chain; cl; cl = cl->next) {
        ll = &cl->next;
    }

    while (in) {
        cl = ngx_alloc_chain_link(pool);
        if (cl == NULL) {
            return NGX_ERROR;
        }

        src = in->buf;
        len = src->last - src->pos;

        if (len) {
            cl->buf = ngx_create_temp_buf(pool, len);
            if (cl->buf == NULL) {
                return NGX_ERROR;
            }

            cl->buf->last = ngx_copy(cl->buf->pos, src->pos, len);
            src->pos = src->last;
            
            cl->buf->last_buf = src->last_buf;
            cl->buf->tag = (ngx_buf_tag_t) &ngx_http_subs_filter_module;
        }
        else {
            cl->buf = src;
        }

        *ll = cl;
        ll = &cl->next;
        in = in->next;
    }

    *ll = NULL;

    return NGX_OK;
}


static ngx_int_t 
ngx_http_subs_body_filter_process_chain(ngx_http_request_t *r, ngx_chain_t *cl)
{
    size_t                     linesize;
    u_char                    *p, *linefeed;
    ngx_log_t                 *log;
    ngx_buf_t                 *b;
    ngx_int_t                  len, rc; 
    ngx_chain_t               *temp_cl;
    ngx_chain_t               *part_line_in_cl;
    ngx_http_subs_ctx_t       *ctx;
{'fraction_non_alphanumeric': 0.10180538555691554, 'fraction_numerical': 0.0030599755201958386, 'mean_word_length': 2.0531576980568014, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 311, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1757201', 'n_tokens_mistral': 12371, 'n_tokens_neox': 11846, 'n_words': 2110}