<commit_before>/** @file

  Transforms content using gzip or deflate

  @section license License

  Licensed to the Apache Software Foundation (ASF) under one
  or more contributor license agreements.  See the NOTICE file
  distributed with this work for additional information
  regarding copyright ownership.  The ASF licenses this file
  to you under the Apache License, Version 2.0 (the
  "License"); you may not use this file except in compliance
  with the License.  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
 */

//for INT64_MAX
#ifndef __STDC_LIMIT_MACROS
#define __STDC_LIMIT_MACROS
#endif

#include <string>
#include <string.h>
#include <zlib.h>
#include <ts/ts.h>
#include "debug_macros.h"
#include "misc.h"
#include "configuration.h"

using namespace std;
using namespace Gzip;

//FIXME: custom dictionaries would be nice. configurable/content-type?
//FIXME: look into autoscaling the compression level based on connection speed
// a gprs device might benefit from a higher compression ratio, whereas a desktop w. high bandwith
// might be served better with little or no compression at all
//FIXME: look into compressing from the task thread pool
//FIXME: make normalizing accept encoding configurable

// from mod_deflate:
// ZLIB's compression algorithm uses a
// 0-9 based scale that GZIP does where '1' is 'Best speed' 
// and '9' is 'Best compression'. Testing has proved level '6' 
// to be about the best level to use in an HTTP Server. 

const int ZLIB_COMPRESSION_LEVEL = 6;

int arg_idx_hooked;
int arg_idx_host_configuration;
int arg_idx_url_disallowed;


const char * global_hidden_header_name;
Configuration* config = NULL;
const char *dictionary = NULL;

static GzipData *
gzip_data_alloc(int compression_type)
{
  GzipData *data;
  int err;

  data = (GzipData *) TSmalloc(sizeof(GzipData));
  data->downstream_vio = NULL;
  data->downstream_buffer = NULL;
  data->downstream_reader = NULL;
  data->downstream_length = 0;
  data->state = transform_state_initialized;
  data->compression_type = compression_type;
  data->zstrm.next_in = Z_NULL;
  data->zstrm.avail_in = 0;
  data->zstrm.total_in = 0;
  data->zstrm.next_out = Z_NULL;
  data->zstrm.avail_out = 0;
  data->zstrm.total_out = 0;
  data->zstrm.zalloc = gzip_alloc;
  data->zstrm.zfree = gzip_free;
  data->zstrm.opaque = (voidpf) 0;
  data->zstrm.data_type = Z_ASCII;

  int window_bits = (compression_type == COMPRESSION_TYPE_GZIP) ? WINDOW_BITS_GZIP : WINDOW_BITS_DEFLATE;

  err = deflateInit2(&data->zstrm, ZLIB_COMPRESSION_LEVEL, Z_DEFLATED, window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY);

  if (err != Z_OK) {
    fatal("gzip-transform: ERROR: deflateInit (%d)!", err);
  }

  if (dictionary) {
    err = deflateSetDictionary(&data->zstrm, (const Bytef *) dictionary, strlen(dictionary));
    if (err != Z_OK) {
      fatal("gzip-transform: ERROR: deflateSetDictionary (%d)!", err);
    }
  }

  return data;
}


static void
gzip_data_destroy(GzipData * data)
{
  TSReleaseAssert(data);

  //deflateEnd returnvalue ignore is intentional
  //it would spew log on every client abort
  deflateEnd(&data->zstrm);

  if (data->downstream_buffer) {
    TSIOBufferDestroy(data->downstream_buffer);
  }

  TSfree(data);
}


//FIXME: ensure the headers are set/appended correctly
//FIXME: some things are potentially compressible. those responses
//FIXME: the etag alteration isn't proper. it should modify the value inside quotes
//       specify a very header..
static void
gzip_transform_init(TSCont contp, GzipData * data)
{
  //update the vary, content-encoding, and etag response headers
  //prepare the downstream for transforming

  TSVConn downstream_conn;
  TSMBuffer bufp;
  TSMLoc hdr_loc;
  TSMLoc ce_loc;                /* for the content encoding mime field */

  data->state = transform_state_output;

  TSHttpTxnTransformRespGet(data->txn, &bufp, &hdr_loc);

  //FIXME: Probably should check for errors 
  TSMimeHdrFieldCreate(bufp, hdr_loc, &ce_loc);
  TSMimeHdrFieldNameSet(bufp, hdr_loc, ce_loc, "Content-Encoding", -1);

  if (data->compression_type == COMPRESSION_TYPE_DEFLATE) {
    TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "deflate", -1);
  } else if (data->compression_type == COMPRESSION_TYPE_GZIP) {
    TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "gzip", -1);
  }

  TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
  TSHandleMLocRelease(bufp, hdr_loc, ce_loc);

  TSMimeHdrFieldCreate(bufp, hdr_loc, &ce_loc);
  TSMimeHdrFieldNameSet(bufp, hdr_loc, ce_loc, "Vary", -1);
  TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "Accept-Encoding", -1);
  TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
  TSHandleMLocRelease(bufp, hdr_loc, ce_loc);

  ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG);

  if (ce_loc) {
    int changetag = 1;
    int strl;
    const char *strv = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, -1, &strl);
    //do not alter weak etags.
    //FIXME: consider just making the etag weak for compressed content
    if (strl >= 2) {
      if ((strv[0] == 'w' || strv[0] == 'W') && strv[1] == '/') {
        changetag = 0;
      }
      if (changetag) {
        TSMimeHdrFieldValueAppend(bufp, hdr_loc, ce_loc, 0, "-df", 3);
      }

      TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
    }
  }

  TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);

  downstream_conn = TSTransformOutputVConnGet(contp);
  data->downstream_buffer = TSIOBufferCreate();
  data->downstream_reader = TSIOBufferReaderAlloc(data->downstream_buffer);
  data->downstream_vio = TSVConnWrite(downstream_conn, contp, data->downstream_reader, INT64_MAX);
}



static void
gzip_transform_one(GzipData * data, TSIOBufferReader upstream_reader, int amount)
{
  TSIOBufferBlock downstream_blkp;
  const char *upstream_buffer;
  char *downstream_buffer;
  int64_t upstream_length, downstream_length;
  int err;

  while (amount > 0) {
    downstream_blkp = TSIOBufferReaderStart(upstream_reader);
    if (!downstream_blkp) {
      error("couldn't get from IOBufferBlock");
      return;
    }
<commit_msg>TS-1455 memory leak when using the gzip plugin<commit_after>/** @file

  Transforms content using gzip or deflate

  @section license License

  Licensed to the Apache Software Foundation (ASF) under one
  or more contributor license agreements.  See the NOTICE file
  distributed with this work for additional information
  regarding copyright ownership.  The ASF licenses this file
  to you under the Apache License, Version 2.0 (the
  "License"); you may not use this file except in compliance
  with the License.  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
 */

//for INT64_MAX
#ifndef __STDC_LIMIT_MACROS
#define __STDC_LIMIT_MACROS
#endif

#include <string>
#include <string.h>
#include <zlib.h>
#include <ts/ts.h>
#include "debug_macros.h"
#include "misc.h"
#include "configuration.h"

using namespace std;
using namespace Gzip;

//FIXME: custom dictionaries would be nice. configurable/content-type?
//FIXME: look into autoscaling the compression level based on connection speed
// a gprs device might benefit from a higher compression ratio, whereas a desktop w. high bandwith
// might be served better with little or no compression at all
//FIXME: look into compressing from the task thread pool
//FIXME: make normalizing accept encoding configurable

// from mod_deflate:
// ZLIB's compression algorithm uses a
// 0-9 based scale that GZIP does where '1' is 'Best speed' 
// and '9' is 'Best compression'. Testing has proved level '6' 
// to be about the best level to use in an HTTP Server. 

const int ZLIB_COMPRESSION_LEVEL = 6;

int arg_idx_hooked;
int arg_idx_host_configuration;
int arg_idx_url_disallowed;


const char * global_hidden_header_name;
Configuration* config = NULL;
const char *dictionary = NULL;

static GzipData *
gzip_data_alloc(int compression_type)
{
  GzipData *data;
  int err;

  data = (GzipData *) TSmalloc(sizeof(GzipData));
  data->downstream_vio = NULL;
  data->downstream_buffer = NULL;
  data->downstream_reader = NULL;
  data->downstream_length = 0;
  data->state = transform_state_initialized;
  data->compression_type = compression_type;
  data->zstrm.next_in = Z_NULL;
  data->zstrm.avail_in = 0;
  data->zstrm.total_in = 0;
  data->zstrm.next_out = Z_NULL;
  data->zstrm.avail_out = 0;
  data->zstrm.total_out = 0;
  data->zstrm.zalloc = gzip_alloc;
  data->zstrm.zfree = gzip_free;
  data->zstrm.opaque = (voidpf) 0;
  data->zstrm.data_type = Z_ASCII;

  int window_bits = (compression_type == COMPRESSION_TYPE_GZIP) ? WINDOW_BITS_GZIP : WINDOW_BITS_DEFLATE;

  err = deflateInit2(&data->zstrm, ZLIB_COMPRESSION_LEVEL, Z_DEFLATED, window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY);

  if (err != Z_OK) {
    fatal("gzip-transform: ERROR: deflateInit (%d)!", err);
  }

  if (dictionary) {
    err = deflateSetDictionary(&data->zstrm, (const Bytef *) dictionary, strlen(dictionary));
    if (err != Z_OK) {
      fatal("gzip-transform: ERROR: deflateSetDictionary (%d)!", err);
    }
  }

  return data;
}


static void
gzip_data_destroy(GzipData * data)
{
  TSReleaseAssert(data);

  //deflateEnd returnvalue ignore is intentional
  //it would spew log on every client abort
  deflateEnd(&data->zstrm);

  if (data->downstream_buffer) {
    TSIOBufferDestroy(data->downstream_buffer);
  }

  TSfree(data);
}


//FIXME: ensure the headers are set/appended correctly
//FIXME: some things are potentially compressible. those responses
//FIXME: the etag alteration isn't proper. it should modify the value inside quotes
//       specify a very header..
static void
gzip_transform_init(TSCont contp, GzipData * data)
{
  //update the vary, content-encoding, and etag response headers
  //prepare the downstream for transforming

  TSVConn downstream_conn;
  TSMBuffer bufp;
  TSMLoc hdr_loc;
  TSMLoc ce_loc;                /* for the content encoding mime field */

  data->state = transform_state_output;

  TSHttpTxnTransformRespGet(data->txn, &bufp, &hdr_loc);

  //FIXME: Probably should check for errors 
  TSMimeHdrFieldCreate(bufp, hdr_loc, &ce_loc);
  TSMimeHdrFieldNameSet(bufp, hdr_loc, ce_loc, "Content-Encoding", -1);

  if (data->compression_type == COMPRESSION_TYPE_DEFLATE) {
    TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "deflate", -1);
  } else if (data->compression_type == COMPRESSION_TYPE_GZIP) {
    TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "gzip", -1);
  }

  TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
  TSHandleMLocRelease(bufp, hdr_loc, ce_loc);

  TSMimeHdrFieldCreate(bufp, hdr_loc, &ce_loc);
  TSMimeHdrFieldNameSet(bufp, hdr_loc, ce_loc, "Vary", -1);
  TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "Accept-Encoding", -1);
  TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
  TSHandleMLocRelease(bufp, hdr_loc, ce_loc);

  ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG);

  if (ce_loc) {
    int changetag = 1;
    int strl;
    const char *strv = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, -1, &strl);
    //do not alter weak etags.
    //FIXME: consider just making the etag weak for compressed content
    if (strl >= 2) {
      if ((strv[0] == 'w' || strv[0] == 'W') && strv[1] == '/') {
        changetag = 0;
      }
      if (changetag) {
        TSMimeHdrFieldValueAppend(bufp, hdr_loc, ce_loc, 0, "-df", 3);
      }
    }
    TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
  }

  TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);

  downstream_conn = TSTransformOutputVConnGet(contp);
  data->downstream_buffer = TSIOBufferCreate();
  data->downstream_reader = TSIOBufferReaderAlloc(data->downstream_buffer);
  data->downstream_vio = TSVConnWrite(downstream_conn, contp, data->downstream_reader, INT64_MAX);
}



static void
gzip_transform_one(GzipData * data, TSIOBufferReader upstream_reader, int amount)
{
  TSIOBufferBlock downstream_blkp;
  const char *upstream_buffer;
  char *downstream_buffer;
  int64_t upstream_length, downstream_length;
  int err;

  while (amount > 0) {
    downstream_blkp = TSIOBufferReaderStart(upstream_reader);
    if (!downstream_blkp) {
      error("couldn't get from IOBufferBlock");
      return;
    }

{'fraction_non_alphanumeric': 0.10086700727666821, 'fraction_numerical': 0.006657377302988079, 'mean_word_length': 4.2860065466448445, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6625569', 'n_tokens_mistral': 4597, 'n_tokens_neox': 4455, 'n_words': 1223}